drm/amdkfd: Add initial VI support for DQM

This patch starts to add support for the VI APU in the DQM module.

Because most (more than 90%) of the DQM code is shared among AMD's APUs, we
chose a design that performs most/all the code in the shared DQM file
(kfd_device_queue_manager.c). If there is H/W specific code to be executed,
than it is written in an asic-specific extension function for that H/W.

That asic-specific extension function is called from the shared function at the
appropriate time. This requires that for every asic-specific extension function
that is implemented in a specific ASIC, there will be an equivalent
implementation in ALL ASICs, even if those implementations are just stubs.

That way we achieve:

- Maintainability: by having one copy of most of the code, we only need to
  fix bugs at one locations

- Readability: very clear what is the shared code and what is done per ASIC

- Extensibility: very easy to add new H/W specific files/functions

Signed-off-by: Ben Goz <ben.goz@amd.com>
Signed-off-by: Oded Gabbay <oded.gabbay@amd.com>
Reviewed-by: Alex Deucher <alexander.deucher@amd.com>

5 files changed, 238 insertions, 86 deletions

diff --git a/drivers/gpu/drm/amd/amdkfd/Makefile b/drivers/gpu/drm/amd/amdkfd/Makefile
index bc6053f8b843..7558683c6935 100644
--- a/drivers/gpu/drm/amd/amdkfd/Makefile
+++ b/drivers/gpu/drm/amd/amdkfd/Makefile
@@ -10,6 +10,7 @@ amdkfd-y	:= kfd_module.o kfd_device.o kfd_chardev.o kfd_topology.o \
 		kfd_mqd_manager_cik.o kfd_mqd_manager_vi.o \
 		kfd_kernel_queue.o kfd_packet_manager.o \
 		kfd_process_queue_manager.o kfd_device_queue_manager.o \
+		kfd_device_queue_manager_cik.o kfd_device_queue_manager_vi.o \
 		kfd_interrupt.o
 
 obj-$(CONFIG_HSA_AMD)	+= amdkfd.o
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 12c84488551e..b2016245073e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -61,7 +61,7 @@ enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
 	return KFD_MQD_TYPE_CP;
 }
 
-static inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
+inline unsigned int get_pipes_num(struct device_queue_manager *dqm)
 {
 	BUG_ON(!dqm || !dqm->dev);
 	return dqm->dev->shared_resources.compute_pipe_count;
@@ -78,7 +78,7 @@ static inline unsigned int get_pipes_num_cpsch(void)
 	return PIPE_PER_ME_CP_SCHEDULING;
 }
 
-static inline unsigned int
+inline unsigned int
 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 {
 	uint32_t nybble;
@@ -88,7 +88,7 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 	return nybble;
 }
 
-static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
+inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 {
 	unsigned int shared_base;
 
@@ -97,41 +97,7 @@ static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 	return shared_base;
 }
 
-static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble);
-static void init_process_memory(struct device_queue_manager *dqm,
-				struct qcm_process_device *qpd)
-{
-	struct kfd_process_device *pdd;
-	unsigned int temp;
-
-	BUG_ON(!dqm || !qpd);
-
-	pdd = qpd_to_pdd(qpd);
-
-	/* check if sh_mem_config register already configured */
-	if (qpd->sh_mem_config == 0) {
-		qpd->sh_mem_config =
-			ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
-			DEFAULT_MTYPE(MTYPE_NONCACHED) |
-			APE1_MTYPE(MTYPE_NONCACHED);
-		qpd->sh_mem_ape1_limit = 0;
-		qpd->sh_mem_ape1_base = 0;
-	}
-
-	if (qpd->pqm->process->is_32bit_user_mode) {
-		temp = get_sh_mem_bases_32(pdd);
-		qpd->sh_mem_bases = SHARED_BASE(temp);
-		qpd->sh_mem_config |= PTR32;
-	} else {
-		temp = get_sh_mem_bases_nybble_64(pdd);
-		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
-	}
-
-	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
-		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
-}
-
-static void program_sh_mem_settings(struct device_queue_manager *dqm,
+void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	return kfd2kgd->program_sh_mem_settings(dqm->dev->kgd, qpd->vmid,
@@ -391,6 +357,7 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	struct device_process_node *n;
+	int retval;
 
 	BUG_ON(!dqm || !qpd);
 
@@ -405,12 +372,13 @@ static int register_process_nocpsch(struct device_queue_manager *dqm,
 	mutex_lock(&dqm->lock);
 	list_add(&n->list, &dqm->queues);
 
-	init_process_memory(dqm, qpd);
+	retval = dqm->ops_asic_specific.register_process(dqm, qpd);
+
 	dqm->processes_count++;
 
 	mutex_unlock(&dqm->lock);
 
-	return 0;
+	return retval;
 }
 
 static int unregister_process_nocpsch(struct device_queue_manager *dqm,
@@ -455,34 +423,7 @@ set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
 						vmid);
 }
 
-static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
-{
-	/* In 64-bit mode, we can only control the top 3 bits of the LDS,
-	 * scratch and GPUVM apertures.
-	 * The hardware fills in the remaining 59 bits according to the
-	 * following pattern:
-	 * LDS:		X0000000'00000000 - X0000001'00000000 (4GB)
-	 * Scratch:	X0000001'00000000 - X0000002'00000000 (4GB)
-	 * GPUVM:	Y0010000'00000000 - Y0020000'00000000 (1TB)
-	 *
-	 * (where X/Y is the configurable nybble with the low-bit 0)
-	 *
-	 * LDS and scratch will have the same top nybble programmed in the
-	 * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
-	 * GPUVM can have a different top nybble programmed in the
-	 * top 3 bits of SH_MEM_BASES.SHARED_BASE.
-	 * We don't bother to support different top nybbles
-	 * for LDS/Scratch and GPUVM.
-	 */
-
-	BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
-		top_address_nybble == 0);
-
-	return PRIVATE_BASE(top_address_nybble << 12) |
-			SHARED_BASE(top_address_nybble << 12);
-}
-
-static int init_pipelines(struct device_queue_manager *dqm,
+int init_pipelines(struct device_queue_manager *dqm,
 			unsigned int pipes_num, unsigned int first_pipe)
 {
 	void *hpdptr;
@@ -715,7 +656,7 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 	dqm->queue_count = dqm->processes_count = 0;
 	dqm->sdma_queue_count = 0;
 	dqm->active_runlist = false;
-	retval = init_pipelines(dqm, get_pipes_num(dqm), 0);
+	retval = dqm->ops_asic_specific.initialize(dqm);
 	if (retval != 0)
 		goto fail_init_pipelines;
 
@@ -1035,8 +976,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 				   void __user *alternate_aperture_base,
 				   uint64_t alternate_aperture_size)
 {
-	uint32_t default_mtype;
-	uint32_t ape1_mtype;
+	bool retval;
 
 	pr_debug("kfd: In func %s\n", __func__);
 
@@ -1073,18 +1013,13 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 		qpd->sh_mem_ape1_limit = limit >> 16;
 	}
 
-	default_mtype = (default_policy == cache_policy_coherent) ?
-			MTYPE_NONCACHED :
-			MTYPE_CACHED;
-
-	ape1_mtype = (alternate_policy == cache_policy_coherent) ?
-			MTYPE_NONCACHED :
-			MTYPE_CACHED;
-
-	qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
-			| ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
-			| DEFAULT_MTYPE(default_mtype)
-			| APE1_MTYPE(ape1_mtype);
+	retval = dqm->ops_asic_specific.set_cache_memory_policy(
+			dqm,
+			qpd,
+			default_policy,
+			alternate_policy,
+			alternate_aperture_base,
+			alternate_aperture_size);
 
 	if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
 		program_sh_mem_settings(dqm, qpd);
@@ -1094,7 +1029,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm,
 		qpd->sh_mem_ape1_limit);
 
 	mutex_unlock(&dqm->lock);
-	return true;
+	return retval;
 
 out:
 	mutex_unlock(&dqm->lock);
@@ -1107,6 +1042,8 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 
 	BUG_ON(!dev);
 
+	pr_debug("kfd: loading device queue manager\n");
+
 	dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
 	if (!dqm)
 		return NULL;
@@ -1149,6 +1086,13 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
 		break;
 	}
 
+	switch (dev->device_info->asic_family) {
+	case CHIP_CARRIZO:
+		device_queue_manager_init_vi(&dqm->ops_asic_specific);
+	case CHIP_KAVERI:
+		device_queue_manager_init_cik(&dqm->ops_asic_specific);
+	}
+
 	if (dqm->ops.initialize(dqm) != 0) {
 		kfree(dqm);
 		return NULL;
@@ -1164,4 +1108,3 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm)
 	dqm->ops.uninitialize(dqm);
 	kfree(dqm);
 }
-
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 72d2ca056e19..19347956eeb9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -134,6 +134,7 @@ struct device_queue_manager_ops {
 
 struct device_queue_manager {
 	struct device_queue_manager_ops ops;
+	struct device_queue_manager_ops ops_asic_specific;
 
 	struct mqd_manager	*mqds[KFD_MQD_TYPE_MAX];
 	struct packet_manager	packets;
@@ -155,6 +156,14 @@ struct device_queue_manager {
 	bool			active_runlist;
 };
 
-
+void device_queue_manager_init_cik(struct device_queue_manager_ops *ops);
+void device_queue_manager_init_vi(struct device_queue_manager_ops *ops);
+void program_sh_mem_settings(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd);
+inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *qpd);
+inline unsigned int get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd);
+int init_pipelines(struct device_queue_manager *dqm,
+		unsigned int pipes_num, unsigned int first_pipe);
+inline unsigned int get_pipes_num(struct device_queue_manager *dqm);
 
 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
new file mode 100644
index 000000000000..6b072466e2a6
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_cik.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_device_queue_manager.h"
+#include "cik_regs.h"
+
+static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
+				   struct qcm_process_device *qpd,
+				   enum cache_policy default_policy,
+				   enum cache_policy alternate_policy,
+				   void __user *alternate_aperture_base,
+				   uint64_t alternate_aperture_size);
+static int register_process_cik(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd);
+static int initialize_cpsch_cik(struct device_queue_manager *dqm);
+
+void device_queue_manager_init_cik(struct device_queue_manager_ops *ops)
+{
+	ops->set_cache_memory_policy = set_cache_memory_policy_cik;
+	ops->register_process = register_process_cik;
+	ops->initialize = initialize_cpsch_cik;
+}
+
+static uint32_t compute_sh_mem_bases_64bit(unsigned int top_address_nybble)
+{
+	/* In 64-bit mode, we can only control the top 3 bits of the LDS,
+	 * scratch and GPUVM apertures.
+	 * The hardware fills in the remaining 59 bits according to the
+	 * following pattern:
+	 * LDS:		X0000000'00000000 - X0000001'00000000 (4GB)
+	 * Scratch:	X0000001'00000000 - X0000002'00000000 (4GB)
+	 * GPUVM:	Y0010000'00000000 - Y0020000'00000000 (1TB)
+	 *
+	 * (where X/Y is the configurable nybble with the low-bit 0)
+	 *
+	 * LDS and scratch will have the same top nybble programmed in the
+	 * top 3 bits of SH_MEM_BASES.PRIVATE_BASE.
+	 * GPUVM can have a different top nybble programmed in the
+	 * top 3 bits of SH_MEM_BASES.SHARED_BASE.
+	 * We don't bother to support different top nybbles
+	 * for LDS/Scratch and GPUVM.
+	 */
+
+	BUG_ON((top_address_nybble & 1) || top_address_nybble > 0xE ||
+		top_address_nybble == 0);
+
+	return PRIVATE_BASE(top_address_nybble << 12) |
+			SHARED_BASE(top_address_nybble << 12);
+}
+
+static bool set_cache_memory_policy_cik(struct device_queue_manager *dqm,
+				   struct qcm_process_device *qpd,
+				   enum cache_policy default_policy,
+				   enum cache_policy alternate_policy,
+				   void __user *alternate_aperture_base,
+				   uint64_t alternate_aperture_size)
+{
+	uint32_t default_mtype;
+	uint32_t ape1_mtype;
+
+	default_mtype = (default_policy == cache_policy_coherent) ?
+			MTYPE_NONCACHED :
+			MTYPE_CACHED;
+
+	ape1_mtype = (alternate_policy == cache_policy_coherent) ?
+			MTYPE_NONCACHED :
+			MTYPE_CACHED;
+
+	qpd->sh_mem_config = (qpd->sh_mem_config & PTR32)
+			| ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
+			| DEFAULT_MTYPE(default_mtype)
+			| APE1_MTYPE(ape1_mtype);
+
+	return true;
+}
+
+static int register_process_cik(struct device_queue_manager *dqm,
+		struct qcm_process_device *qpd)
+{
+	struct kfd_process_device *pdd;
+	unsigned int temp;
+
+	BUG_ON(!dqm || !qpd);
+
+	pdd = qpd_to_pdd(qpd);
+
+	/* check if sh_mem_config register already configured */
+	if (qpd->sh_mem_config == 0) {
+		qpd->sh_mem_config =
+			ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED) |
+			DEFAULT_MTYPE(MTYPE_NONCACHED) |
+			APE1_MTYPE(MTYPE_NONCACHED);
+		qpd->sh_mem_ape1_limit = 0;
+		qpd->sh_mem_ape1_base = 0;
+	}
+
+	if (qpd->pqm->process->is_32bit_user_mode) {
+		temp = get_sh_mem_bases_32(pdd);
+		qpd->sh_mem_bases = SHARED_BASE(temp);
+		qpd->sh_mem_config |= PTR32;
+	} else {
+		temp = get_sh_mem_bases_nybble_64(pdd);
+		qpd->sh_mem_bases = compute_sh_mem_bases_64bit(temp);
+	}
+
+	pr_debug("kfd: is32bit process: %d sh_mem_bases nybble: 0x%X and register 0x%X\n",
+		qpd->pqm->process->is_32bit_user_mode, temp, qpd->sh_mem_bases);
+
+	return 0;
+}
+
+static int initialize_cpsch_cik(struct device_queue_manager *dqm)
+{
+	return init_pipelines(dqm, get_pipes_num(dqm), 0);
+}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
new file mode 100644
index 000000000000..20553dcd257d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_vi.c
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2014 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "kfd_device_queue_manager.h"
+
+static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
+				   struct qcm_process_device *qpd,
+				   enum cache_policy default_policy,
+				   enum cache_policy alternate_policy,
+				   void __user *alternate_aperture_base,
+				   uint64_t alternate_aperture_size);
+static int register_process_vi(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd);
+static int initialize_cpsch_vi(struct device_queue_manager *dqm);
+
+void device_queue_manager_init_vi(struct device_queue_manager_ops *ops)
+{
+	pr_warn("amdkfd: VI DQM is not currently supported\n");
+
+	ops->set_cache_memory_policy = set_cache_memory_policy_vi;
+	ops->register_process = register_process_vi;
+	ops->initialize = initialize_cpsch_vi;
+}
+
+static bool set_cache_memory_policy_vi(struct device_queue_manager *dqm,
+				   struct qcm_process_device *qpd,
+				   enum cache_policy default_policy,
+				   enum cache_policy alternate_policy,
+				   void __user *alternate_aperture_base,
+				   uint64_t alternate_aperture_size)
+{
+	return false;
+}
+
+static int register_process_vi(struct device_queue_manager *dqm,
+					struct qcm_process_device *qpd)
+{
+	return -1;
+}
+
+static int initialize_cpsch_vi(struct device_queue_manager *dqm)
+{
+	return 0;
+}