diff options
Diffstat (limited to 'drivers/dma/ptdma')
| -rw-r--r-- | drivers/dma/ptdma/Kconfig | 13 | ||||
| -rw-r--r-- | drivers/dma/ptdma/Makefile | 10 | ||||
| -rw-r--r-- | drivers/dma/ptdma/ptdma-debugfs.c | 106 | ||||
| -rw-r--r-- | drivers/dma/ptdma/ptdma-dev.c | 309 | ||||
| -rw-r--r-- | drivers/dma/ptdma/ptdma-dmaengine.c | 411 | ||||
| -rw-r--r-- | drivers/dma/ptdma/ptdma-pci.c | 243 | ||||
| -rw-r--r-- | drivers/dma/ptdma/ptdma.h | 337 |
7 files changed, 0 insertions, 1429 deletions
diff --git a/drivers/dma/ptdma/Kconfig b/drivers/dma/ptdma/Kconfig deleted file mode 100644 index b430edd709f9..000000000000 --- a/drivers/dma/ptdma/Kconfig +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config AMD_PTDMA - tristate "AMD PassThru DMA Engine" - depends on X86_64 && PCI - select DMA_ENGINE - select DMA_VIRTUAL_CHANNELS - help - Enable support for the AMD PTDMA controller. This controller - provides DMA capabilities to perform high bandwidth memory to - memory and IO copy operations. It performs DMA transfer through - queue-based descriptor management. This DMA controller is intended - to be used with AMD Non-Transparent Bridge devices and not for - general purpose peripheral DMA. diff --git a/drivers/dma/ptdma/Makefile b/drivers/dma/ptdma/Makefile deleted file mode 100644 index ce5410268a9a..000000000000 --- a/drivers/dma/ptdma/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# AMD Passthru DMA driver -# - -obj-$(CONFIG_AMD_PTDMA) += ptdma.o - -ptdma-objs := ptdma-dev.o ptdma-dmaengine.o ptdma-debugfs.o - -ptdma-$(CONFIG_PCI) += ptdma-pci.o diff --git a/drivers/dma/ptdma/ptdma-debugfs.c b/drivers/dma/ptdma/ptdma-debugfs.c deleted file mode 100644 index c8307d3044a3..000000000000 --- a/drivers/dma/ptdma/ptdma-debugfs.c +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthrough DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta <sanju.mehta@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - */ - -#include <linux/debugfs.h> -#include <linux/seq_file.h> - -#include "ptdma.h" - -/* DebugFS helpers */ -#define RI_VERSION_NUM 0x0000003F - -#define RI_NUM_VQM 0x00078000 -#define RI_NVQM_SHIFT 15 - -static int pt_debugfs_info_show(struct seq_file *s, void *p) -{ - struct pt_device *pt = s->private; - unsigned int regval; - - seq_printf(s, "Device name: %s\n", dev_name(pt->dev)); - seq_printf(s, " # Queues: %d\n", 1); - seq_printf(s, " # Cmds: %d\n", pt->cmd_count); - - regval = ioread32(pt->io_regs + CMD_PT_VERSION); - - seq_printf(s, " Version: %d\n", regval & RI_VERSION_NUM); - seq_puts(s, " Engines:"); - seq_puts(s, "\n"); - seq_printf(s, " Queues: %d\n", (regval & RI_NUM_VQM) >> RI_NVQM_SHIFT); - - return 0; -} - -/* - * Return a formatted buffer containing the current - * statistics of queue for PTDMA - */ -static int pt_debugfs_stats_show(struct seq_file *s, void *p) -{ - struct pt_device *pt = s->private; - - seq_printf(s, "Total Interrupts Handled: %ld\n", pt->total_interrupts); - - return 0; -} - -static int pt_debugfs_queue_show(struct seq_file *s, void *p) -{ - struct pt_cmd_queue *cmd_q = s->private; - unsigned int regval; - - if (!cmd_q) - return 0; - - seq_printf(s, " Pass-Thru: %ld\n", cmd_q->total_pt_ops); - - regval = ioread32(cmd_q->reg_control + 0x000C); - - seq_puts(s, " Enabled Interrupts:"); - if (regval & INT_EMPTY_QUEUE) - seq_puts(s, " EMPTY"); - if (regval & INT_QUEUE_STOPPED) - seq_puts(s, " STOPPED"); - if (regval & INT_ERROR) - seq_puts(s, " ERROR"); - if (regval & INT_COMPLETION) - seq_puts(s, " COMPLETION"); - seq_puts(s, "\n"); - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_info); -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_queue); -DEFINE_SHOW_ATTRIBUTE(pt_debugfs_stats); - -void ptdma_debugfs_setup(struct pt_device *pt) -{ - struct pt_cmd_queue *cmd_q; - struct dentry *debugfs_q_instance; - - if (!debugfs_initialized()) - return; - - debugfs_create_file("info", 0400, pt->dma_dev.dbg_dev_root, pt, - &pt_debugfs_info_fops); - - debugfs_create_file("stats", 0400, pt->dma_dev.dbg_dev_root, pt, - &pt_debugfs_stats_fops); - - cmd_q = &pt->cmd_q; - - debugfs_q_instance = - debugfs_create_dir("q", pt->dma_dev.dbg_dev_root); - - debugfs_create_file("stats", 0400, debugfs_q_instance, cmd_q, - &pt_debugfs_queue_fops); -} diff --git a/drivers/dma/ptdma/ptdma-dev.c b/drivers/dma/ptdma/ptdma-dev.c deleted file mode 100644 index a2bf13ff18b6..000000000000 --- a/drivers/dma/ptdma/ptdma-dev.c +++ /dev/null @@ -1,309 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta <sanju.mehta@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - */ - -#include <linux/bitfield.h> -#include <linux/dma-mapping.h> -#include <linux/debugfs.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/pci.h> - -#include "ptdma.h" - -/* Human-readable error strings */ -static char *pt_error_codes[] = { - "", - "ERR 01: ILLEGAL_ENGINE", - "ERR 03: ILLEGAL_FUNCTION_TYPE", - "ERR 04: ILLEGAL_FUNCTION_MODE", - "ERR 06: ILLEGAL_FUNCTION_SIZE", - "ERR 08: ILLEGAL_FUNCTION_RSVD", - "ERR 09: ILLEGAL_BUFFER_LENGTH", - "ERR 10: VLSB_FAULT", - "ERR 11: ILLEGAL_MEM_ADDR", - "ERR 12: ILLEGAL_MEM_SEL", - "ERR 13: ILLEGAL_CONTEXT_ID", - "ERR 15: 0xF Reserved", - "ERR 18: CMD_TIMEOUT", - "ERR 19: IDMA0_AXI_SLVERR", - "ERR 20: IDMA0_AXI_DECERR", - "ERR 21: 0x15 Reserved", - "ERR 22: IDMA1_AXI_SLAVE_FAULT", - "ERR 23: IDMA1_AIXI_DECERR", - "ERR 24: 0x18 Reserved", - "ERR 27: 0x1B Reserved", - "ERR 38: ODMA0_AXI_SLVERR", - "ERR 39: ODMA0_AXI_DECERR", - "ERR 40: 0x28 Reserved", - "ERR 41: ODMA1_AXI_SLVERR", - "ERR 42: ODMA1_AXI_DECERR", - "ERR 43: LSB_PARITY_ERR", -}; - -static void pt_log_error(struct pt_device *d, int e) -{ - dev_err(d->dev, "PTDMA error: %s (0x%x)\n", pt_error_codes[e], e); -} - -void pt_start_queue(struct pt_cmd_queue *cmd_q) -{ - /* Turn on the run bit */ - iowrite32(cmd_q->qcontrol | CMD_Q_RUN, cmd_q->reg_control); -} - -void pt_stop_queue(struct pt_cmd_queue *cmd_q) -{ - /* Turn off the run bit */ - iowrite32(cmd_q->qcontrol & ~CMD_Q_RUN, cmd_q->reg_control); -} - -static int pt_core_execute_cmd(struct ptdma_desc *desc, struct pt_cmd_queue *cmd_q) -{ - bool soc = FIELD_GET(DWORD0_SOC, desc->dw0); - u8 *q_desc = (u8 *)&cmd_q->qbase[cmd_q->qidx]; - u32 tail; - unsigned long flags; - - if (soc) { - desc->dw0 |= FIELD_PREP(DWORD0_IOC, desc->dw0); - desc->dw0 &= ~DWORD0_SOC; - } - spin_lock_irqsave(&cmd_q->q_lock, flags); - - /* Copy 32-byte command descriptor to hw queue. */ - memcpy(q_desc, desc, 32); - cmd_q->qidx = (cmd_q->qidx + 1) % CMD_Q_LEN; - - /* The data used by this command must be flushed to memory */ - wmb(); - - /* Write the new tail address back to the queue register */ - tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); - iowrite32(tail, cmd_q->reg_control + 0x0004); - - /* Turn the queue back on using our cached control register */ - pt_start_queue(cmd_q); - spin_unlock_irqrestore(&cmd_q->q_lock, flags); - - return 0; -} - -int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, - struct pt_passthru_engine *pt_engine) -{ - struct ptdma_desc desc; - struct pt_device *pt = container_of(cmd_q, struct pt_device, cmd_q); - - cmd_q->cmd_error = 0; - cmd_q->total_pt_ops++; - memset(&desc, 0, sizeof(desc)); - desc.dw0 = CMD_DESC_DW0_VAL; - desc.length = pt_engine->src_len; - desc.src_lo = lower_32_bits(pt_engine->src_dma); - desc.dw3.src_hi = upper_32_bits(pt_engine->src_dma); - desc.dst_lo = lower_32_bits(pt_engine->dst_dma); - desc.dw5.dst_hi = upper_32_bits(pt_engine->dst_dma); - - if (cmd_q->int_en) - pt_core_enable_queue_interrupts(pt); - else - pt_core_disable_queue_interrupts(pt); - - return pt_core_execute_cmd(&desc, cmd_q); -} - -static void pt_do_cmd_complete(unsigned long data) -{ - struct pt_tasklet_data *tdata = (struct pt_tasklet_data *)data; - struct pt_cmd *cmd = tdata->cmd; - struct pt_cmd_queue *cmd_q = &cmd->pt->cmd_q; - u32 tail; - - if (cmd_q->cmd_error) { - /* - * Log the error and flush the queue by - * moving the head pointer - */ - tail = lower_32_bits(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE); - pt_log_error(cmd_q->pt, cmd_q->cmd_error); - iowrite32(tail, cmd_q->reg_control + 0x0008); - } - - cmd->pt_cmd_callback(cmd->data, cmd->ret); -} - -void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q) -{ - u32 status; - - status = ioread32(cmd_q->reg_control + 0x0010); - if (status) { - cmd_q->int_status = status; - cmd_q->q_status = ioread32(cmd_q->reg_control + 0x0100); - cmd_q->q_int_status = ioread32(cmd_q->reg_control + 0x0104); - - /* On error, only save the first error value */ - if ((status & INT_ERROR) && !cmd_q->cmd_error) - cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); - - /* Acknowledge the completion */ - iowrite32(status, cmd_q->reg_control + 0x0010); - pt_do_cmd_complete((ulong)&pt->tdata); - } -} - -static irqreturn_t pt_core_irq_handler(int irq, void *data) -{ - struct pt_device *pt = data; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - - pt_core_disable_queue_interrupts(pt); - pt->total_interrupts++; - pt_check_status_trans(pt, cmd_q); - pt_core_enable_queue_interrupts(pt); - return IRQ_HANDLED; -} - -int pt_core_init(struct pt_device *pt) -{ - char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - u32 dma_addr_lo, dma_addr_hi; - struct device *dev = pt->dev; - struct dma_pool *dma_pool; - int ret; - - /* Allocate a dma pool for the queue */ - snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q", dev_name(pt->dev)); - - dma_pool = dma_pool_create(dma_pool_name, dev, - PT_DMAPOOL_MAX_SIZE, - PT_DMAPOOL_ALIGN, 0); - if (!dma_pool) - return -ENOMEM; - - /* ptdma core initialisation */ - iowrite32(CMD_CONFIG_VHB_EN, pt->io_regs + CMD_CONFIG_OFFSET); - iowrite32(CMD_QUEUE_PRIO, pt->io_regs + CMD_QUEUE_PRIO_OFFSET); - iowrite32(CMD_TIMEOUT_DISABLE, pt->io_regs + CMD_TIMEOUT_OFFSET); - iowrite32(CMD_CLK_GATE_CONFIG, pt->io_regs + CMD_CLK_GATE_CTL_OFFSET); - iowrite32(CMD_CONFIG_REQID, pt->io_regs + CMD_REQID_CONFIG_OFFSET); - - cmd_q->pt = pt; - cmd_q->dma_pool = dma_pool; - spin_lock_init(&cmd_q->q_lock); - - /* Page alignment satisfies our needs for N <= 128 */ - cmd_q->qsize = Q_SIZE(Q_DESC_SIZE); - cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize, - &cmd_q->qbase_dma, - GFP_KERNEL); - if (!cmd_q->qbase) { - dev_err(dev, "unable to allocate command queue\n"); - ret = -ENOMEM; - goto e_destroy_pool; - } - - cmd_q->qidx = 0; - - /* Preset some register values */ - cmd_q->reg_control = pt->io_regs + CMD_Q_STATUS_INCR; - - /* Turn off the queues and disable interrupts until ready */ - pt_core_disable_queue_interrupts(pt); - - cmd_q->qcontrol = 0; /* Start with nothing */ - iowrite32(cmd_q->qcontrol, cmd_q->reg_control); - - ioread32(cmd_q->reg_control + 0x0104); - ioread32(cmd_q->reg_control + 0x0100); - - /* Clear the interrupt status */ - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - - /* Request an irq */ - ret = request_irq(pt->pt_irq, pt_core_irq_handler, 0, dev_name(pt->dev), pt); - if (ret) { - dev_err(dev, "unable to allocate an IRQ\n"); - goto e_free_dma; - } - - /* Update the device registers with queue information. */ - cmd_q->qcontrol &= ~CMD_Q_SIZE; - cmd_q->qcontrol |= FIELD_PREP(CMD_Q_SIZE, QUEUE_SIZE_VAL); - - cmd_q->qdma_tail = cmd_q->qbase_dma; - dma_addr_lo = lower_32_bits(cmd_q->qdma_tail); - iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0004); - iowrite32((u32)dma_addr_lo, cmd_q->reg_control + 0x0008); - - dma_addr_hi = upper_32_bits(cmd_q->qdma_tail); - cmd_q->qcontrol |= (dma_addr_hi << 16); - iowrite32(cmd_q->qcontrol, cmd_q->reg_control); - - pt_core_enable_queue_interrupts(pt); - - /* Register the DMA engine support */ - ret = pt_dmaengine_register(pt); - if (ret) - goto e_free_irq; - - /* Set up debugfs entries */ - ptdma_debugfs_setup(pt); - - return 0; - -e_free_irq: - free_irq(pt->pt_irq, pt); - -e_free_dma: - dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, cmd_q->qbase_dma); - -e_destroy_pool: - dma_pool_destroy(pt->cmd_q.dma_pool); - - return ret; -} - -void pt_core_destroy(struct pt_device *pt) -{ - struct device *dev = pt->dev; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - struct pt_cmd *cmd; - - /* Unregister the DMA engine */ - pt_dmaengine_unregister(pt); - - /* Disable and clear interrupts */ - pt_core_disable_queue_interrupts(pt); - - /* Turn off the run bit */ - pt_stop_queue(cmd_q); - - /* Clear the interrupt status */ - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - ioread32(cmd_q->reg_control + 0x0104); - ioread32(cmd_q->reg_control + 0x0100); - - free_irq(pt->pt_irq, pt); - - dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase, - cmd_q->qbase_dma); - - /* Flush the cmd queue */ - while (!list_empty(&pt->cmd)) { - /* Invoke the callback directly with an error code */ - cmd = list_first_entry(&pt->cmd, struct pt_cmd, entry); - list_del(&cmd->entry); - cmd->pt_cmd_callback(cmd->data, -ENODEV); - } -} diff --git a/drivers/dma/ptdma/ptdma-dmaengine.c b/drivers/dma/ptdma/ptdma-dmaengine.c deleted file mode 100644 index f79240734807..000000000000 --- a/drivers/dma/ptdma/ptdma-dmaengine.c +++ /dev/null @@ -1,411 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthrough DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta <sanju.mehta@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - */ - -#include "ptdma.h" -#include "../dmaengine.h" -#include "../virt-dma.h" - -static inline struct pt_dma_chan *to_pt_chan(struct dma_chan *dma_chan) -{ - return container_of(dma_chan, struct pt_dma_chan, vc.chan); -} - -static inline struct pt_dma_desc *to_pt_desc(struct virt_dma_desc *vd) -{ - return container_of(vd, struct pt_dma_desc, vd); -} - -static void pt_free_chan_resources(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - - vchan_free_chan_resources(&chan->vc); -} - -static void pt_synchronize(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - - vchan_synchronize(&chan->vc); -} - -static void pt_do_cleanup(struct virt_dma_desc *vd) -{ - struct pt_dma_desc *desc = to_pt_desc(vd); - struct pt_device *pt = desc->pt; - - kmem_cache_free(pt->dma_desc_cache, desc); -} - -static int pt_dma_start_desc(struct pt_dma_desc *desc) -{ - struct pt_passthru_engine *pt_engine; - struct pt_device *pt; - struct pt_cmd *pt_cmd; - struct pt_cmd_queue *cmd_q; - - desc->issued_to_hw = 1; - - pt_cmd = &desc->pt_cmd; - pt = pt_cmd->pt; - cmd_q = &pt->cmd_q; - pt_engine = &pt_cmd->passthru; - - pt->tdata.cmd = pt_cmd; - - /* Execute the command */ - pt_cmd->ret = pt_core_perform_passthru(cmd_q, pt_engine); - - return 0; -} - -static struct pt_dma_desc *pt_next_dma_desc(struct pt_dma_chan *chan) -{ - /* Get the next DMA descriptor on the active list */ - struct virt_dma_desc *vd = vchan_next_desc(&chan->vc); - - return vd ? to_pt_desc(vd) : NULL; -} - -static struct pt_dma_desc *pt_handle_active_desc(struct pt_dma_chan *chan, - struct pt_dma_desc *desc) -{ - struct dma_async_tx_descriptor *tx_desc; - struct virt_dma_desc *vd; - unsigned long flags; - - /* Loop over descriptors until one is found with commands */ - do { - if (desc) { - if (!desc->issued_to_hw) { - /* No errors, keep going */ - if (desc->status != DMA_ERROR) - return desc; - } - - tx_desc = &desc->vd.tx; - vd = &desc->vd; - } else { - tx_desc = NULL; - } - - spin_lock_irqsave(&chan->vc.lock, flags); - - if (desc) { - if (desc->status != DMA_COMPLETE) { - if (desc->status != DMA_ERROR) - desc->status = DMA_COMPLETE; - - dma_cookie_complete(tx_desc); - dma_descriptor_unmap(tx_desc); - list_del(&desc->vd.node); - } else { - /* Don't handle it twice */ - tx_desc = NULL; - } - } - - desc = pt_next_dma_desc(chan); - - spin_unlock_irqrestore(&chan->vc.lock, flags); - - if (tx_desc) { - dmaengine_desc_get_callback_invoke(tx_desc, NULL); - dma_run_dependencies(tx_desc); - vchan_vdesc_fini(vd); - } - } while (desc); - - return NULL; -} - -static void pt_cmd_callback(void *data, int err) -{ - struct pt_dma_desc *desc = data; - struct dma_chan *dma_chan; - struct pt_dma_chan *chan; - int ret; - - if (err == -EINPROGRESS) - return; - - dma_chan = desc->vd.tx.chan; - chan = to_pt_chan(dma_chan); - - if (err) - desc->status = DMA_ERROR; - - while (true) { - /* Check for DMA descriptor completion */ - desc = pt_handle_active_desc(chan, desc); - - /* Don't submit cmd if no descriptor or DMA is paused */ - if (!desc) - break; - - ret = pt_dma_start_desc(desc); - if (!ret) - break; - - desc->status = DMA_ERROR; - } -} - -static struct pt_dma_desc *pt_alloc_dma_desc(struct pt_dma_chan *chan, - unsigned long flags) -{ - struct pt_dma_desc *desc; - - desc = kmem_cache_zalloc(chan->pt->dma_desc_cache, GFP_NOWAIT); - if (!desc) - return NULL; - - vchan_tx_prep(&chan->vc, &desc->vd, flags); - - desc->pt = chan->pt; - desc->pt->cmd_q.int_en = !!(flags & DMA_PREP_INTERRUPT); - desc->issued_to_hw = 0; - desc->status = DMA_IN_PROGRESS; - - return desc; -} - -static struct pt_dma_desc *pt_create_desc(struct dma_chan *dma_chan, - dma_addr_t dst, - dma_addr_t src, - unsigned int len, - unsigned long flags) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_passthru_engine *pt_engine; - struct pt_dma_desc *desc; - struct pt_cmd *pt_cmd; - - desc = pt_alloc_dma_desc(chan, flags); - if (!desc) - return NULL; - - pt_cmd = &desc->pt_cmd; - pt_cmd->pt = chan->pt; - pt_engine = &pt_cmd->passthru; - pt_cmd->engine = PT_ENGINE_PASSTHRU; - pt_engine->src_dma = src; - pt_engine->dst_dma = dst; - pt_engine->src_len = len; - pt_cmd->pt_cmd_callback = pt_cmd_callback; - pt_cmd->data = desc; - - desc->len = len; - - return desc; -} - -static struct dma_async_tx_descriptor * -pt_prep_dma_memcpy(struct dma_chan *dma_chan, dma_addr_t dst, - dma_addr_t src, size_t len, unsigned long flags) -{ - struct pt_dma_desc *desc; - - desc = pt_create_desc(dma_chan, dst, src, len, flags); - if (!desc) - return NULL; - - return &desc->vd.tx; -} - -static struct dma_async_tx_descriptor * -pt_prep_dma_interrupt(struct dma_chan *dma_chan, unsigned long flags) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc; - - desc = pt_alloc_dma_desc(chan, flags); - if (!desc) - return NULL; - - return &desc->vd.tx; -} - -static void pt_issue_pending(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc; - unsigned long flags; - bool engine_is_idle = true; - - spin_lock_irqsave(&chan->vc.lock, flags); - - desc = pt_next_dma_desc(chan); - if (desc) - engine_is_idle = false; - - vchan_issue_pending(&chan->vc); - - desc = pt_next_dma_desc(chan); - - spin_unlock_irqrestore(&chan->vc.lock, flags); - - /* If there was nothing active, start processing */ - if (engine_is_idle && desc) - pt_cmd_callback(desc, 0); -} - -static enum dma_status -pt_tx_status(struct dma_chan *c, dma_cookie_t cookie, - struct dma_tx_state *txstate) -{ - struct pt_device *pt = to_pt_chan(c)->pt; - struct pt_cmd_queue *cmd_q = &pt->cmd_q; - - pt_check_status_trans(pt, cmd_q); - return dma_cookie_status(c, cookie, txstate); -} - -static int pt_pause(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - unsigned long flags; - - spin_lock_irqsave(&chan->vc.lock, flags); - pt_stop_queue(&chan->pt->cmd_q); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - return 0; -} - -static int pt_resume(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - struct pt_dma_desc *desc = NULL; - unsigned long flags; - - spin_lock_irqsave(&chan->vc.lock, flags); - pt_start_queue(&chan->pt->cmd_q); - desc = pt_next_dma_desc(chan); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - /* If there was something active, re-start */ - if (desc) - pt_cmd_callback(desc, 0); - - return 0; -} - -static int pt_terminate_all(struct dma_chan *dma_chan) -{ - struct pt_dma_chan *chan = to_pt_chan(dma_chan); - unsigned long flags; - struct pt_cmd_queue *cmd_q = &chan->pt->cmd_q; - LIST_HEAD(head); - - iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_control + 0x0010); - spin_lock_irqsave(&chan->vc.lock, flags); - vchan_get_all_descriptors(&chan->vc, &head); - spin_unlock_irqrestore(&chan->vc.lock, flags); - - vchan_dma_desc_free_list(&chan->vc, &head); - vchan_free_chan_resources(&chan->vc); - - return 0; -} - -int pt_dmaengine_register(struct pt_device *pt) -{ - struct pt_dma_chan *chan; - struct dma_device *dma_dev = &pt->dma_dev; - char *cmd_cache_name; - char *desc_cache_name; - int ret; - - pt->pt_dma_chan = devm_kzalloc(pt->dev, sizeof(*pt->pt_dma_chan), - GFP_KERNEL); - if (!pt->pt_dma_chan) - return -ENOMEM; - - cmd_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, - "%s-dmaengine-cmd-cache", - dev_name(pt->dev)); - if (!cmd_cache_name) - return -ENOMEM; - - desc_cache_name = devm_kasprintf(pt->dev, GFP_KERNEL, - "%s-dmaengine-desc-cache", - dev_name(pt->dev)); - if (!desc_cache_name) { - ret = -ENOMEM; - goto err_cache; - } - - pt->dma_desc_cache = kmem_cache_create(desc_cache_name, - sizeof(struct pt_dma_desc), 0, - SLAB_HWCACHE_ALIGN, NULL); - if (!pt->dma_desc_cache) { - ret = -ENOMEM; - goto err_cache; - } - - dma_dev->dev = pt->dev; - dma_dev->src_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; - dma_dev->dst_addr_widths = DMA_SLAVE_BUSWIDTH_64_BYTES; - dma_dev->directions = DMA_MEM_TO_MEM; - dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; - dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); - dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); - - /* - * PTDMA is intended to be used with the AMD NTB devices, hence - * marking it as DMA_PRIVATE. - */ - dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); - - INIT_LIST_HEAD(&dma_dev->channels); - - chan = pt->pt_dma_chan; - chan->pt = pt; - - /* Set base and prep routines */ - dma_dev->device_free_chan_resources = pt_free_chan_resources; - dma_dev->device_prep_dma_memcpy = pt_prep_dma_memcpy; - dma_dev->device_prep_dma_interrupt = pt_prep_dma_interrupt; - dma_dev->device_issue_pending = pt_issue_pending; - dma_dev->device_tx_status = pt_tx_status; - dma_dev->device_pause = pt_pause; - dma_dev->device_resume = pt_resume; - dma_dev->device_terminate_all = pt_terminate_all; - dma_dev->device_synchronize = pt_synchronize; - - chan->vc.desc_free = pt_do_cleanup; - vchan_init(&chan->vc, dma_dev); - - ret = dma_async_device_register(dma_dev); - if (ret) - goto err_reg; - - return 0; - -err_reg: - kmem_cache_destroy(pt->dma_desc_cache); - -err_cache: - kmem_cache_destroy(pt->dma_cmd_cache); - - return ret; -} - -void pt_dmaengine_unregister(struct pt_device *pt) -{ - struct dma_device *dma_dev = &pt->dma_dev; - - dma_async_device_unregister(dma_dev); - - kmem_cache_destroy(pt->dma_desc_cache); - kmem_cache_destroy(pt->dma_cmd_cache); -} diff --git a/drivers/dma/ptdma/ptdma-pci.c b/drivers/dma/ptdma/ptdma-pci.c deleted file mode 100644 index 22739ff0c3c5..000000000000 --- a/drivers/dma/ptdma/ptdma-pci.c +++ /dev/null @@ -1,243 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta <sanju.mehta@amd.com> - * Author: Tom Lendacky <thomas.lendacky@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - */ - -#include <linux/device.h> -#include <linux/dma-mapping.h> -#include <linux/delay.h> -#include <linux/interrupt.h> -#include <linux/kernel.h> -#include <linux/kthread.h> -#include <linux/module.h> -#include <linux/pci_ids.h> -#include <linux/pci.h> -#include <linux/spinlock.h> - -#include "ptdma.h" - -struct pt_msix { - int msix_count; - struct msix_entry msix_entry; -}; - -/* - * pt_alloc_struct - allocate and initialize the pt_device struct - * - * @dev: device struct of the PTDMA - */ -static struct pt_device *pt_alloc_struct(struct device *dev) -{ - struct pt_device *pt; - - pt = devm_kzalloc(dev, sizeof(*pt), GFP_KERNEL); - - if (!pt) - return NULL; - pt->dev = dev; - - INIT_LIST_HEAD(&pt->cmd); - - return pt; -} - -static int pt_get_msix_irqs(struct pt_device *pt) -{ - struct pt_msix *pt_msix = pt->pt_msix; - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - pt_msix->msix_entry.entry = 0; - - ret = pci_enable_msix_range(pdev, &pt_msix->msix_entry, 1, 1); - if (ret < 0) - return ret; - - pt_msix->msix_count = ret; - - pt->pt_irq = pt_msix->msix_entry.vector; - - return 0; -} - -static int pt_get_msi_irq(struct pt_device *pt) -{ - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - int ret; - - ret = pci_enable_msi(pdev); - if (ret) - return ret; - - pt->pt_irq = pdev->irq; - - return 0; -} - -static int pt_get_irqs(struct pt_device *pt) -{ - struct device *dev = pt->dev; - int ret; - - ret = pt_get_msix_irqs(pt); - if (!ret) - return 0; - - /* Couldn't get MSI-X vectors, try MSI */ - dev_err(dev, "could not enable MSI-X (%d), trying MSI\n", ret); - ret = pt_get_msi_irq(pt); - if (!ret) - return 0; - - /* Couldn't get MSI interrupt */ - dev_err(dev, "could not enable MSI (%d)\n", ret); - - return ret; -} - -static void pt_free_irqs(struct pt_device *pt) -{ - struct pt_msix *pt_msix = pt->pt_msix; - struct device *dev = pt->dev; - struct pci_dev *pdev = to_pci_dev(dev); - - if (pt_msix->msix_count) - pci_disable_msix(pdev); - else if (pt->pt_irq) - pci_disable_msi(pdev); - - pt->pt_irq = 0; -} - -static int pt_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) -{ - struct pt_device *pt; - struct pt_msix *pt_msix; - struct device *dev = &pdev->dev; - void __iomem * const *iomap_table; - int bar_mask; - int ret = -ENOMEM; - - pt = pt_alloc_struct(dev); - if (!pt) - goto e_err; - - pt_msix = devm_kzalloc(dev, sizeof(*pt_msix), GFP_KERNEL); - if (!pt_msix) - goto e_err; - - pt->pt_msix = pt_msix; - pt->dev_vdata = (struct pt_dev_vdata *)id->driver_data; - if (!pt->dev_vdata) { - ret = -ENODEV; - dev_err(dev, "missing driver data\n"); - goto e_err; - } - - ret = pcim_enable_device(pdev); - if (ret) { - dev_err(dev, "pcim_enable_device failed (%d)\n", ret); - goto e_err; - } - - bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); - ret = pcim_iomap_regions(pdev, bar_mask, "ptdma"); - if (ret) { - dev_err(dev, "pcim_iomap_regions failed (%d)\n", ret); - goto e_err; - } - - iomap_table = pcim_iomap_table(pdev); - if (!iomap_table) { - dev_err(dev, "pcim_iomap_table failed\n"); - ret = -ENOMEM; - goto e_err; - } - - pt->io_regs = iomap_table[pt->dev_vdata->bar]; - if (!pt->io_regs) { - dev_err(dev, "ioremap failed\n"); - ret = -ENOMEM; - goto e_err; - } - - ret = pt_get_irqs(pt); - if (ret) - goto e_err; - - pci_set_master(pdev); - - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48)); - if (ret) { - ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); - if (ret) { - dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", - ret); - goto e_err; - } - } - - dev_set_drvdata(dev, pt); - - if (pt->dev_vdata) - ret = pt_core_init(pt); - - if (ret) - goto e_err; - - return 0; - -e_err: - dev_err(dev, "initialization failed ret = %d\n", ret); - - return ret; -} - -static void pt_pci_remove(struct pci_dev *pdev) -{ - struct device *dev = &pdev->dev; - struct pt_device *pt = dev_get_drvdata(dev); - - if (!pt) - return; - - if (pt->dev_vdata) - pt_core_destroy(pt); - - pt_free_irqs(pt); -} - -static const struct pt_dev_vdata dev_vdata[] = { - { - .bar = 2, - }, -}; - -static const struct pci_device_id pt_pci_table[] = { - { PCI_VDEVICE(AMD, 0x1498), (kernel_ulong_t)&dev_vdata[0] }, - /* Last entry must be zero */ - { 0, } -}; -MODULE_DEVICE_TABLE(pci, pt_pci_table); - -static struct pci_driver pt_pci_driver = { - .name = "ptdma", - .id_table = pt_pci_table, - .probe = pt_pci_probe, - .remove = pt_pci_remove, -}; - -module_pci_driver(pt_pci_driver); - -MODULE_AUTHOR("Sanjay R Mehta <sanju.mehta@amd.com>"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AMD PassThru DMA driver"); diff --git a/drivers/dma/ptdma/ptdma.h b/drivers/dma/ptdma/ptdma.h deleted file mode 100644 index 21b4bf895200..000000000000 --- a/drivers/dma/ptdma/ptdma.h +++ /dev/null @@ -1,337 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * AMD Passthru DMA device driver - * -- Based on the CCP driver - * - * Copyright (C) 2016,2021 Advanced Micro Devices, Inc. - * - * Author: Sanjay R Mehta <sanju.mehta@amd.com> - * Author: Tom Lendacky <thomas.lendacky@amd.com> - * Author: Gary R Hook <gary.hook@amd.com> - */ - -#ifndef __PT_DEV_H__ -#define __PT_DEV_H__ - -#include <linux/device.h> -#include <linux/dmaengine.h> -#include <linux/pci.h> -#include <linux/spinlock.h> -#include <linux/mutex.h> -#include <linux/list.h> -#include <linux/wait.h> -#include <linux/dmapool.h> - -#include "../virt-dma.h" - -#define MAX_PT_NAME_LEN 16 -#define MAX_DMAPOOL_NAME_LEN 32 - -#define MAX_HW_QUEUES 1 -#define MAX_CMD_QLEN 100 - -#define PT_ENGINE_PASSTHRU 5 - -/* Register Mappings */ -#define IRQ_MASK_REG 0x040 -#define IRQ_STATUS_REG 0x200 - -#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f) - -#define CMD_QUEUE_PRIO_OFFSET 0x00 -#define CMD_REQID_CONFIG_OFFSET 0x04 -#define CMD_TIMEOUT_OFFSET 0x08 -#define CMD_PT_VERSION 0x10 - -#define CMD_Q_CONTROL_BASE 0x0000 -#define CMD_Q_TAIL_LO_BASE 0x0004 -#define CMD_Q_HEAD_LO_BASE 0x0008 -#define CMD_Q_INT_ENABLE_BASE 0x000C -#define CMD_Q_INTERRUPT_STATUS_BASE 0x0010 - -#define CMD_Q_STATUS_BASE 0x0100 -#define CMD_Q_INT_STATUS_BASE 0x0104 -#define CMD_Q_DMA_STATUS_BASE 0x0108 -#define CMD_Q_DMA_READ_STATUS_BASE 0x010C -#define CMD_Q_DMA_WRITE_STATUS_BASE 0x0110 -#define CMD_Q_ABORT_BASE 0x0114 -#define CMD_Q_AX_CACHE_BASE 0x0118 - -#define CMD_CONFIG_OFFSET 0x1120 -#define CMD_CLK_GATE_CTL_OFFSET 0x6004 - -#define CMD_DESC_DW0_VAL 0x500012 - -/* Address offset for virtual queue registers */ -#define CMD_Q_STATUS_INCR 0x1000 - -/* Bit masks */ -#define CMD_CONFIG_REQID 0 -#define CMD_TIMEOUT_DISABLE 0 -#define CMD_CLK_DYN_GATING_DIS 0 -#define CMD_CLK_SW_GATE_MODE 0 -#define CMD_CLK_GATE_CTL 0 -#define CMD_QUEUE_PRIO GENMASK(2, 1) -#define CMD_CONFIG_VHB_EN BIT(0) -#define CMD_CLK_DYN_GATING_EN BIT(0) -#define CMD_CLK_HW_GATE_MODE BIT(0) -#define CMD_CLK_GATE_ON_DELAY BIT(12) -#define CMD_CLK_GATE_OFF_DELAY BIT(12) - -#define CMD_CLK_GATE_CONFIG (CMD_CLK_GATE_CTL | \ - CMD_CLK_HW_GATE_MODE | \ - CMD_CLK_GATE_ON_DELAY | \ - CMD_CLK_DYN_GATING_EN | \ - CMD_CLK_GATE_OFF_DELAY) - -#define CMD_Q_LEN 32 -#define CMD_Q_RUN BIT(0) -#define CMD_Q_HALT BIT(1) -#define CMD_Q_MEM_LOCATION BIT(2) -#define CMD_Q_SIZE_MASK GENMASK(4, 0) -#define CMD_Q_SIZE GENMASK(7, 3) -#define CMD_Q_SHIFT GENMASK(1, 0) -#define QUEUE_SIZE_VAL ((ffs(CMD_Q_LEN) - 2) & \ - CMD_Q_SIZE_MASK) -#define Q_PTR_MASK (2 << (QUEUE_SIZE_VAL + 5) - 1) -#define Q_DESC_SIZE sizeof(struct ptdma_desc) -#define Q_SIZE(n) (CMD_Q_LEN * (n)) - -#define INT_COMPLETION BIT(0) -#define INT_ERROR BIT(1) -#define INT_QUEUE_STOPPED BIT(2) -#define INT_EMPTY_QUEUE BIT(3) -#define SUPPORTED_INTERRUPTS (INT_COMPLETION | INT_ERROR) - -/****** Local Storage Block ******/ -#define LSB_START 0 -#define LSB_END 127 -#define LSB_COUNT (LSB_END - LSB_START + 1) - -#define PT_DMAPOOL_MAX_SIZE 64 -#define PT_DMAPOOL_ALIGN BIT(5) - -#define PT_PASSTHRU_BLOCKSIZE 512 - -struct pt_device; - -struct pt_tasklet_data { - struct completion completion; - struct pt_cmd *cmd; -}; - -/* - * struct pt_passthru_engine - pass-through operation - * without performing DMA mapping - * @mask: mask to be applied to data - * @mask_len: length in bytes of mask - * @src_dma: data to be used for this operation - * @dst_dma: data produced by this operation - * @src_len: length in bytes of data used for this operation - * - * Variables required to be set when calling pt_enqueue_cmd(): - * - bit_mod, byte_swap, src, dst, src_len - * - mask, mask_len if bit_mod is not PT_PASSTHRU_BITWISE_NOOP - */ -struct pt_passthru_engine { - dma_addr_t mask; - u32 mask_len; /* In bytes */ - - dma_addr_t src_dma, dst_dma; - u64 src_len; /* In bytes */ -}; - -/* - * struct pt_cmd - PTDMA operation request - * @entry: list element - * @work: work element used for callbacks - * @pt: PT device to be run on - * @ret: operation return code - * @flags: cmd processing flags - * @engine: PTDMA operation to perform (passthru) - * @engine_error: PT engine return code - * @passthru: engine specific structures, refer to specific engine struct below - * @callback: operation completion callback function - * @data: parameter value to be supplied to the callback function - * - * Variables required to be set when calling pt_enqueue_cmd(): - * - engine, callback - * - See the operation structures below for what is required for each - * operation. - */ -struct pt_cmd { - struct list_head entry; - struct work_struct work; - struct pt_device *pt; - int ret; - u32 engine; - u32 engine_error; - struct pt_passthru_engine passthru; - /* Completion callback support */ - void (*pt_cmd_callback)(void *data, int err); - void *data; -}; - -struct pt_dma_desc { - struct virt_dma_desc vd; - struct pt_device *pt; - enum dma_status status; - size_t len; - bool issued_to_hw; - struct pt_cmd pt_cmd; -}; - -struct pt_dma_chan { - struct virt_dma_chan vc; - struct pt_device *pt; -}; - -struct pt_cmd_queue { - struct pt_device *pt; - - /* Queue dma pool */ - struct dma_pool *dma_pool; - - /* Queue base address (not neccessarily aligned)*/ - struct ptdma_desc *qbase; - - /* Aligned queue start address (per requirement) */ - spinlock_t q_lock ____cacheline_aligned; - unsigned int qidx; - - unsigned int qsize; - dma_addr_t qbase_dma; - dma_addr_t qdma_tail; - - unsigned int active; - unsigned int suspended; - - /* Interrupt flag */ - bool int_en; - - /* Register addresses for queue */ - void __iomem *reg_control; - u32 qcontrol; /* Cached control register */ - - /* Status values from job */ - u32 int_status; - u32 q_status; - u32 q_int_status; - u32 cmd_error; - /* Queue Statistics */ - unsigned long total_pt_ops; -} ____cacheline_aligned; - -struct pt_device { - struct list_head entry; - - unsigned int ord; - char name[MAX_PT_NAME_LEN]; - - struct device *dev; - - /* Bus specific device information */ - struct pt_msix *pt_msix; - - struct pt_dev_vdata *dev_vdata; - - unsigned int pt_irq; - - /* I/O area used for device communication */ - void __iomem *io_regs; - - spinlock_t cmd_lock ____cacheline_aligned; - unsigned int cmd_count; - struct list_head cmd; - - /* - * The command queue. This represent the queue available on the - * PTDMA that are available for processing cmds - */ - struct pt_cmd_queue cmd_q; - - /* Support for the DMA Engine capabilities */ - struct dma_device dma_dev; - struct pt_dma_chan *pt_dma_chan; - struct kmem_cache *dma_cmd_cache; - struct kmem_cache *dma_desc_cache; - - wait_queue_head_t lsb_queue; - - /* Device Statistics */ - unsigned long total_interrupts; - - struct pt_tasklet_data tdata; -}; - -/* - * descriptor for PTDMA commands - * 8 32-bit words: - * word 0: function; engine; control bits - * word 1: length of source data - * word 2: low 32 bits of source pointer - * word 3: upper 16 bits of source pointer; source memory type - * word 4: low 32 bits of destination pointer - * word 5: upper 16 bits of destination pointer; destination memory type - * word 6: reserved 32 bits - * word 7: reserved 32 bits - */ - -#define DWORD0_SOC BIT(0) -#define DWORD0_IOC BIT(1) - -struct dword3 { - unsigned int src_hi:16; - unsigned int src_mem:2; - unsigned int lsb_cxt_id:8; - unsigned int rsvd1:5; - unsigned int fixed:1; -}; - -struct dword5 { - unsigned int dst_hi:16; - unsigned int dst_mem:2; - unsigned int rsvd1:13; - unsigned int fixed:1; -}; - -struct ptdma_desc { - u32 dw0; - u32 length; - u32 src_lo; - struct dword3 dw3; - u32 dst_lo; - struct dword5 dw5; - __le32 rsvd1; - __le32 rsvd2; -}; - -/* Structure to hold PT device data */ -struct pt_dev_vdata { - const unsigned int bar; -}; - -int pt_dmaengine_register(struct pt_device *pt); -void pt_dmaengine_unregister(struct pt_device *pt); - -void ptdma_debugfs_setup(struct pt_device *pt); -int pt_core_init(struct pt_device *pt); -void pt_core_destroy(struct pt_device *pt); - -int pt_core_perform_passthru(struct pt_cmd_queue *cmd_q, - struct pt_passthru_engine *pt_engine); - -void pt_check_status_trans(struct pt_device *pt, struct pt_cmd_queue *cmd_q); -void pt_start_queue(struct pt_cmd_queue *cmd_q); -void pt_stop_queue(struct pt_cmd_queue *cmd_q); - -static inline void pt_core_disable_queue_interrupts(struct pt_device *pt) -{ - iowrite32(0, pt->cmd_q.reg_control + 0x000C); -} - -static inline void pt_core_enable_queue_interrupts(struct pt_device *pt) -{ - iowrite32(SUPPORTED_INTERRUPTS, pt->cmd_q.reg_control + 0x000C); -} -#endif |