summaryrefslogtreecommitdiff
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig3
-rw-r--r--drivers/dma/dmaengine.c60
-rw-r--r--drivers/dma/dmatest.c15
-rw-r--r--drivers/dma/idxd/device.c7
-rw-r--r--drivers/dma/idxd/irq.c26
-rw-r--r--drivers/dma/mmp_tdma.c5
-rw-r--r--drivers/dma/owl-dma.c8
-rw-r--r--drivers/dma/pch_dma.c2
-rw-r--r--drivers/dma/tegra20-apb-dma.c9
-rw-r--r--drivers/dma/tegra210-adma.c2
-rw-r--r--drivers/dma/ti/k3-psil.c1
-rw-r--r--drivers/dma/ti/k3-udma.c6
-rw-r--r--drivers/dma/xilinx/xilinx_dma.c20
-rw-r--r--drivers/dma/xilinx/zynqmp_dma.c3
14 files changed, 96 insertions, 71 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 092483644315..023db6883d05 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -241,7 +241,8 @@ config FSL_RAID
config HISI_DMA
tristate "HiSilicon DMA Engine support"
- depends on ARM64 || (COMPILE_TEST && PCI_MSI)
+ depends on ARM64 || COMPILE_TEST
+ depends on PCI_MSI
select DMA_ENGINE
select DMA_VIRTUAL_CHANNELS
help
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 4830ba658ce1..d31076d9ef25 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -232,10 +232,6 @@ static void chan_dev_release(struct device *dev)
struct dma_chan_dev *chan_dev;
chan_dev = container_of(dev, typeof(*chan_dev), device);
- if (atomic_dec_and_test(chan_dev->idr_ref)) {
- ida_free(&dma_ida, chan_dev->dev_id);
- kfree(chan_dev->idr_ref);
- }
kfree(chan_dev);
}
@@ -1043,27 +1039,9 @@ static int get_dma_id(struct dma_device *device)
}
static int __dma_async_device_channel_register(struct dma_device *device,
- struct dma_chan *chan,
- int chan_id)
+ struct dma_chan *chan)
{
int rc = 0;
- int chancnt = device->chancnt;
- atomic_t *idr_ref;
- struct dma_chan *tchan;
-
- tchan = list_first_entry_or_null(&device->channels,
- struct dma_chan, device_node);
- if (!tchan)
- return -ENODEV;
-
- if (tchan->dev) {
- idr_ref = tchan->dev->idr_ref;
- } else {
- idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
- if (!idr_ref)
- return -ENOMEM;
- atomic_set(idr_ref, 0);
- }
chan->local = alloc_percpu(typeof(*chan->local));
if (!chan->local)
@@ -1079,29 +1057,36 @@ static int __dma_async_device_channel_register(struct dma_device *device,
* When the chan_id is a negative value, we are dynamically adding
* the channel. Otherwise we are static enumerating.
*/
- chan->chan_id = chan_id < 0 ? chancnt : chan_id;
+ mutex_lock(&device->chan_mutex);
+ chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
+ mutex_unlock(&device->chan_mutex);
+ if (chan->chan_id < 0) {
+ pr_err("%s: unable to alloc ida for chan: %d\n",
+ __func__, chan->chan_id);
+ goto err_out;
+ }
+
chan->dev->device.class = &dma_devclass;
chan->dev->device.parent = device->dev;
chan->dev->chan = chan;
- chan->dev->idr_ref = idr_ref;
chan->dev->dev_id = device->dev_id;
- atomic_inc(idr_ref);
dev_set_name(&chan->dev->device, "dma%dchan%d",
device->dev_id, chan->chan_id);
-
rc = device_register(&chan->dev->device);
if (rc)
- goto err_out;
+ goto err_out_ida;
chan->client_count = 0;
- device->chancnt = chan->chan_id + 1;
+ device->chancnt++;
return 0;
+ err_out_ida:
+ mutex_lock(&device->chan_mutex);
+ ida_free(&device->chan_ida, chan->chan_id);
+ mutex_unlock(&device->chan_mutex);
err_out:
free_percpu(chan->local);
kfree(chan->dev);
- if (atomic_dec_return(idr_ref) == 0)
- kfree(idr_ref);
return rc;
}
@@ -1110,7 +1095,7 @@ int dma_async_device_channel_register(struct dma_device *device,
{
int rc;
- rc = __dma_async_device_channel_register(device, chan, -1);
+ rc = __dma_async_device_channel_register(device, chan);
if (rc < 0)
return rc;
@@ -1130,6 +1115,9 @@ static void __dma_async_device_channel_unregister(struct dma_device *device,
device->chancnt--;
chan->dev->chan = NULL;
mutex_unlock(&dma_list_mutex);
+ mutex_lock(&device->chan_mutex);
+ ida_free(&device->chan_ida, chan->chan_id);
+ mutex_unlock(&device->chan_mutex);
device_unregister(&chan->dev->device);
free_percpu(chan->local);
}
@@ -1152,7 +1140,7 @@ EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
*/
int dma_async_device_register(struct dma_device *device)
{
- int rc, i = 0;
+ int rc;
struct dma_chan* chan;
if (!device)
@@ -1257,9 +1245,12 @@ int dma_async_device_register(struct dma_device *device)
if (rc != 0)
return rc;
+ mutex_init(&device->chan_mutex);
+ ida_init(&device->chan_ida);
+
/* represent channels in sysfs. Probably want devs too */
list_for_each_entry(chan, &device->channels, device_node) {
- rc = __dma_async_device_channel_register(device, chan, i++);
+ rc = __dma_async_device_channel_register(device, chan);
if (rc < 0)
goto err_out;
}
@@ -1334,6 +1325,7 @@ void dma_async_device_unregister(struct dma_device *device)
*/
dma_cap_set(DMA_PRIVATE, device->cap_mask);
dma_channel_rebalance();
+ ida_free(&dma_ida, device->dev_id);
dma_device_put(device);
mutex_unlock(&dma_list_mutex);
}
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
index a2cadfa2e6d7..0425984db118 100644
--- a/drivers/dma/dmatest.c
+++ b/drivers/dma/dmatest.c
@@ -240,7 +240,7 @@ static bool is_threaded_test_run(struct dmatest_info *info)
struct dmatest_thread *thread;
list_for_each_entry(thread, &dtc->threads, node) {
- if (!thread->done)
+ if (!thread->done && !thread->pending)
return true;
}
}
@@ -662,8 +662,8 @@ static int dmatest_func(void *data)
flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
ktime = ktime_get();
- while (!kthread_should_stop()
- && !(params->iterations && total_tests >= params->iterations)) {
+ while (!(kthread_should_stop() ||
+ (params->iterations && total_tests >= params->iterations))) {
struct dma_async_tx_descriptor *tx = NULL;
struct dmaengine_unmap_data *um;
dma_addr_t *dsts;
@@ -1166,10 +1166,11 @@ static int dmatest_run_set(const char *val, const struct kernel_param *kp)
mutex_unlock(&info->lock);
return ret;
} else if (dmatest_run) {
- if (is_threaded_test_pending(info))
- start_threaded_tests(info);
- else
- pr_info("Could not start test, no channels configured\n");
+ if (!is_threaded_test_pending(info)) {
+ pr_info("No channels configured, continue with any\n");
+ add_threaded_test(info);
+ }
+ start_threaded_tests(info);
} else {
stop_threaded_test(info);
}
diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index f6f49f0f6fae..8d79a8787104 100644
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -62,6 +62,13 @@ int idxd_unmask_msix_vector(struct idxd_device *idxd, int vec_id)
perm.ignore = 0;
iowrite32(perm.bits, idxd->reg_base + offset);
+ /*
+ * A readback from the device ensures that any previously generated
+ * completion record writes are visible to software based on PCI
+ * ordering rules.
+ */
+ perm.bits = ioread32(idxd->reg_base + offset);
+
return 0;
}
diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c
index d6fcd2e60103..6510791b9921 100644
--- a/drivers/dma/idxd/irq.c
+++ b/drivers/dma/idxd/irq.c
@@ -173,6 +173,7 @@ static int irq_process_pending_llist(struct idxd_irq_entry *irq_entry,
struct llist_node *head;
int queued = 0;
+ *processed = 0;
head = llist_del_all(&irq_entry->pending_llist);
if (!head)
return 0;
@@ -197,6 +198,7 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
struct list_head *node, *next;
int queued = 0;
+ *processed = 0;
if (list_empty(&irq_entry->work_list))
return 0;
@@ -218,10 +220,9 @@ static int irq_process_work_list(struct idxd_irq_entry *irq_entry,
return queued;
}
-irqreturn_t idxd_wq_thread(int irq, void *data)
+static int idxd_desc_process(struct idxd_irq_entry *irq_entry)
{
- struct idxd_irq_entry *irq_entry = data;
- int rc, processed = 0, retry = 0;
+ int rc, processed, total = 0;
/*
* There are two lists we are processing. The pending_llist is where
@@ -244,15 +245,26 @@ irqreturn_t idxd_wq_thread(int irq, void *data)
*/
do {
rc = irq_process_work_list(irq_entry, &processed);
- if (rc != 0) {
- retry++;
+ total += processed;
+ if (rc != 0)
continue;
- }
rc = irq_process_pending_llist(irq_entry, &processed);
- } while (rc != 0 && retry != 10);
+ total += processed;
+ } while (rc != 0);
+
+ return total;
+}
+
+irqreturn_t idxd_wq_thread(int irq, void *data)
+{
+ struct idxd_irq_entry *irq_entry = data;
+ int processed;
+ processed = idxd_desc_process(irq_entry);
idxd_unmask_msix_vector(irq_entry->idxd, irq_entry->id);
+ /* catch anything unprocessed after unmasking */
+ processed += idxd_desc_process(irq_entry);
if (processed == 0)
return IRQ_NONE;
diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c
index 10117f271b12..d683232d7fea 100644
--- a/drivers/dma/mmp_tdma.c
+++ b/drivers/dma/mmp_tdma.c
@@ -363,6 +363,8 @@ static void mmp_tdma_free_descriptor(struct mmp_tdma_chan *tdmac)
gen_pool_free(gpool, (unsigned long)tdmac->desc_arr,
size);
tdmac->desc_arr = NULL;
+ if (tdmac->status == DMA_ERROR)
+ tdmac->status = DMA_COMPLETE;
return;
}
@@ -443,7 +445,8 @@ static struct dma_async_tx_descriptor *mmp_tdma_prep_dma_cyclic(
if (!desc)
goto err_out;
- mmp_tdma_config_write(chan, direction, &tdmac->slave_config);
+ if (mmp_tdma_config_write(chan, direction, &tdmac->slave_config))
+ goto err_out;
while (buf < buf_len) {
desc = &tdmac->desc_arr[i];
diff --git a/drivers/dma/owl-dma.c b/drivers/dma/owl-dma.c
index c683051257fd..66ef70b00ec0 100644
--- a/drivers/dma/owl-dma.c
+++ b/drivers/dma/owl-dma.c
@@ -175,13 +175,11 @@ struct owl_dma_txd {
* @id: physical index to this channel
* @base: virtual memory base for the dma channel
* @vchan: the virtual channel currently being served by this physical channel
- * @lock: a lock to use when altering an instance of this struct
*/
struct owl_dma_pchan {
u32 id;
void __iomem *base;
struct owl_dma_vchan *vchan;
- spinlock_t lock;
};
/**
@@ -437,14 +435,14 @@ static struct owl_dma_pchan *owl_dma_get_pchan(struct owl_dma *od,
for (i = 0; i < od->nr_pchans; i++) {
pchan = &od->pchans[i];
- spin_lock_irqsave(&pchan->lock, flags);
+ spin_lock_irqsave(&od->lock, flags);
if (!pchan->vchan) {
pchan->vchan = vchan;
- spin_unlock_irqrestore(&pchan->lock, flags);
+ spin_unlock_irqrestore(&od->lock, flags);
break;
}
- spin_unlock_irqrestore(&pchan->lock, flags);
+ spin_unlock_irqrestore(&od->lock, flags);
}
return pchan;
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index 581e7a290d98..a3b0b4c56a19 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -865,6 +865,7 @@ static int pch_dma_probe(struct pci_dev *pdev,
}
pci_set_master(pdev);
+ pd->dma.dev = &pdev->dev;
err = request_irq(pdev->irq, pd_irq, IRQF_SHARED, DRV_NAME, pd);
if (err) {
@@ -880,7 +881,6 @@ static int pch_dma_probe(struct pci_dev *pdev,
goto err_free_irq;
}
- pd->dma.dev = &pdev->dev;
INIT_LIST_HEAD(&pd->dma.channels);
diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c
index f6a2f42ffc51..b9f0d9636620 100644
--- a/drivers/dma/tegra20-apb-dma.c
+++ b/drivers/dma/tegra20-apb-dma.c
@@ -816,6 +816,13 @@ static bool tegra_dma_eoc_interrupt_deasserted(struct tegra_dma_channel *tdc)
static void tegra_dma_synchronize(struct dma_chan *dc)
{
struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
+ int err;
+
+ err = pm_runtime_get_sync(tdc->tdma->dev);
+ if (err < 0) {
+ dev_err(tdc2dev(tdc), "Failed to synchronize DMA: %d\n", err);
+ return;
+ }
/*
* CPU, which handles interrupt, could be busy in
@@ -825,6 +832,8 @@ static void tegra_dma_synchronize(struct dma_chan *dc)
wait_event(tdc->wq, tegra_dma_eoc_interrupt_deasserted(tdc));
tasklet_kill(&tdc->tasklet);
+
+ pm_runtime_put(tdc->tdma->dev);
}
static unsigned int tegra_dma_sg_bytes_xferred(struct tegra_dma_channel *tdc,
diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c
index c4ce5dfb149b..db58d7e4f9fe 100644
--- a/drivers/dma/tegra210-adma.c
+++ b/drivers/dma/tegra210-adma.c
@@ -900,7 +900,7 @@ static int tegra_adma_probe(struct platform_device *pdev)
ret = dma_async_device_register(&tdma->dma_dev);
if (ret < 0) {
dev_err(&pdev->dev, "ADMA registration failed: %d\n", ret);
- goto irq_dispose;
+ goto rpm_put;
}
ret = of_dma_controller_register(pdev->dev.of_node,
diff --git a/drivers/dma/ti/k3-psil.c b/drivers/dma/ti/k3-psil.c
index d7b965049ccb..fb7c8150b0d1 100644
--- a/drivers/dma/ti/k3-psil.c
+++ b/drivers/dma/ti/k3-psil.c
@@ -27,6 +27,7 @@ struct psil_endpoint_config *psil_get_ep_config(u32 thread_id)
soc_ep_map = &j721e_ep_map;
} else {
pr_err("PSIL: No compatible machine found for map\n");
+ mutex_unlock(&ep_map_mutex);
return ERR_PTR(-ENOTSUPP);
}
pr_debug("%s: Using map for %s\n", __func__, soc_ep_map->name);
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index a9c0251adf1a..a90e154b0ae0 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -2156,7 +2156,8 @@ udma_prep_slave_sg_tr(struct udma_chan *uc, struct scatterlist *sgl,
d->residue += sg_dma_len(sgent);
}
- cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags, CPPI5_TR_CSF_EOP);
+ cppi5_tr_csf_set(&tr_req[tr_idx - 1].flags,
+ CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
return d;
}
@@ -2733,7 +2734,8 @@ udma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
tr_req[1].dicnt3 = 1;
}
- cppi5_tr_csf_set(&tr_req[num_tr - 1].flags, CPPI5_TR_CSF_EOP);
+ cppi5_tr_csf_set(&tr_req[num_tr - 1].flags,
+ CPPI5_TR_CSF_SUPR_EVT | CPPI5_TR_CSF_EOP);
if (uc->config.metadata_size)
d->vd.tx.metadata_ops = &metadata_ops;
diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c
index aecd5a35a296..5429497d3560 100644
--- a/drivers/dma/xilinx/xilinx_dma.c
+++ b/drivers/dma/xilinx/xilinx_dma.c
@@ -1230,16 +1230,16 @@ static enum dma_status xilinx_dma_tx_status(struct dma_chan *dchan,
return ret;
spin_lock_irqsave(&chan->lock, flags);
-
- desc = list_last_entry(&chan->active_list,
- struct xilinx_dma_tx_descriptor, node);
- /*
- * VDMA and simple mode do not support residue reporting, so the
- * residue field will always be 0.
- */
- if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)
- residue = xilinx_dma_get_residue(chan, desc);
-
+ if (!list_empty(&chan->active_list)) {
+ desc = list_last_entry(&chan->active_list,
+ struct xilinx_dma_tx_descriptor, node);
+ /*
+ * VDMA and simple mode do not support residue reporting, so the
+ * residue field will always be 0.
+ */
+ if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)
+ residue = xilinx_dma_get_residue(chan, desc);
+ }
spin_unlock_irqrestore(&chan->lock, flags);
dma_set_residue(txstate, residue);
diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c
index d47749a35863..ff253696d183 100644
--- a/drivers/dma/xilinx/zynqmp_dma.c
+++ b/drivers/dma/xilinx/zynqmp_dma.c
@@ -434,6 +434,7 @@ static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan,
struct zynqmp_dma_desc_sw *child, *next;
chan->desc_free_cnt++;
+ list_del(&sdesc->node);
list_add_tail(&sdesc->node, &chan->free_list);
list_for_each_entry_safe(child, next, &sdesc->tx_list, node) {
chan->desc_free_cnt++;
@@ -608,8 +609,6 @@ static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan)
dma_async_tx_callback callback;
void *callback_param;
- list_del(&desc->node);
-
callback = desc->async_tx.callback;
callback_param = desc->async_tx.callback_param;
if (callback) {
generated by cgit (git 2.34.1) at 2025-04-26 02:23:20 +0000