diff options
Diffstat (limited to 'drivers/iio/buffer/industrialio-buffer-dma.c')
| -rw-r--r-- | drivers/iio/buffer/industrialio-buffer-dma.c | 214 |
1 files changed, 188 insertions, 26 deletions
diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c index 13b1a858969e..7a7a9d37339b 100644 --- a/drivers/iio/buffer/industrialio-buffer-dma.c +++ b/drivers/iio/buffer/industrialio-buffer-dma.c @@ -4,6 +4,8 @@ * Author: Lars-Peter Clausen <lars@metafoo.de> */ +#include <linux/atomic.h> +#include <linux/cleanup.h> #include <linux/slab.h> #include <linux/kernel.h> #include <linux/module.h> @@ -14,6 +16,8 @@ #include <linux/poll.h> #include <linux/iio/buffer_impl.h> #include <linux/iio/buffer-dma.h> +#include <linux/dma-buf.h> +#include <linux/dma-fence.h> #include <linux/dma-mapping.h> #include <linux/sizes.h> @@ -94,13 +98,18 @@ static void iio_buffer_block_release(struct kref *kref) { struct iio_dma_buffer_block *block = container_of(kref, struct iio_dma_buffer_block, kref); + struct iio_dma_buffer_queue *queue = block->queue; - WARN_ON(block->state != IIO_BLOCK_STATE_DEAD); + WARN_ON(block->fileio && block->state != IIO_BLOCK_STATE_DEAD); - dma_free_coherent(block->queue->dev, PAGE_ALIGN(block->size), - block->vaddr, block->phys_addr); + if (block->fileio) { + dma_free_coherent(queue->dev, PAGE_ALIGN(block->size), + block->vaddr, block->phys_addr); + } else { + atomic_dec(&queue->num_dmabufs); + } - iio_buffer_put(&block->queue->buffer); + iio_buffer_put(&queue->buffer); kfree(block); } @@ -163,7 +172,7 @@ static struct iio_dma_buffer_queue *iio_buffer_to_queue(struct iio_buffer *buf) } static struct iio_dma_buffer_block *iio_dma_buffer_alloc_block( - struct iio_dma_buffer_queue *queue, size_t size) + struct iio_dma_buffer_queue *queue, size_t size, bool fileio) { struct iio_dma_buffer_block *block; @@ -171,13 +180,16 @@ static struct iio_dma_buffer_block *iio_dma_buffer_alloc_block( if (!block) return NULL; - block->vaddr = dma_alloc_coherent(queue->dev, PAGE_ALIGN(size), - &block->phys_addr, GFP_KERNEL); - if (!block->vaddr) { - kfree(block); - return NULL; + if (fileio) { + block->vaddr = dma_alloc_coherent(queue->dev, PAGE_ALIGN(size), + &block->phys_addr, GFP_KERNEL); + if (!block->vaddr) { + kfree(block); + return NULL; + } } + block->fileio = fileio; block->size = size; block->state = IIO_BLOCK_STATE_DONE; block->queue = queue; @@ -186,6 +198,9 @@ static struct iio_dma_buffer_block *iio_dma_buffer_alloc_block( iio_buffer_get(&queue->buffer); + if (!fileio) + atomic_inc(&queue->num_dmabufs); + return block; } @@ -218,15 +233,22 @@ void iio_dma_buffer_block_done(struct iio_dma_buffer_block *block) { struct iio_dma_buffer_queue *queue = block->queue; unsigned long flags; + bool cookie; + + cookie = dma_fence_begin_signalling(); spin_lock_irqsave(&queue->list_lock, flags); _iio_dma_buffer_block_done(block); spin_unlock_irqrestore(&queue->list_lock, flags); + if (!block->fileio) + iio_buffer_signal_dmabuf_done(block->fence, 0); + iio_buffer_block_put_atomic(block); iio_dma_buffer_queue_wake(queue); + dma_fence_end_signalling(cookie); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_block_done); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_block_done, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_block_list_abort() - Indicate that a list block has been @@ -243,19 +265,29 @@ void iio_dma_buffer_block_list_abort(struct iio_dma_buffer_queue *queue, { struct iio_dma_buffer_block *block, *_block; unsigned long flags; + bool cookie; + + cookie = dma_fence_begin_signalling(); spin_lock_irqsave(&queue->list_lock, flags); list_for_each_entry_safe(block, _block, list, head) { list_del(&block->head); block->bytes_used = 0; _iio_dma_buffer_block_done(block); + + if (!block->fileio) + iio_buffer_signal_dmabuf_done(block->fence, -EINTR); iio_buffer_block_put_atomic(block); } spin_unlock_irqrestore(&queue->list_lock, flags); + if (queue->fileio.enabled) + queue->fileio.enabled = false; + iio_dma_buffer_queue_wake(queue); + dma_fence_end_signalling(cookie); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_block_list_abort); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_block_list_abort, "IIO_DMA_BUFFER"); static bool iio_dma_block_reusable(struct iio_dma_buffer_block *block) { @@ -273,6 +305,16 @@ static bool iio_dma_block_reusable(struct iio_dma_buffer_block *block) } } +static bool iio_dma_buffer_can_use_fileio(struct iio_dma_buffer_queue *queue) +{ + /* + * Note that queue->num_dmabufs cannot increase while the queue is + * locked, it can only decrease, so it does not race against + * iio_dma_buffer_alloc_block(). + */ + return queue->fileio.enabled || !atomic_read(&queue->num_dmabufs); +} + /** * iio_dma_buffer_request_update() - DMA buffer request_update callback * @buffer: The buffer which to request an update @@ -299,6 +341,12 @@ int iio_dma_buffer_request_update(struct iio_buffer *buffer) mutex_lock(&queue->lock); + queue->fileio.enabled = iio_dma_buffer_can_use_fileio(queue); + + /* If DMABUFs were created, disable fileio interface */ + if (!queue->fileio.enabled) + goto out_unlock; + /* Allocations are page aligned */ if (PAGE_ALIGN(queue->fileio.block_size) == PAGE_ALIGN(size)) try_reuse = true; @@ -339,7 +387,7 @@ int iio_dma_buffer_request_update(struct iio_buffer *buffer) } if (!block) { - block = iio_dma_buffer_alloc_block(queue, size); + block = iio_dma_buffer_alloc_block(queue, size, true); if (!block) { ret = -ENOMEM; goto out_unlock; @@ -372,7 +420,7 @@ out_unlock: return ret; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_request_update); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_request_update, "IIO_DMA_BUFFER"); static void iio_dma_buffer_fileio_free(struct iio_dma_buffer_queue *queue) { @@ -412,8 +460,12 @@ static void iio_dma_buffer_submit_block(struct iio_dma_buffer_queue *queue, block->state = IIO_BLOCK_STATE_ACTIVE; iio_buffer_block_get(block); + ret = queue->ops->submit(queue, block); if (ret) { + if (!block->fileio) + iio_buffer_signal_dmabuf_done(block->fence, ret); + /* * This is a bit of a problem and there is not much we can do * other then wait for the buffer to be disabled and re-enabled @@ -454,7 +506,7 @@ int iio_dma_buffer_enable(struct iio_buffer *buffer, return 0; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_enable); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_enable, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_disable() - Disable DMA buffer @@ -478,7 +530,7 @@ int iio_dma_buffer_disable(struct iio_buffer *buffer, return 0; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_disable); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_disable, "IIO_DMA_BUFFER"); static void iio_dma_buffer_enqueue(struct iio_dma_buffer_queue *queue, struct iio_dma_buffer_block *block) @@ -572,7 +624,7 @@ out_unlock: /** * iio_dma_buffer_read() - DMA buffer read callback - * @buffer: Buffer to read form + * @buffer: Buffer to read from * @n: Number of bytes to read * @user_buffer: Userspace buffer to copy the data to * @@ -584,11 +636,11 @@ int iio_dma_buffer_read(struct iio_buffer *buffer, size_t n, { return iio_dma_buffer_io(buffer, n, user_buffer, false); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_read); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_read, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_write() - DMA buffer write callback - * @buffer: Buffer to read form + * @buffer: Buffer to write to * @n: Number of bytes to read * @user_buffer: Userspace buffer to copy the data from * @@ -601,7 +653,7 @@ int iio_dma_buffer_write(struct iio_buffer *buffer, size_t n, return iio_dma_buffer_io(buffer, n, (__force __user char *)user_buffer, true); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_write); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_write, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_usage() - DMA buffer data_available and @@ -644,7 +696,117 @@ size_t iio_dma_buffer_usage(struct iio_buffer *buf) return data_available; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_usage); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_usage, "IIO_DMA_BUFFER"); + +struct iio_dma_buffer_block * +iio_dma_buffer_attach_dmabuf(struct iio_buffer *buffer, + struct dma_buf_attachment *attach) +{ + struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer); + struct iio_dma_buffer_block *block; + + guard(mutex)(&queue->lock); + + /* + * If the buffer is enabled and in fileio mode new blocks can't be + * allocated. + */ + if (queue->fileio.enabled) + return ERR_PTR(-EBUSY); + + block = iio_dma_buffer_alloc_block(queue, attach->dmabuf->size, false); + if (!block) + return ERR_PTR(-ENOMEM); + + /* Free memory that might be in use for fileio mode */ + iio_dma_buffer_fileio_free(queue); + + return block; +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_attach_dmabuf, "IIO_DMA_BUFFER"); + +void iio_dma_buffer_detach_dmabuf(struct iio_buffer *buffer, + struct iio_dma_buffer_block *block) +{ + block->state = IIO_BLOCK_STATE_DEAD; + iio_buffer_block_put_atomic(block); +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_detach_dmabuf, "IIO_DMA_BUFFER"); + +static int iio_dma_can_enqueue_block(struct iio_dma_buffer_block *block) +{ + struct iio_dma_buffer_queue *queue = block->queue; + + /* If in fileio mode buffers can't be enqueued. */ + if (queue->fileio.enabled) + return -EBUSY; + + switch (block->state) { + case IIO_BLOCK_STATE_QUEUED: + return -EPERM; + case IIO_BLOCK_STATE_ACTIVE: + case IIO_BLOCK_STATE_DEAD: + return -EBUSY; + case IIO_BLOCK_STATE_DONE: + break; + } + + return 0; +} + +int iio_dma_buffer_enqueue_dmabuf(struct iio_buffer *buffer, + struct iio_dma_buffer_block *block, + struct dma_fence *fence, + struct sg_table *sgt, + size_t size, bool cyclic) +{ + struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer); + bool cookie; + int ret; + + WARN_ON(!mutex_is_locked(&queue->lock)); + + cookie = dma_fence_begin_signalling(); + + ret = iio_dma_can_enqueue_block(block); + if (ret < 0) + goto out_end_signalling; + + block->bytes_used = size; + block->cyclic = cyclic; + block->sg_table = sgt; + block->fence = fence; + + iio_dma_buffer_enqueue(queue, block); + +out_end_signalling: + dma_fence_end_signalling(cookie); + + return ret; +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_enqueue_dmabuf, "IIO_DMA_BUFFER"); + +struct device *iio_dma_buffer_get_dma_dev(struct iio_buffer *buffer) +{ + return iio_buffer_to_queue(buffer)->dev; +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_get_dma_dev, "IIO_DMA_BUFFER"); + +void iio_dma_buffer_lock_queue(struct iio_buffer *buffer) +{ + struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer); + + mutex_lock(&queue->lock); +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_lock_queue, "IIO_DMA_BUFFER"); + +void iio_dma_buffer_unlock_queue(struct iio_buffer *buffer) +{ + struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer); + + mutex_unlock(&queue->lock); +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_unlock_queue, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_set_bytes_per_datum() - DMA buffer set_bytes_per_datum callback @@ -660,7 +822,7 @@ int iio_dma_buffer_set_bytes_per_datum(struct iio_buffer *buffer, size_t bpd) return 0; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_set_bytes_per_datum); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_set_bytes_per_datum, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_set_length - DMA buffer set_length callback @@ -680,7 +842,7 @@ int iio_dma_buffer_set_length(struct iio_buffer *buffer, unsigned int length) return 0; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_set_length); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_set_length, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_init() - Initialize DMA buffer queue @@ -708,7 +870,7 @@ int iio_dma_buffer_init(struct iio_dma_buffer_queue *queue, return 0; } -EXPORT_SYMBOL_GPL(iio_dma_buffer_init); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_init, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_exit() - Cleanup DMA buffer queue @@ -726,7 +888,7 @@ void iio_dma_buffer_exit(struct iio_dma_buffer_queue *queue) mutex_unlock(&queue->lock); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_exit); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_exit, "IIO_DMA_BUFFER"); /** * iio_dma_buffer_release() - Release final buffer resources @@ -740,7 +902,7 @@ void iio_dma_buffer_release(struct iio_dma_buffer_queue *queue) { mutex_destroy(&queue->lock); } -EXPORT_SYMBOL_GPL(iio_dma_buffer_release); +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_release, "IIO_DMA_BUFFER"); MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); MODULE_DESCRIPTION("DMA buffer for the IIO framework"); |