diff options
Diffstat (limited to 'drivers/dma-buf')
| -rw-r--r-- | drivers/dma-buf/Makefile | 2 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-buf-mapping.c | 248 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-buf.c | 10 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-fence.c | 54 | ||||
| -rw-r--r-- | drivers/dma-buf/heaps/Kconfig | 10 | ||||
| -rw-r--r-- | drivers/dma-buf/heaps/cma_heap.c | 47 | ||||
| -rw-r--r-- | drivers/dma-buf/heaps/system_heap.c | 33 | ||||
| -rw-r--r-- | drivers/dma-buf/sw_sync.c | 4 | ||||
| -rw-r--r-- | drivers/dma-buf/sync_debug.c | 2 |
9 files changed, 336 insertions, 74 deletions
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index 70ec901edf2c..2008fb7481b3 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \ - dma-fence-unwrap.o dma-resv.o + dma-fence-unwrap.o dma-resv.o dma-buf-mapping.o obj-$(CONFIG_DMABUF_HEAPS) += dma-heap.o obj-$(CONFIG_DMABUF_HEAPS) += heaps/ obj-$(CONFIG_SYNC_FILE) += sync_file.o diff --git a/drivers/dma-buf/dma-buf-mapping.c b/drivers/dma-buf/dma-buf-mapping.c new file mode 100644 index 000000000000..b7352e609fbd --- /dev/null +++ b/drivers/dma-buf/dma-buf-mapping.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DMA BUF Mapping Helpers + * + */ +#include <linux/dma-buf-mapping.h> +#include <linux/dma-resv.h> + +static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length, + dma_addr_t addr) +{ + unsigned int len, nents; + int i; + + nents = DIV_ROUND_UP(length, UINT_MAX); + for (i = 0; i < nents; i++) { + len = min_t(size_t, length, UINT_MAX); + length -= len; + /* + * DMABUF abuses scatterlist to create a scatterlist + * that does not have any CPU list, only the DMA list. + * Always set the page related values to NULL to ensure + * importers can't use it. The phys_addr based DMA API + * does not require the CPU list for mapping or unmapping. + */ + sg_set_page(sgl, NULL, 0, 0); + sg_dma_address(sgl) = addr + (dma_addr_t)i * UINT_MAX; + sg_dma_len(sgl) = len; + sgl = sg_next(sgl); + } + + return sgl; +} + +static unsigned int calc_sg_nents(struct dma_iova_state *state, + struct dma_buf_phys_vec *phys_vec, + size_t nr_ranges, size_t size) +{ + unsigned int nents = 0; + size_t i; + + if (!state || !dma_use_iova(state)) { + for (i = 0; i < nr_ranges; i++) + nents += DIV_ROUND_UP(phys_vec[i].len, UINT_MAX); + } else { + /* + * In IOVA case, there is only one SG entry which spans + * for whole IOVA address space, but we need to make sure + * that it fits sg->length, maybe we need more. + */ + nents = DIV_ROUND_UP(size, UINT_MAX); + } + + return nents; +} + +/** + * struct dma_buf_dma - holds DMA mapping information + * @sgt: Scatter-gather table + * @state: DMA IOVA state relevant in IOMMU-based DMA + * @size: Total size of DMA transfer + */ +struct dma_buf_dma { + struct sg_table sgt; + struct dma_iova_state *state; + size_t size; +}; + +/** + * dma_buf_phys_vec_to_sgt - Returns the scatterlist table of the attachment + * from arrays of physical vectors. This funciton is intended for MMIO memory + * only. + * @attach: [in] attachment whose scatterlist is to be returned + * @provider: [in] p2pdma provider + * @phys_vec: [in] array of physical vectors + * @nr_ranges: [in] number of entries in phys_vec array + * @size: [in] total size of phys_vec + * @dir: [in] direction of DMA transfer + * + * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR + * on error. May return -EINTR if it is interrupted by a signal. + * + * On success, the DMA addresses and lengths in the returned scatterlist are + * PAGE_SIZE aligned. + * + * A mapping must be unmapped by using dma_buf_free_sgt(). + * + * NOTE: This function is intended for exporters. If direct traffic routing is + * mandatory exporter should call routing pci_p2pdma_map_type() before calling + * this function. + */ +struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach, + struct p2pdma_provider *provider, + struct dma_buf_phys_vec *phys_vec, + size_t nr_ranges, size_t size, + enum dma_data_direction dir) +{ + unsigned int nents, mapped_len = 0; + struct dma_buf_dma *dma; + struct scatterlist *sgl; + dma_addr_t addr; + size_t i; + int ret; + + dma_resv_assert_held(attach->dmabuf->resv); + + if (WARN_ON(!attach || !attach->dmabuf || !provider)) + /* This function is supposed to work on MMIO memory only */ + return ERR_PTR(-EINVAL); + + dma = kzalloc(sizeof(*dma), GFP_KERNEL); + if (!dma) + return ERR_PTR(-ENOMEM); + + switch (pci_p2pdma_map_type(provider, attach->dev)) { + case PCI_P2PDMA_MAP_BUS_ADDR: + /* + * There is no need in IOVA at all for this flow. + */ + break; + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: + dma->state = kzalloc(sizeof(*dma->state), GFP_KERNEL); + if (!dma->state) { + ret = -ENOMEM; + goto err_free_dma; + } + + dma_iova_try_alloc(attach->dev, dma->state, 0, size); + break; + default: + ret = -EINVAL; + goto err_free_dma; + } + + nents = calc_sg_nents(dma->state, phys_vec, nr_ranges, size); + ret = sg_alloc_table(&dma->sgt, nents, GFP_KERNEL | __GFP_ZERO); + if (ret) + goto err_free_state; + + sgl = dma->sgt.sgl; + + for (i = 0; i < nr_ranges; i++) { + if (!dma->state) { + addr = pci_p2pdma_bus_addr_map(provider, + phys_vec[i].paddr); + } else if (dma_use_iova(dma->state)) { + ret = dma_iova_link(attach->dev, dma->state, + phys_vec[i].paddr, 0, + phys_vec[i].len, dir, + DMA_ATTR_MMIO); + if (ret) + goto err_unmap_dma; + + mapped_len += phys_vec[i].len; + } else { + addr = dma_map_phys(attach->dev, phys_vec[i].paddr, + phys_vec[i].len, dir, + DMA_ATTR_MMIO); + ret = dma_mapping_error(attach->dev, addr); + if (ret) + goto err_unmap_dma; + } + + if (!dma->state || !dma_use_iova(dma->state)) + sgl = fill_sg_entry(sgl, phys_vec[i].len, addr); + } + + if (dma->state && dma_use_iova(dma->state)) { + WARN_ON_ONCE(mapped_len != size); + ret = dma_iova_sync(attach->dev, dma->state, 0, mapped_len); + if (ret) + goto err_unmap_dma; + + sgl = fill_sg_entry(sgl, mapped_len, dma->state->addr); + } + + dma->size = size; + + /* + * No CPU list included — set orig_nents = 0 so others can detect + * this via SG table (use nents only). + */ + dma->sgt.orig_nents = 0; + + + /* + * SGL must be NULL to indicate that SGL is the last one + * and we allocated correct number of entries in sg_alloc_table() + */ + WARN_ON_ONCE(sgl); + return &dma->sgt; + +err_unmap_dma: + if (!i || !dma->state) { + ; /* Do nothing */ + } else if (dma_use_iova(dma->state)) { + dma_iova_destroy(attach->dev, dma->state, mapped_len, dir, + DMA_ATTR_MMIO); + } else { + for_each_sgtable_dma_sg(&dma->sgt, sgl, i) + dma_unmap_phys(attach->dev, sg_dma_address(sgl), + sg_dma_len(sgl), dir, DMA_ATTR_MMIO); + } + sg_free_table(&dma->sgt); +err_free_state: + kfree(dma->state); +err_free_dma: + kfree(dma); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_NS_GPL(dma_buf_phys_vec_to_sgt, "DMA_BUF"); + +/** + * dma_buf_free_sgt- unmaps the buffer + * @attach: [in] attachment to unmap buffer from + * @sgt: [in] scatterlist info of the buffer to unmap + * @dir: [in] direction of DMA transfer + * + * This unmaps a DMA mapping for @attached obtained + * by dma_buf_phys_vec_to_sgt(). + */ +void dma_buf_free_sgt(struct dma_buf_attachment *attach, struct sg_table *sgt, + enum dma_data_direction dir) +{ + struct dma_buf_dma *dma = container_of(sgt, struct dma_buf_dma, sgt); + int i; + + dma_resv_assert_held(attach->dmabuf->resv); + + if (!dma->state) { + ; /* Do nothing */ + } else if (dma_use_iova(dma->state)) { + dma_iova_destroy(attach->dev, dma->state, dma->size, dir, + DMA_ATTR_MMIO); + } else { + struct scatterlist *sgl; + + for_each_sgtable_dma_sg(sgt, sgl, i) + dma_unmap_phys(attach->dev, sg_dma_address(sgl), + sg_dma_len(sgl), dir, DMA_ATTR_MMIO); + } + + sg_free_table(sgt); + kfree(dma->state); + kfree(dma); + +} +EXPORT_SYMBOL_NS_GPL(dma_buf_free_sgt, "DMA_BUF"); diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 2bcf9ceca997..edaa9e4ee4ae 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -768,18 +768,10 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF"); */ int dma_buf_fd(struct dma_buf *dmabuf, int flags) { - int fd; - if (!dmabuf || !dmabuf->file) return -EINVAL; - fd = get_unused_fd_flags(flags); - if (fd < 0) - return fd; - - fd_install(fd, dmabuf->file); - - return fd; + return FD_ADD(flags, dmabuf->file); } EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF"); diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 3f78c56b58dc..b4f5c8635276 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -121,29 +121,27 @@ static const struct dma_fence_ops dma_fence_stub_ops = { .get_timeline_name = dma_fence_stub_get_name, }; +static int __init dma_fence_init_stub(void) +{ + dma_fence_init(&dma_fence_stub, &dma_fence_stub_ops, + &dma_fence_stub_lock, 0, 0); + + set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &dma_fence_stub.flags); + + dma_fence_signal(&dma_fence_stub); + return 0; +} +subsys_initcall(dma_fence_init_stub); + /** * dma_fence_get_stub - return a signaled fence * - * Return a stub fence which is already signaled. The fence's - * timestamp corresponds to the first time after boot this - * function is called. + * Return a stub fence which is already signaled. The fence's timestamp + * corresponds to the initialisation time of the linux kernel. */ struct dma_fence *dma_fence_get_stub(void) { - spin_lock(&dma_fence_stub_lock); - if (!dma_fence_stub.ops) { - dma_fence_init(&dma_fence_stub, - &dma_fence_stub_ops, - &dma_fence_stub_lock, - 0, 0); - - set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, - &dma_fence_stub.flags); - - dma_fence_signal_locked(&dma_fence_stub); - } - spin_unlock(&dma_fence_stub_lock); - return dma_fence_get(&dma_fence_stub); } EXPORT_SYMBOL(dma_fence_get_stub); @@ -999,19 +997,21 @@ EXPORT_SYMBOL(dma_fence_set_deadline); */ void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq) { - const char __rcu *timeline; - const char __rcu *driver; + const char __rcu *timeline = ""; + const char __rcu *driver = ""; + const char *signaled = ""; rcu_read_lock(); - timeline = dma_fence_timeline_name(fence); - driver = dma_fence_driver_name(fence); + if (!dma_fence_is_signaled(fence)) { + timeline = dma_fence_timeline_name(fence); + driver = dma_fence_driver_name(fence); + signaled = "un"; + } - seq_printf(seq, "%s %s seq %llu %ssignalled\n", - rcu_dereference(driver), - rcu_dereference(timeline), - fence->seqno, - dma_fence_is_signaled(fence) ? "" : "un"); + seq_printf(seq, "%llu:%llu %s %s %ssignalled\n", + fence->context, fence->seqno, timeline, driver, + signaled); rcu_read_unlock(); } @@ -1141,7 +1141,7 @@ const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) "RCU protection is required for safe access to returned string"); if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) - return fence->ops->get_driver_name(fence); + return fence->ops->get_timeline_name(fence); else return "signaled-timeline"; } diff --git a/drivers/dma-buf/heaps/Kconfig b/drivers/dma-buf/heaps/Kconfig index bb369b38b001..a5eef06c4226 100644 --- a/drivers/dma-buf/heaps/Kconfig +++ b/drivers/dma-buf/heaps/Kconfig @@ -12,13 +12,3 @@ config DMABUF_HEAPS_CMA Choose this option to enable dma-buf CMA heap. This heap is backed by the Contiguous Memory Allocator (CMA). If your system has these regions, you should say Y here. - -config DMABUF_HEAPS_CMA_LEGACY - bool "Legacy DMA-BUF CMA Heap" - default y - depends on DMABUF_HEAPS_CMA - help - Add a duplicate CMA-backed dma-buf heap with legacy naming derived - from the CMA area's devicetree node, or "reserved" if the area is not - defined in the devicetree. This uses the same underlying allocator as - CONFIG_DMABUF_HEAPS_CMA. diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 0df007111975..42f88193eab9 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -14,6 +14,7 @@ #include <linux/cma.h> #include <linux/dma-buf.h> +#include <linux/dma-buf/heaps/cma.h> #include <linux/dma-heap.h> #include <linux/dma-map-ops.h> #include <linux/err.h> @@ -21,12 +22,27 @@ #include <linux/io.h> #include <linux/mm.h> #include <linux/module.h> +#include <linux/of.h> +#include <linux/of_reserved_mem.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/vmalloc.h> #define DEFAULT_CMA_NAME "default_cma_region" +static struct cma *dma_areas[MAX_CMA_AREAS] __initdata; +static unsigned int dma_areas_num __initdata; + +int __init dma_heap_cma_register_heap(struct cma *cma) +{ + if (dma_areas_num >= ARRAY_SIZE(dma_areas)) + return -EINVAL; + + dma_areas[dma_areas_num++] = cma; + + return 0; +} + struct cma_heap { struct dma_heap *heap; struct cma *cma; @@ -395,33 +411,30 @@ static int __init __add_cma_heap(struct cma *cma, const char *name) return 0; } -static int __init add_default_cma_heap(void) +static int __init add_cma_heaps(void) { struct cma *default_cma = dev_get_cma_area(NULL); - const char *legacy_cma_name; + unsigned int i; int ret; - if (!default_cma) - return 0; + if (default_cma) { + ret = __add_cma_heap(default_cma, DEFAULT_CMA_NAME); + if (ret) + return ret; + } - ret = __add_cma_heap(default_cma, DEFAULT_CMA_NAME); - if (ret) - return ret; + for (i = 0; i < dma_areas_num; i++) { + struct cma *cma = dma_areas[i]; - if (IS_ENABLED(CONFIG_DMABUF_HEAPS_CMA_LEGACY)) { - legacy_cma_name = cma_get_name(default_cma); - if (!strcmp(legacy_cma_name, DEFAULT_CMA_NAME)) { - pr_warn("legacy name and default name are the same, skipping legacy heap\n"); - return 0; + ret = __add_cma_heap(cma, cma_get_name(cma)); + if (ret) { + pr_warn("Failed to add CMA heap %s", cma_get_name(cma)); + continue; } - ret = __add_cma_heap(default_cma, legacy_cma_name); - if (ret) - pr_warn("failed to add legacy heap: %pe\n", - ERR_PTR(ret)); } return 0; } -module_init(add_default_cma_heap); +module_init(add_cma_heaps); MODULE_DESCRIPTION("DMA-BUF CMA Heap"); diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index bbe7881f1360..4c782fe33fd4 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -186,20 +186,35 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) struct system_heap_buffer *buffer = dmabuf->priv; struct sg_table *table = &buffer->sg_table; unsigned long addr = vma->vm_start; - struct sg_page_iter piter; - int ret; + unsigned long pgoff = vma->vm_pgoff; + struct scatterlist *sg; + int i, ret; + + for_each_sgtable_sg(table, sg, i) { + unsigned long n = sg->length >> PAGE_SHIFT; - for_each_sgtable_page(table, &piter, vma->vm_pgoff) { - struct page *page = sg_page_iter_page(&piter); + if (pgoff < n) + break; + pgoff -= n; + } + + for (; sg && addr < vma->vm_end; sg = sg_next(sg)) { + unsigned long n = (sg->length >> PAGE_SHIFT) - pgoff; + struct page *page = sg_page(sg) + pgoff; + unsigned long size = n << PAGE_SHIFT; + + if (addr + size > vma->vm_end) + size = vma->vm_end - addr; - ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, - vma->vm_page_prot); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), + size, vma->vm_page_prot); if (ret) return ret; - addr += PAGE_SIZE; - if (addr >= vma->vm_end) - return 0; + + addr += size; + pgoff = 0; } + return 0; } diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c index 3c20f1d31cf5..6f09d13be6b6 100644 --- a/drivers/dma-buf/sw_sync.c +++ b/drivers/dma-buf/sw_sync.c @@ -8,6 +8,7 @@ #include <linux/file.h> #include <linux/fs.h> #include <linux/uaccess.h> +#include <linux/panic.h> #include <linux/slab.h> #include <linux/sync_file.h> @@ -349,6 +350,9 @@ static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, struct sync_file *sync_file; struct sw_sync_create_fence_data data; + /* SW sync fence are inherently unsafe and can deadlock the kernel */ + add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); + if (fd < 0) return fd; diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c index 67cd69551e42..9e5d662cd4e8 100644 --- a/drivers/dma-buf/sync_debug.c +++ b/drivers/dma-buf/sync_debug.c @@ -59,7 +59,7 @@ static void sync_print_fence(struct seq_file *s, struct timespec64 ts64 = ktime_to_timespec64(fence->timestamp); - seq_printf(s, "@%lld.%09ld", (s64)ts64.tv_sec, ts64.tv_nsec); + seq_printf(s, "@%ptSp", &ts64); } seq_printf(s, ": %lld", fence->seqno); |