diff options
Diffstat (limited to 'drivers/dma-buf')
| -rw-r--r-- | drivers/dma-buf/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/dma-buf/Makefile | 2 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-buf-mapping.c | 248 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-buf-sysfs-stats.c | 4 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-buf.c | 383 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-fence-array.c | 123 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-fence-chain.c | 19 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-fence-unwrap.c | 179 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-fence.c | 290 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-heap.c | 37 | ||||
| -rw-r--r-- | drivers/dma-buf/dma-resv.c | 67 | ||||
| -rw-r--r-- | drivers/dma-buf/heaps/cma_heap.c | 76 | ||||
| -rw-r--r-- | drivers/dma-buf/heaps/system_heap.c | 92 | ||||
| -rw-r--r-- | drivers/dma-buf/st-dma-fence-chain.c | 18 | ||||
| -rw-r--r-- | drivers/dma-buf/st-dma-fence-unwrap.c | 268 | ||||
| -rw-r--r-- | drivers/dma-buf/st-dma-fence.c | 18 | ||||
| -rw-r--r-- | drivers/dma-buf/sw_sync.c | 131 | ||||
| -rw-r--r-- | drivers/dma-buf/sync_debug.c | 76 | ||||
| -rw-r--r-- | drivers/dma-buf/sync_debug.h | 4 | ||||
| -rw-r--r-- | drivers/dma-buf/sync_file.c | 52 | ||||
| -rw-r--r-- | drivers/dma-buf/sync_trace.h | 2 | ||||
| -rw-r--r-- | drivers/dma-buf/udmabuf.c | 377 |
22 files changed, 1762 insertions, 705 deletions
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig index e4dc53a36428..b46eb8a552d7 100644 --- a/drivers/dma-buf/Kconfig +++ b/drivers/dma-buf/Kconfig @@ -35,6 +35,7 @@ config UDMABUF default n depends on DMA_SHARED_BUFFER depends on MEMFD_CREATE || COMPILE_TEST + depends on MMU help A driver to let userspace turn memfd regions into dma-bufs. Qemu can use this to create host dmabufs for guest framebuffers. diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile index 70ec901edf2c..2008fb7481b3 100644 --- a/drivers/dma-buf/Makefile +++ b/drivers/dma-buf/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-y := dma-buf.o dma-fence.o dma-fence-array.o dma-fence-chain.o \ - dma-fence-unwrap.o dma-resv.o + dma-fence-unwrap.o dma-resv.o dma-buf-mapping.o obj-$(CONFIG_DMABUF_HEAPS) += dma-heap.o obj-$(CONFIG_DMABUF_HEAPS) += heaps/ obj-$(CONFIG_SYNC_FILE) += sync_file.o diff --git a/drivers/dma-buf/dma-buf-mapping.c b/drivers/dma-buf/dma-buf-mapping.c new file mode 100644 index 000000000000..b7352e609fbd --- /dev/null +++ b/drivers/dma-buf/dma-buf-mapping.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * DMA BUF Mapping Helpers + * + */ +#include <linux/dma-buf-mapping.h> +#include <linux/dma-resv.h> + +static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length, + dma_addr_t addr) +{ + unsigned int len, nents; + int i; + + nents = DIV_ROUND_UP(length, UINT_MAX); + for (i = 0; i < nents; i++) { + len = min_t(size_t, length, UINT_MAX); + length -= len; + /* + * DMABUF abuses scatterlist to create a scatterlist + * that does not have any CPU list, only the DMA list. + * Always set the page related values to NULL to ensure + * importers can't use it. The phys_addr based DMA API + * does not require the CPU list for mapping or unmapping. + */ + sg_set_page(sgl, NULL, 0, 0); + sg_dma_address(sgl) = addr + (dma_addr_t)i * UINT_MAX; + sg_dma_len(sgl) = len; + sgl = sg_next(sgl); + } + + return sgl; +} + +static unsigned int calc_sg_nents(struct dma_iova_state *state, + struct dma_buf_phys_vec *phys_vec, + size_t nr_ranges, size_t size) +{ + unsigned int nents = 0; + size_t i; + + if (!state || !dma_use_iova(state)) { + for (i = 0; i < nr_ranges; i++) + nents += DIV_ROUND_UP(phys_vec[i].len, UINT_MAX); + } else { + /* + * In IOVA case, there is only one SG entry which spans + * for whole IOVA address space, but we need to make sure + * that it fits sg->length, maybe we need more. + */ + nents = DIV_ROUND_UP(size, UINT_MAX); + } + + return nents; +} + +/** + * struct dma_buf_dma - holds DMA mapping information + * @sgt: Scatter-gather table + * @state: DMA IOVA state relevant in IOMMU-based DMA + * @size: Total size of DMA transfer + */ +struct dma_buf_dma { + struct sg_table sgt; + struct dma_iova_state *state; + size_t size; +}; + +/** + * dma_buf_phys_vec_to_sgt - Returns the scatterlist table of the attachment + * from arrays of physical vectors. This funciton is intended for MMIO memory + * only. + * @attach: [in] attachment whose scatterlist is to be returned + * @provider: [in] p2pdma provider + * @phys_vec: [in] array of physical vectors + * @nr_ranges: [in] number of entries in phys_vec array + * @size: [in] total size of phys_vec + * @dir: [in] direction of DMA transfer + * + * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR + * on error. May return -EINTR if it is interrupted by a signal. + * + * On success, the DMA addresses and lengths in the returned scatterlist are + * PAGE_SIZE aligned. + * + * A mapping must be unmapped by using dma_buf_free_sgt(). + * + * NOTE: This function is intended for exporters. If direct traffic routing is + * mandatory exporter should call routing pci_p2pdma_map_type() before calling + * this function. + */ +struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach, + struct p2pdma_provider *provider, + struct dma_buf_phys_vec *phys_vec, + size_t nr_ranges, size_t size, + enum dma_data_direction dir) +{ + unsigned int nents, mapped_len = 0; + struct dma_buf_dma *dma; + struct scatterlist *sgl; + dma_addr_t addr; + size_t i; + int ret; + + dma_resv_assert_held(attach->dmabuf->resv); + + if (WARN_ON(!attach || !attach->dmabuf || !provider)) + /* This function is supposed to work on MMIO memory only */ + return ERR_PTR(-EINVAL); + + dma = kzalloc(sizeof(*dma), GFP_KERNEL); + if (!dma) + return ERR_PTR(-ENOMEM); + + switch (pci_p2pdma_map_type(provider, attach->dev)) { + case PCI_P2PDMA_MAP_BUS_ADDR: + /* + * There is no need in IOVA at all for this flow. + */ + break; + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: + dma->state = kzalloc(sizeof(*dma->state), GFP_KERNEL); + if (!dma->state) { + ret = -ENOMEM; + goto err_free_dma; + } + + dma_iova_try_alloc(attach->dev, dma->state, 0, size); + break; + default: + ret = -EINVAL; + goto err_free_dma; + } + + nents = calc_sg_nents(dma->state, phys_vec, nr_ranges, size); + ret = sg_alloc_table(&dma->sgt, nents, GFP_KERNEL | __GFP_ZERO); + if (ret) + goto err_free_state; + + sgl = dma->sgt.sgl; + + for (i = 0; i < nr_ranges; i++) { + if (!dma->state) { + addr = pci_p2pdma_bus_addr_map(provider, + phys_vec[i].paddr); + } else if (dma_use_iova(dma->state)) { + ret = dma_iova_link(attach->dev, dma->state, + phys_vec[i].paddr, 0, + phys_vec[i].len, dir, + DMA_ATTR_MMIO); + if (ret) + goto err_unmap_dma; + + mapped_len += phys_vec[i].len; + } else { + addr = dma_map_phys(attach->dev, phys_vec[i].paddr, + phys_vec[i].len, dir, + DMA_ATTR_MMIO); + ret = dma_mapping_error(attach->dev, addr); + if (ret) + goto err_unmap_dma; + } + + if (!dma->state || !dma_use_iova(dma->state)) + sgl = fill_sg_entry(sgl, phys_vec[i].len, addr); + } + + if (dma->state && dma_use_iova(dma->state)) { + WARN_ON_ONCE(mapped_len != size); + ret = dma_iova_sync(attach->dev, dma->state, 0, mapped_len); + if (ret) + goto err_unmap_dma; + + sgl = fill_sg_entry(sgl, mapped_len, dma->state->addr); + } + + dma->size = size; + + /* + * No CPU list included — set orig_nents = 0 so others can detect + * this via SG table (use nents only). + */ + dma->sgt.orig_nents = 0; + + + /* + * SGL must be NULL to indicate that SGL is the last one + * and we allocated correct number of entries in sg_alloc_table() + */ + WARN_ON_ONCE(sgl); + return &dma->sgt; + +err_unmap_dma: + if (!i || !dma->state) { + ; /* Do nothing */ + } else if (dma_use_iova(dma->state)) { + dma_iova_destroy(attach->dev, dma->state, mapped_len, dir, + DMA_ATTR_MMIO); + } else { + for_each_sgtable_dma_sg(&dma->sgt, sgl, i) + dma_unmap_phys(attach->dev, sg_dma_address(sgl), + sg_dma_len(sgl), dir, DMA_ATTR_MMIO); + } + sg_free_table(&dma->sgt); +err_free_state: + kfree(dma->state); +err_free_dma: + kfree(dma); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_NS_GPL(dma_buf_phys_vec_to_sgt, "DMA_BUF"); + +/** + * dma_buf_free_sgt- unmaps the buffer + * @attach: [in] attachment to unmap buffer from + * @sgt: [in] scatterlist info of the buffer to unmap + * @dir: [in] direction of DMA transfer + * + * This unmaps a DMA mapping for @attached obtained + * by dma_buf_phys_vec_to_sgt(). + */ +void dma_buf_free_sgt(struct dma_buf_attachment *attach, struct sg_table *sgt, + enum dma_data_direction dir) +{ + struct dma_buf_dma *dma = container_of(sgt, struct dma_buf_dma, sgt); + int i; + + dma_resv_assert_held(attach->dmabuf->resv); + + if (!dma->state) { + ; /* Do nothing */ + } else if (dma_use_iova(dma->state)) { + dma_iova_destroy(attach->dev, dma->state, dma->size, dir, + DMA_ATTR_MMIO); + } else { + struct scatterlist *sgl; + + for_each_sgtable_dma_sg(sgt, sgl, i) + dma_unmap_phys(attach->dev, sg_dma_address(sgl), + sg_dma_len(sgl), dir, DMA_ATTR_MMIO); + } + + sg_free_table(sgt); + kfree(dma->state); + kfree(dma); + +} +EXPORT_SYMBOL_NS_GPL(dma_buf_free_sgt, "DMA_BUF"); diff --git a/drivers/dma-buf/dma-buf-sysfs-stats.c b/drivers/dma-buf/dma-buf-sysfs-stats.c index fbf725fae7c1..b5b62e40ccc1 100644 --- a/drivers/dma-buf/dma-buf-sysfs-stats.c +++ b/drivers/dma-buf/dma-buf-sysfs-stats.c @@ -33,7 +33,7 @@ * into their address space. This necessitated the creation of the DMA-BUF sysfs * statistics interface to provide per-buffer information on production systems. * - * The interface at ``/sys/kernel/dma-buf/buffers`` exposes information about + * The interface at ``/sys/kernel/dmabuf/buffers`` exposes information about * every DMA-BUF when ``CONFIG_DMABUF_SYSFS_STATS`` is enabled. * * The following stats are exposed by the interface: @@ -112,7 +112,7 @@ static void dma_buf_sysfs_release(struct kobject *kobj) kfree(sysfs_entry); } -static struct kobj_type dma_buf_ktype = { +static const struct kobj_type dma_buf_ktype = { .sysfs_ops = &dma_buf_stats_sysfs_ops, .release = dma_buf_sysfs_release, .default_groups = dma_buf_stats_default_groups, diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 757c0fb77a6c..edaa9e4ee4ae 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -19,7 +19,9 @@ #include <linux/anon_inodes.h> #include <linux/export.h> #include <linux/debugfs.h> +#include <linux/list.h> #include <linux/module.h> +#include <linux/mutex.h> #include <linux/seq_file.h> #include <linux/sync_file.h> #include <linux/poll.h> @@ -35,23 +37,102 @@ static inline int is_dma_buf_file(struct file *); -struct dma_buf_list { - struct list_head head; - struct mutex lock; -}; +static DEFINE_MUTEX(dmabuf_list_mutex); +static LIST_HEAD(dmabuf_list); + +static void __dma_buf_list_add(struct dma_buf *dmabuf) +{ + mutex_lock(&dmabuf_list_mutex); + list_add(&dmabuf->list_node, &dmabuf_list); + mutex_unlock(&dmabuf_list_mutex); +} + +static void __dma_buf_list_del(struct dma_buf *dmabuf) +{ + if (!dmabuf) + return; + + mutex_lock(&dmabuf_list_mutex); + list_del(&dmabuf->list_node); + mutex_unlock(&dmabuf_list_mutex); +} + +/** + * dma_buf_iter_begin - begin iteration through global list of all DMA buffers + * + * Returns the first buffer in the global list of DMA-bufs that's not in the + * process of being destroyed. Increments that buffer's reference count to + * prevent buffer destruction. Callers must release the reference, either by + * continuing iteration with dma_buf_iter_next(), or with dma_buf_put(). + * + * Return: + * * First buffer from global list, with refcount elevated + * * NULL if no active buffers are present + */ +struct dma_buf *dma_buf_iter_begin(void) +{ + struct dma_buf *ret = NULL, *dmabuf; + + /* + * The list mutex does not protect a dmabuf's refcount, so it can be + * zeroed while we are iterating. We cannot call get_dma_buf() since the + * caller may not already own a reference to the buffer. + */ + mutex_lock(&dmabuf_list_mutex); + list_for_each_entry(dmabuf, &dmabuf_list, list_node) { + if (file_ref_get(&dmabuf->file->f_ref)) { + ret = dmabuf; + break; + } + } + mutex_unlock(&dmabuf_list_mutex); + return ret; +} -static struct dma_buf_list db_list; +/** + * dma_buf_iter_next - continue iteration through global list of all DMA buffers + * @dmabuf: [in] pointer to dma_buf + * + * Decrements the reference count on the provided buffer. Returns the next + * buffer from the remainder of the global list of DMA-bufs with its reference + * count incremented. Callers must release the reference, either by continuing + * iteration with dma_buf_iter_next(), or with dma_buf_put(). + * + * Return: + * * Next buffer from global list, with refcount elevated + * * NULL if no additional active buffers are present + */ +struct dma_buf *dma_buf_iter_next(struct dma_buf *dmabuf) +{ + struct dma_buf *ret = NULL; + + /* + * The list mutex does not protect a dmabuf's refcount, so it can be + * zeroed while we are iterating. We cannot call get_dma_buf() since the + * caller may not already own a reference to the buffer. + */ + mutex_lock(&dmabuf_list_mutex); + dma_buf_put(dmabuf); + list_for_each_entry_continue(dmabuf, &dmabuf_list, list_node) { + if (file_ref_get(&dmabuf->file->f_ref)) { + ret = dmabuf; + break; + } + } + mutex_unlock(&dmabuf_list_mutex); + return ret; +} static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) { struct dma_buf *dmabuf; char name[DMA_BUF_NAME_LEN]; - size_t ret = 0; + ssize_t ret = 0; dmabuf = dentry->d_fsdata; spin_lock(&dmabuf->name_lock); if (dmabuf->name) - ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN); + ret = strscpy(name, dmabuf->name, sizeof(name)); spin_unlock(&dmabuf->name_lock); return dynamic_dname(buffer, buflen, "/%s:%s", @@ -89,17 +170,10 @@ static void dma_buf_release(struct dentry *dentry) static int dma_buf_file_release(struct inode *inode, struct file *file) { - struct dma_buf *dmabuf; - if (!is_dma_buf_file(file)) return -EINVAL; - dmabuf = file->private_data; - if (dmabuf) { - mutex_lock(&db_list.lock); - list_del(&dmabuf->list_node); - mutex_unlock(&db_list.lock); - } + __dma_buf_list_del(file->private_data); return 0; } @@ -131,7 +205,6 @@ static struct file_system_type dma_buf_fs_type = { static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) { struct dma_buf *dmabuf; - int ret; if (!is_dma_buf_file(file)) return -EINVAL; @@ -147,11 +220,7 @@ static int dma_buf_mmap_internal(struct file *file, struct vm_area_struct *vma) dmabuf->size >> PAGE_SHIFT) return -EINVAL; - dma_resv_lock(dmabuf->resv, NULL); - ret = dmabuf->ops->mmap(dmabuf, vma); - dma_resv_unlock(dmabuf->resv); - - return ret; + return dmabuf->ops->mmap(dmabuf, vma); } static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) @@ -165,8 +234,9 @@ static loff_t dma_buf_llseek(struct file *file, loff_t offset, int whence) dmabuf = file->private_data; /* only support discovering the end of the buffer, - but also allow SEEK_SET to maintain the idiomatic - SEEK_END(0), SEEK_CUR(0) pattern */ + * but also allow SEEK_SET to maintain the idiomatic + * SEEK_END(0), SEEK_CUR(0) pattern. + */ if (whence == SEEK_END) base = dmabuf->size; else if (whence == SEEK_SET) @@ -547,7 +617,7 @@ static struct file *dma_buf_getfile(size_t size, int flags) * Override ->i_ino with the unique and dmabuffs specific * value. */ - inode->i_ino = atomic64_add_return(1, &dmabuf_inode); + inode->i_ino = atomic64_inc_return(&dmabuf_inode); flags &= O_ACCMODE | O_NONBLOCK; file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf", flags, &dma_buf_fops); @@ -624,10 +694,6 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) || !exp_info->ops->release)) return ERR_PTR(-EINVAL); - if (WARN_ON(exp_info->ops->cache_sgt_mapping && - (exp_info->ops->pin || exp_info->ops->unpin))) - return ERR_PTR(-EINVAL); - if (WARN_ON(!exp_info->ops->pin != !exp_info->ops->unpin)) return ERR_PTR(-EINVAL); @@ -677,9 +743,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) file->f_path.dentry->d_fsdata = dmabuf; dmabuf->file = file; - mutex_lock(&db_list.lock); - list_add(&dmabuf->list_node, &db_list.head); - mutex_unlock(&db_list.lock); + __dma_buf_list_add(dmabuf); return dmabuf; @@ -693,7 +757,7 @@ err_module: module_put(exp_info->owner); return ERR_PTR(ret); } -EXPORT_SYMBOL_NS_GPL(dma_buf_export, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF"); /** * dma_buf_fd - returns a file descriptor for the given struct dma_buf @@ -704,20 +768,12 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, DMA_BUF); */ int dma_buf_fd(struct dma_buf *dmabuf, int flags) { - int fd; - if (!dmabuf || !dmabuf->file) return -EINVAL; - fd = get_unused_fd_flags(flags); - if (fd < 0) - return fd; - - fd_install(fd, dmabuf->file); - - return fd; + return FD_ADD(flags, dmabuf->file); } -EXPORT_SYMBOL_NS_GPL(dma_buf_fd, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF"); /** * dma_buf_get - returns the struct dma_buf related to an fd @@ -743,7 +799,7 @@ struct dma_buf *dma_buf_get(int fd) return file->private_data; } -EXPORT_SYMBOL_NS_GPL(dma_buf_get, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_get, "DMA_BUF"); /** * dma_buf_put - decreases refcount of the buffer @@ -762,7 +818,7 @@ void dma_buf_put(struct dma_buf *dmabuf) fput(dmabuf->file); } -EXPORT_SYMBOL_NS_GPL(dma_buf_put, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_put, "DMA_BUF"); static void mangle_sg_table(struct sg_table *sg_table) { @@ -772,36 +828,27 @@ static void mangle_sg_table(struct sg_table *sg_table) /* To catch abuse of the underlying struct page by importers mix * up the bits, but take care to preserve the low SG_ bits to - * not corrupt the sgt. The mixing is undone in __unmap_dma_buf - * before passing the sgt back to the exporter. */ + * not corrupt the sgt. The mixing is undone on unmap + * before passing the sgt back to the exporter. + */ for_each_sgtable_sg(sg_table, sg, i) sg->page_link ^= ~0xffUL; #endif } -static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, - enum dma_data_direction direction) -{ - struct sg_table *sg_table; - signed long ret; - - sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); - if (IS_ERR_OR_NULL(sg_table)) - return sg_table; - if (!dma_buf_attachment_is_dynamic(attach)) { - ret = dma_resv_wait_timeout(attach->dmabuf->resv, - DMA_RESV_USAGE_KERNEL, true, - MAX_SCHEDULE_TIMEOUT); - if (ret < 0) { - attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, - direction); - return ERR_PTR(ret); - } - } +static inline bool +dma_buf_attachment_is_dynamic(struct dma_buf_attachment *attach) +{ + return !!attach->importer_ops; +} - mangle_sg_table(sg_table); - return sg_table; +static bool +dma_buf_pin_on_map(struct dma_buf_attachment *attach) +{ + return attach->dmabuf->ops->pin && + (!dma_buf_attachment_is_dynamic(attach) || + !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)); } /** @@ -828,7 +875,7 @@ static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, * - dma_buf_attach() * - dma_buf_dynamic_attach() * - dma_buf_detach() - * - dma_buf_export( + * - dma_buf_export() * - dma_buf_fd() * - dma_buf_get() * - dma_buf_put() @@ -850,6 +897,7 @@ static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, * - &dma_buf_ops.release() * - &dma_buf_ops.begin_cpu_access() * - &dma_buf_ops.end_cpu_access() + * - &dma_buf_ops.mmap() * * 2. These &dma_buf_ops callbacks are invoked with locked dma-buf * reservation and exporter can't take the lock: @@ -858,7 +906,6 @@ static struct sg_table * __map_dma_buf(struct dma_buf_attachment *attach, * - &dma_buf_ops.unpin() * - &dma_buf_ops.map_dma_buf() * - &dma_buf_ops.unmap_dma_buf() - * - &dma_buf_ops.mmap() * - &dma_buf_ops.vmap() * - &dma_buf_ops.vunmap() * @@ -924,50 +971,13 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev, list_add(&attach->node, &dmabuf->attachments); dma_resv_unlock(dmabuf->resv); - /* When either the importer or the exporter can't handle dynamic - * mappings we cache the mapping here to avoid issues with the - * reservation object lock. - */ - if (dma_buf_attachment_is_dynamic(attach) != - dma_buf_is_dynamic(dmabuf)) { - struct sg_table *sgt; - - dma_resv_lock(attach->dmabuf->resv, NULL); - if (dma_buf_is_dynamic(attach->dmabuf)) { - ret = dmabuf->ops->pin(attach); - if (ret) - goto err_unlock; - } - - sgt = __map_dma_buf(attach, DMA_BIDIRECTIONAL); - if (!sgt) - sgt = ERR_PTR(-ENOMEM); - if (IS_ERR(sgt)) { - ret = PTR_ERR(sgt); - goto err_unpin; - } - dma_resv_unlock(attach->dmabuf->resv); - attach->sgt = sgt; - attach->dir = DMA_BIDIRECTIONAL; - } - return attach; err_attach: kfree(attach); return ERR_PTR(ret); - -err_unpin: - if (dma_buf_is_dynamic(attach->dmabuf)) - dmabuf->ops->unpin(attach); - -err_unlock: - dma_resv_unlock(attach->dmabuf->resv); - - dma_buf_detach(dmabuf, attach); - return ERR_PTR(ret); } -EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_dynamic_attach, "DMA_BUF"); /** * dma_buf_attach - Wrapper for dma_buf_dynamic_attach @@ -982,17 +992,7 @@ struct dma_buf_attachment *dma_buf_attach(struct dma_buf *dmabuf, { return dma_buf_dynamic_attach(dmabuf, dev, NULL, NULL); } -EXPORT_SYMBOL_NS_GPL(dma_buf_attach, DMA_BUF); - -static void __unmap_dma_buf(struct dma_buf_attachment *attach, - struct sg_table *sg_table, - enum dma_data_direction direction) -{ - /* uses XOR, hence this unmangles */ - mangle_sg_table(sg_table); - - attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); -} +EXPORT_SYMBOL_NS_GPL(dma_buf_attach, "DMA_BUF"); /** * dma_buf_detach - Remove the given attachment from dmabuf's attachments list @@ -1009,16 +1009,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) return; dma_resv_lock(dmabuf->resv, NULL); - - if (attach->sgt) { - - __unmap_dma_buf(attach, attach->sgt, attach->dir); - - if (dma_buf_is_dynamic(attach->dmabuf)) - dmabuf->ops->unpin(attach); - } list_del(&attach->node); - dma_resv_unlock(dmabuf->resv); if (dmabuf->ops->detach) @@ -1026,7 +1017,7 @@ void dma_buf_detach(struct dma_buf *dmabuf, struct dma_buf_attachment *attach) kfree(attach); } -EXPORT_SYMBOL_NS_GPL(dma_buf_detach, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_detach, "DMA_BUF"); /** * dma_buf_pin - Lock down the DMA-buf @@ -1047,7 +1038,7 @@ int dma_buf_pin(struct dma_buf_attachment *attach) struct dma_buf *dmabuf = attach->dmabuf; int ret = 0; - WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + WARN_ON(!attach->importer_ops); dma_resv_assert_held(dmabuf->resv); @@ -1056,7 +1047,7 @@ int dma_buf_pin(struct dma_buf_attachment *attach) return ret; } -EXPORT_SYMBOL_NS_GPL(dma_buf_pin, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_pin, "DMA_BUF"); /** * dma_buf_unpin - Unpin a DMA-buf @@ -1070,14 +1061,14 @@ void dma_buf_unpin(struct dma_buf_attachment *attach) { struct dma_buf *dmabuf = attach->dmabuf; - WARN_ON(!dma_buf_attachment_is_dynamic(attach)); + WARN_ON(!attach->importer_ops); dma_resv_assert_held(dmabuf->resv); if (dmabuf->ops->unpin) dmabuf->ops->unpin(attach); } -EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_unpin, "DMA_BUF"); /** * dma_buf_map_attachment - Returns the scatterlist table of the attachment; @@ -1104,7 +1095,7 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, enum dma_data_direction direction) { struct sg_table *sg_table; - int r; + signed long ret; might_sleep(); @@ -1113,41 +1104,37 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, dma_resv_assert_held(attach->dmabuf->resv); - if (attach->sgt) { + if (dma_buf_pin_on_map(attach)) { + ret = attach->dmabuf->ops->pin(attach); /* - * Two mappings with different directions for the same - * attachment are not allowed. + * Catch exporters making buffers inaccessible even when + * attachments preventing that exist. */ - if (attach->dir != direction && - attach->dir != DMA_BIDIRECTIONAL) - return ERR_PTR(-EBUSY); - - return attach->sgt; - } - - if (dma_buf_is_dynamic(attach->dmabuf)) { - if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) { - r = attach->dmabuf->ops->pin(attach); - if (r) - return ERR_PTR(r); - } + WARN_ON_ONCE(ret == -EBUSY); + if (ret) + return ERR_PTR(ret); } - sg_table = __map_dma_buf(attach, direction); + sg_table = attach->dmabuf->ops->map_dma_buf(attach, direction); if (!sg_table) sg_table = ERR_PTR(-ENOMEM); + if (IS_ERR(sg_table)) + goto error_unpin; - if (IS_ERR(sg_table) && dma_buf_is_dynamic(attach->dmabuf) && - !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) - attach->dmabuf->ops->unpin(attach); - - if (!IS_ERR(sg_table) && attach->dmabuf->ops->cache_sgt_mapping) { - attach->sgt = sg_table; - attach->dir = direction; + /* + * Importers with static attachments don't wait for fences. + */ + if (!dma_buf_attachment_is_dynamic(attach)) { + ret = dma_resv_wait_timeout(attach->dmabuf->resv, + DMA_RESV_USAGE_KERNEL, true, + MAX_SCHEDULE_TIMEOUT); + if (ret < 0) + goto error_unmap; } + mangle_sg_table(sg_table); #ifdef CONFIG_DMA_API_DEBUG - if (!IS_ERR(sg_table)) { + { struct scatterlist *sg; u64 addr; int len; @@ -1164,8 +1151,18 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *attach, } #endif /* CONFIG_DMA_API_DEBUG */ return sg_table; + +error_unmap: + attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); + sg_table = ERR_PTR(ret); + +error_unpin: + if (dma_buf_pin_on_map(attach)) + attach->dmabuf->ops->unpin(attach); + + return sg_table; } -EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment, "DMA_BUF"); /** * dma_buf_map_attachment_unlocked - Returns the scatterlist table of the attachment; @@ -1193,7 +1190,7 @@ dma_buf_map_attachment_unlocked(struct dma_buf_attachment *attach, return sg_table; } -EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment_unlocked, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_map_attachment_unlocked, "DMA_BUF"); /** * dma_buf_unmap_attachment - unmaps and decreases usecount of the buffer;might @@ -1216,16 +1213,13 @@ void dma_buf_unmap_attachment(struct dma_buf_attachment *attach, dma_resv_assert_held(attach->dmabuf->resv); - if (attach->sgt == sg_table) - return; - - __unmap_dma_buf(attach, sg_table, direction); + mangle_sg_table(sg_table); + attach->dmabuf->ops->unmap_dma_buf(attach, sg_table, direction); - if (dma_buf_is_dynamic(attach->dmabuf) && - !IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) - dma_buf_unpin(attach); + if (dma_buf_pin_on_map(attach)) + attach->dmabuf->ops->unpin(attach); } -EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment, "DMA_BUF"); /** * dma_buf_unmap_attachment_unlocked - unmaps and decreases usecount of the buffer;might @@ -1250,7 +1244,7 @@ void dma_buf_unmap_attachment_unlocked(struct dma_buf_attachment *attach, dma_buf_unmap_attachment(attach, sg_table, direction); dma_resv_unlock(attach->dmabuf->resv); } -EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment_unlocked, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_unmap_attachment_unlocked, "DMA_BUF"); /** * dma_buf_move_notify - notify attachments that DMA-buf is moving @@ -1270,7 +1264,7 @@ void dma_buf_move_notify(struct dma_buf *dmabuf) if (attach->importer_ops) attach->importer_ops->move_notify(attach); } -EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, "DMA_BUF"); /** * DOC: cpu access @@ -1287,10 +1281,12 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, DMA_BUF); * vmap interface is introduced. Note that on very old 32-bit architectures * vmalloc space might be limited and result in vmap calls failing. * - * Interfaces:: + * Interfaces: * - * void \*dma_buf_vmap(struct dma_buf \*dmabuf, struct iosys_map \*map) - * void dma_buf_vunmap(struct dma_buf \*dmabuf, struct iosys_map \*map) + * .. code-block:: c + * + * void *dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map) + * void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) * * The vmap call can fail if there is no vmap support in the exporter, or if * it runs out of vmalloc space. Note that the dma-buf layer keeps a reference @@ -1347,10 +1343,11 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_move_notify, DMA_BUF); * enough, since adding interfaces to intercept pagefaults and allow pte * shootdowns would increase the complexity quite a bit. * - * Interface:: + * Interface: + * + * .. code-block:: c * - * int dma_buf_mmap(struct dma_buf \*, struct vm_area_struct \*, - * unsigned long); + * int dma_buf_mmap(struct dma_buf *, struct vm_area_struct *, unsigned long); * * If the importing subsystem simply provides a special-purpose mmap call to * set up a mapping in userspace, calling do_mmap with &dma_buf.file will @@ -1415,7 +1412,7 @@ int dma_buf_begin_cpu_access(struct dma_buf *dmabuf, return ret; } -EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_begin_cpu_access, "DMA_BUF"); /** * dma_buf_end_cpu_access - Must be called after accessing a dma_buf from the @@ -1443,7 +1440,7 @@ int dma_buf_end_cpu_access(struct dma_buf *dmabuf, return ret; } -EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, "DMA_BUF"); /** @@ -1463,8 +1460,6 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_end_cpu_access, DMA_BUF); int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, unsigned long pgoff) { - int ret; - if (WARN_ON(!dmabuf || !vma)) return -EINVAL; @@ -1485,13 +1480,9 @@ int dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma, vma_set_file(vma, dmabuf->file); vma->vm_pgoff = pgoff; - dma_resv_lock(dmabuf->resv, NULL); - ret = dmabuf->ops->mmap(dmabuf, vma); - dma_resv_unlock(dmabuf->resv); - - return ret; + return dmabuf->ops->mmap(dmabuf, vma); } -EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_mmap, "DMA_BUF"); /** * dma_buf_vmap - Create virtual mapping for the buffer object into kernel @@ -1544,7 +1535,7 @@ int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map) return 0; } -EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, "DMA_BUF"); /** * dma_buf_vmap_unlocked - Create virtual mapping for the buffer object into kernel @@ -1571,7 +1562,7 @@ int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map) return ret; } -EXPORT_SYMBOL_NS_GPL(dma_buf_vmap_unlocked, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_vmap_unlocked, "DMA_BUF"); /** * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap. @@ -1595,7 +1586,7 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) iosys_map_clear(&dmabuf->vmap_ptr); } } -EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, "DMA_BUF"); /** * dma_buf_vunmap_unlocked - Unmap a vmap obtained by dma_buf_vmap. @@ -1611,7 +1602,7 @@ void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map) dma_buf_vunmap(dmabuf, map); dma_resv_unlock(dmabuf->resv); } -EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, DMA_BUF); +EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, "DMA_BUF"); #ifdef CONFIG_DEBUG_FS static int dma_buf_debug_show(struct seq_file *s, void *unused) @@ -1622,7 +1613,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) size_t size = 0; int ret; - ret = mutex_lock_interruptible(&db_list.lock); + ret = mutex_lock_interruptible(&dmabuf_list_mutex); if (ret) return ret; @@ -1631,7 +1622,7 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\tname\n", "size", "flags", "mode", "count", "ino"); - list_for_each_entry(buf_obj, &db_list.head, list_node) { + list_for_each_entry(buf_obj, &dmabuf_list, list_node) { ret = dma_resv_lock_interruptible(buf_obj->resv, NULL); if (ret) @@ -1668,11 +1659,11 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) seq_printf(s, "\nTotal %d objects, %zu bytes\n", count, size); - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); return 0; error_unlock: - mutex_unlock(&db_list.lock); + mutex_unlock(&dmabuf_list_mutex); return ret; } @@ -1691,7 +1682,7 @@ static int dma_buf_init_debugfs(void) dma_buf_debugfs_dir = d; - d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir, + d = debugfs_create_file("bufinfo", 0444, dma_buf_debugfs_dir, NULL, &dma_buf_debug_fops); if (IS_ERR(d)) { pr_debug("dma_buf: debugfs: failed to create node bufinfo\n"); @@ -1729,8 +1720,6 @@ static int __init dma_buf_init(void) if (IS_ERR(dma_buf_mnt)) return PTR_ERR(dma_buf_mnt); - mutex_init(&db_list.lock); - INIT_LIST_HEAD(&db_list.head); dma_buf_init_debugfs(); return 0; } diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c index 5c8a7084577b..6657d4b30af9 100644 --- a/drivers/dma-buf/dma-fence-array.c +++ b/drivers/dma-buf/dma-fence-array.c @@ -70,7 +70,7 @@ static void dma_fence_array_cb_func(struct dma_fence *f, static bool dma_fence_array_enable_signaling(struct dma_fence *fence) { struct dma_fence_array *array = to_dma_fence_array(fence); - struct dma_fence_array_cb *cb = (void *)(&array[1]); + struct dma_fence_array_cb *cb = array->callbacks; unsigned i; for (i = 0; i < array->num_fences; ++i) { @@ -103,10 +103,36 @@ static bool dma_fence_array_enable_signaling(struct dma_fence *fence) static bool dma_fence_array_signaled(struct dma_fence *fence) { struct dma_fence_array *array = to_dma_fence_array(fence); + int num_pending; + unsigned int i; - if (atomic_read(&array->num_pending) > 0) + /* + * We need to read num_pending before checking the enable_signal bit + * to avoid racing with the enable_signaling() implementation, which + * might decrement the counter, and cause a partial check. + * atomic_read_acquire() pairs with atomic_dec_and_test() in + * dma_fence_array_enable_signaling() + * + * The !--num_pending check is here to account for the any_signaled case + * if we race with enable_signaling(), that means the !num_pending check + * in the is_signalling_enabled branch might be outdated (num_pending + * might have been decremented), but that's fine. The user will get the + * right value when testing again later. + */ + num_pending = atomic_read_acquire(&array->num_pending); + if (test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &array->base.flags)) { + if (num_pending <= 0) + goto signal; return false; + } + + for (i = 0; i < array->num_fences; ++i) { + if (dma_fence_is_signaled(array->fences[i]) && !--num_pending) + goto signal; + } + return false; +signal: dma_fence_array_clear_pending_error(array); return true; } @@ -123,56 +149,66 @@ static void dma_fence_array_release(struct dma_fence *fence) dma_fence_free(fence); } +static void dma_fence_array_set_deadline(struct dma_fence *fence, + ktime_t deadline) +{ + struct dma_fence_array *array = to_dma_fence_array(fence); + unsigned i; + + for (i = 0; i < array->num_fences; ++i) + dma_fence_set_deadline(array->fences[i], deadline); +} + const struct dma_fence_ops dma_fence_array_ops = { .get_driver_name = dma_fence_array_get_driver_name, .get_timeline_name = dma_fence_array_get_timeline_name, .enable_signaling = dma_fence_array_enable_signaling, .signaled = dma_fence_array_signaled, .release = dma_fence_array_release, + .set_deadline = dma_fence_array_set_deadline, }; EXPORT_SYMBOL(dma_fence_array_ops); /** - * dma_fence_array_create - Create a custom fence array + * dma_fence_array_alloc - Allocate a custom fence array + * @num_fences: [in] number of fences to add in the array + * + * Return dma fence array on success, NULL on failure + */ +struct dma_fence_array *dma_fence_array_alloc(int num_fences) +{ + struct dma_fence_array *array; + + return kzalloc(struct_size(array, callbacks, num_fences), GFP_KERNEL); +} +EXPORT_SYMBOL(dma_fence_array_alloc); + +/** + * dma_fence_array_init - Init a custom fence array + * @array: [in] dma fence array to arm * @num_fences: [in] number of fences to add in the array * @fences: [in] array containing the fences * @context: [in] fence context to use * @seqno: [in] sequence number to use * @signal_on_any: [in] signal on any fence in the array * - * Allocate a dma_fence_array object and initialize the base fence with - * dma_fence_init(). - * In case of error it returns NULL. - * - * The caller should allocate the fences array with num_fences size - * and fill it with the fences it wants to add to the object. Ownership of this - * array is taken and dma_fence_put() is used on each fence on release. - * - * If @signal_on_any is true the fence array signals if any fence in the array - * signals, otherwise it signals when all fences in the array signal. + * Implementation of @dma_fence_array_create without allocation. Useful to init + * a preallocated dma fence array in the path of reclaim or dma fence signaling. */ -struct dma_fence_array *dma_fence_array_create(int num_fences, - struct dma_fence **fences, - u64 context, unsigned seqno, - bool signal_on_any) +void dma_fence_array_init(struct dma_fence_array *array, + int num_fences, struct dma_fence **fences, + u64 context, unsigned seqno, + bool signal_on_any) { - struct dma_fence_array *array; - size_t size = sizeof(*array); - WARN_ON(!num_fences || !fences); - /* Allocate the callback structures behind the array. */ - size += num_fences * sizeof(struct dma_fence_array_cb); - array = kzalloc(size, GFP_KERNEL); - if (!array) - return NULL; + array->num_fences = num_fences; spin_lock_init(&array->lock); dma_fence_init(&array->base, &dma_fence_array_ops, &array->lock, context, seqno); init_irq_work(&array->work, irq_dma_fence_array_work); - array->num_fences = num_fences; atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences); array->fences = fences; @@ -191,6 +227,41 @@ struct dma_fence_array *dma_fence_array_create(int num_fences, */ while (num_fences--) WARN_ON(dma_fence_is_container(fences[num_fences])); +} +EXPORT_SYMBOL(dma_fence_array_init); + +/** + * dma_fence_array_create - Create a custom fence array + * @num_fences: [in] number of fences to add in the array + * @fences: [in] array containing the fences + * @context: [in] fence context to use + * @seqno: [in] sequence number to use + * @signal_on_any: [in] signal on any fence in the array + * + * Allocate a dma_fence_array object and initialize the base fence with + * dma_fence_init(). + * In case of error it returns NULL. + * + * The caller should allocate the fences array with num_fences size + * and fill it with the fences it wants to add to the object. Ownership of this + * array is taken and dma_fence_put() is used on each fence on release. + * + * If @signal_on_any is true the fence array signals if any fence in the array + * signals, otherwise it signals when all fences in the array signal. + */ +struct dma_fence_array *dma_fence_array_create(int num_fences, + struct dma_fence **fences, + u64 context, unsigned seqno, + bool signal_on_any) +{ + struct dma_fence_array *array; + + array = dma_fence_array_alloc(num_fences); + if (!array) + return NULL; + + dma_fence_array_init(array, num_fences, fences, + context, seqno, signal_on_any); return array; } diff --git a/drivers/dma-buf/dma-fence-chain.c b/drivers/dma-buf/dma-fence-chain.c index a0d920576ba6..a8a90acf4f34 100644 --- a/drivers/dma-buf/dma-fence-chain.c +++ b/drivers/dma-buf/dma-fence-chain.c @@ -206,13 +206,24 @@ static void dma_fence_chain_release(struct dma_fence *fence) dma_fence_free(fence); } + +static void dma_fence_chain_set_deadline(struct dma_fence *fence, + ktime_t deadline) +{ + dma_fence_chain_for_each(fence, fence) { + struct dma_fence *f = dma_fence_chain_contained(fence); + + dma_fence_set_deadline(f, deadline); + } +} + const struct dma_fence_ops dma_fence_chain_ops = { - .use_64bit_seqno = true, .get_driver_name = dma_fence_chain_get_driver_name, .get_timeline_name = dma_fence_chain_get_timeline_name, .enable_signaling = dma_fence_chain_enable_signaling, .signaled = dma_fence_chain_signaled, .release = dma_fence_chain_release, + .set_deadline = dma_fence_chain_set_deadline, }; EXPORT_SYMBOL(dma_fence_chain_ops); @@ -240,7 +251,7 @@ void dma_fence_chain_init(struct dma_fence_chain *chain, chain->prev_seqno = 0; /* Try to reuse the context of the previous chain node. */ - if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) { + if (prev_chain && __dma_fence_is_later(prev, seqno, prev->seqno)) { context = prev->context; chain->prev_seqno = prev->seqno; } else { @@ -250,8 +261,8 @@ void dma_fence_chain_init(struct dma_fence_chain *chain, seqno = max(prev->seqno, seqno); } - dma_fence_init(&chain->base, &dma_fence_chain_ops, - &chain->lock, context, seqno); + dma_fence_init64(&chain->base, &dma_fence_chain_ops, &chain->lock, + context, seqno); /* * Chaining dma_fence_chain container together is only allowed through diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c index 7002bca792ff..a495d8a6c2e3 100644 --- a/drivers/dma-buf/dma-fence-unwrap.c +++ b/drivers/dma-buf/dma-fence-unwrap.c @@ -12,6 +12,7 @@ #include <linux/dma-fence-chain.h> #include <linux/dma-fence-unwrap.h> #include <linux/slab.h> +#include <linux/sort.h> /* Internal helper to start new array iteration, don't use directly */ static struct dma_fence * @@ -59,102 +60,142 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) } EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); + +static int fence_cmp(const void *_a, const void *_b) +{ + struct dma_fence *a = *(struct dma_fence **)_a; + struct dma_fence *b = *(struct dma_fence **)_b; + + if (a->context < b->context) + return -1; + else if (a->context > b->context) + return 1; + + if (dma_fence_is_later(b, a)) + return 1; + else if (dma_fence_is_later(a, b)) + return -1; + + return 0; +} + +/** + * dma_fence_dedup_array - Sort and deduplicate an array of dma_fence pointers + * @fences: Array of dma_fence pointers to be deduplicated + * @num_fences: Number of entries in the @fences array + * + * Sorts the input array by context, then removes duplicate + * fences with the same context, keeping only the most recent one. + * + * The array is modified in-place and unreferenced duplicate fences are released + * via dma_fence_put(). The function returns the new number of fences after + * deduplication. + * + * Return: Number of unique fences remaining in the array. + */ +int dma_fence_dedup_array(struct dma_fence **fences, int num_fences) +{ + int i, j; + + sort(fences, num_fences, sizeof(*fences), fence_cmp, NULL); + + /* + * Only keep the most recent fence for each context. + */ + j = 0; + for (i = 1; i < num_fences; i++) { + if (fences[i]->context == fences[j]->context) + dma_fence_put(fences[i]); + else + fences[++j] = fences[i]; + } + + return ++j; +} +EXPORT_SYMBOL_GPL(dma_fence_dedup_array); + /* Implementation for the dma_fence_merge() marco, don't use directly */ struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, struct dma_fence **fences, struct dma_fence_unwrap *iter) { + struct dma_fence *tmp, *unsignaled = NULL, **array; struct dma_fence_array *result; - struct dma_fence *tmp, **array; - unsigned int i; - size_t count; + ktime_t timestamp; + int i, count; count = 0; + timestamp = ns_to_ktime(0); for (i = 0; i < num_fences; ++i) { - dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) - if (!dma_fence_is_signaled(tmp)) + dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { + if (!dma_fence_is_signaled(tmp)) { + dma_fence_put(unsignaled); + unsignaled = dma_fence_get(tmp); ++count; + } else { + ktime_t t = dma_fence_timestamp(tmp); + + if (ktime_after(t, timestamp)) + timestamp = t; + } + } } + /* + * If we couldn't find a pending fence just return a private signaled + * fence with the timestamp of the last signaled one. + * + * Or if there was a single unsignaled fence left we can return it + * directly and early since that is a major path on many workloads. + */ if (count == 0) - return dma_fence_get_stub(); + return dma_fence_allocate_private_stub(timestamp); + else if (count == 1) + return unsignaled; + + dma_fence_put(unsignaled); array = kmalloc_array(count, sizeof(*array), GFP_KERNEL); if (!array) return NULL; - /* - * This trashes the input fence array and uses it as position for the - * following merge loop. This works because the dma_fence_merge() - * wrapper macro is creating this temporary array on the stack together - * with the iterators. - */ - for (i = 0; i < num_fences; ++i) - fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]); - count = 0; - do { - unsigned int sel; - -restart: - tmp = NULL; - for (i = 0; i < num_fences; ++i) { - struct dma_fence *next; - - while (fences[i] && dma_fence_is_signaled(fences[i])) - fences[i] = dma_fence_unwrap_next(&iter[i]); - - next = fences[i]; - if (!next) - continue; - - /* - * We can't guarantee that inpute fences are ordered by - * context, but it is still quite likely when this - * function is used multiple times. So attempt to order - * the fences by context as we pass over them and merge - * fences with the same context. - */ - if (!tmp || tmp->context > next->context) { - tmp = next; - sel = i; - - } else if (tmp->context < next->context) { - continue; - - } else if (dma_fence_is_later(tmp, next)) { - fences[i] = dma_fence_unwrap_next(&iter[i]); - goto restart; + for (i = 0; i < num_fences; ++i) { + dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { + if (!dma_fence_is_signaled(tmp)) { + array[count++] = dma_fence_get(tmp); } else { - fences[sel] = dma_fence_unwrap_next(&iter[sel]); - goto restart; + ktime_t t = dma_fence_timestamp(tmp); + + if (ktime_after(t, timestamp)) + timestamp = t; } } + } - if (tmp) { - array[count++] = dma_fence_get(tmp); - fences[sel] = dma_fence_unwrap_next(&iter[sel]); - } - } while (tmp); + if (count == 0 || count == 1) + goto return_fastpath; - if (count == 0) { - tmp = dma_fence_get_stub(); - goto return_tmp; - } + count = dma_fence_dedup_array(array, count); - if (count == 1) { - tmp = array[0]; - goto return_tmp; + if (count > 1) { + result = dma_fence_array_create(count, array, + dma_fence_context_alloc(1), + 1, false); + if (!result) { + for (i = 0; i < count; i++) + dma_fence_put(array[i]); + tmp = NULL; + goto return_tmp; + } + return &result->base; } - result = dma_fence_array_create(count, array, - dma_fence_context_alloc(1), - 1, false); - if (!result) { - tmp = NULL; - goto return_tmp; - } - return &result->base; +return_fastpath: + if (count == 0) + tmp = dma_fence_allocate_private_stub(timestamp); + else + tmp = array[0]; return_tmp: kfree(array); diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 0de0482cd36e..b4f5c8635276 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -102,7 +102,7 @@ static atomic64_t dma_fence_context_counter = ATOMIC64_INIT(1); * * * Drivers are allowed to call dma_fence_wait() from their &mmu_notifier * respectively &mmu_interval_notifier callbacks. This means any code required - * for fence completeion cannot allocate memory with GFP_NOFS or GFP_NOIO. + * for fence completion cannot allocate memory with GFP_NOFS or GFP_NOIO. * Only GFP_ATOMIC is permissible, which might fail. * * Note that only GPU drivers have a reasonable excuse for both requiring @@ -121,45 +121,44 @@ static const struct dma_fence_ops dma_fence_stub_ops = { .get_timeline_name = dma_fence_stub_get_name, }; +static int __init dma_fence_init_stub(void) +{ + dma_fence_init(&dma_fence_stub, &dma_fence_stub_ops, + &dma_fence_stub_lock, 0, 0); + + set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &dma_fence_stub.flags); + + dma_fence_signal(&dma_fence_stub); + return 0; +} +subsys_initcall(dma_fence_init_stub); + /** * dma_fence_get_stub - return a signaled fence * - * Return a stub fence which is already signaled. The fence's - * timestamp corresponds to the first time after boot this - * function is called. + * Return a stub fence which is already signaled. The fence's timestamp + * corresponds to the initialisation time of the linux kernel. */ struct dma_fence *dma_fence_get_stub(void) { - spin_lock(&dma_fence_stub_lock); - if (!dma_fence_stub.ops) { - dma_fence_init(&dma_fence_stub, - &dma_fence_stub_ops, - &dma_fence_stub_lock, - 0, 0); - - set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, - &dma_fence_stub.flags); - - dma_fence_signal_locked(&dma_fence_stub); - } - spin_unlock(&dma_fence_stub_lock); - return dma_fence_get(&dma_fence_stub); } EXPORT_SYMBOL(dma_fence_get_stub); /** * dma_fence_allocate_private_stub - return a private, signaled fence + * @timestamp: timestamp when the fence was signaled * * Return a newly allocated and signaled stub fence. */ -struct dma_fence *dma_fence_allocate_private_stub(void) +struct dma_fence *dma_fence_allocate_private_stub(ktime_t timestamp) { struct dma_fence *fence; fence = kzalloc(sizeof(*fence), GFP_KERNEL); if (fence == NULL) - return ERR_PTR(-ENOMEM); + return NULL; dma_fence_init(fence, &dma_fence_stub_ops, @@ -169,7 +168,7 @@ struct dma_fence *dma_fence_allocate_private_stub(void) set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, &fence->flags); - dma_fence_signal(fence); + dma_fence_signal_timestamp(fence, timestamp); return fence; } @@ -308,8 +307,8 @@ bool dma_fence_begin_signalling(void) if (in_atomic()) return true; - /* ... and non-recursive readlock */ - lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _RET_IP_); + /* ... and non-recursive successful read_trylock */ + lock_acquire(&dma_fence_lockdep_map, 0, 1, 1, 1, NULL, _RET_IP_); return false; } @@ -340,7 +339,7 @@ void __dma_fence_might_wait(void) lock_map_acquire(&dma_fence_lockdep_map); lock_map_release(&dma_fence_lockdep_map); if (tmp) - lock_acquire(&dma_fence_lockdep_map, 0, 0, 1, 1, NULL, _THIS_IP_); + lock_acquire(&dma_fence_lockdep_map, 0, 1, 1, 1, NULL, _THIS_IP_); } #endif @@ -411,7 +410,7 @@ int dma_fence_signal_timestamp(struct dma_fence *fence, ktime_t timestamp) unsigned long flags; int ret; - if (!fence) + if (WARN_ON(!fence)) return -EINVAL; spin_lock_irqsave(fence->lock, flags); @@ -463,7 +462,7 @@ int dma_fence_signal(struct dma_fence *fence) int ret; bool tmp; - if (!fence) + if (WARN_ON(!fence)) return -EINVAL; tmp = dma_fence_begin_signalling(); @@ -510,18 +509,26 @@ dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout) dma_fence_enable_sw_signaling(fence); - trace_dma_fence_wait_start(fence); + if (trace_dma_fence_wait_start_enabled()) { + rcu_read_lock(); + trace_dma_fence_wait_start(fence); + rcu_read_unlock(); + } if (fence->ops->wait) ret = fence->ops->wait(fence, intr, timeout); else ret = dma_fence_default_wait(fence, intr, timeout); - trace_dma_fence_wait_end(fence); + if (trace_dma_fence_wait_end_enabled()) { + rcu_read_lock(); + trace_dma_fence_wait_end(fence); + rcu_read_unlock(); + } return ret; } EXPORT_SYMBOL(dma_fence_wait_timeout); /** - * dma_fence_release - default relese function for fences + * dma_fence_release - default release function for fences * @kref: &dma_fence.recfount * * This is the default release functions for &dma_fence. Drivers shouldn't call @@ -532,16 +539,23 @@ void dma_fence_release(struct kref *kref) struct dma_fence *fence = container_of(kref, struct dma_fence, refcount); + rcu_read_lock(); trace_dma_fence_destroy(fence); - if (WARN(!list_empty(&fence->cb_list) && - !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags), - "Fence %s:%s:%llx:%llx released with pending signals!\n", - fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), - fence->context, fence->seqno)) { + if (!list_empty(&fence->cb_list) && + !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { + const char __rcu *timeline; + const char __rcu *driver; unsigned long flags; + driver = dma_fence_driver_name(fence); + timeline = dma_fence_timeline_name(fence); + + WARN(1, + "Fence %s:%s:%llx:%llx released with pending signals!\n", + rcu_dereference(driver), rcu_dereference(timeline), + fence->context, fence->seqno); + /* * Failed to signal before release, likely a refcounting issue. * @@ -555,6 +569,8 @@ void dma_fence_release(struct kref *kref) spin_unlock_irqrestore(fence->lock, flags); } + rcu_read_unlock(); + if (fence->ops->release) fence->ops->release(fence); else @@ -913,21 +929,113 @@ err_free_cb: EXPORT_SYMBOL(dma_fence_wait_any_timeout); /** - * dma_fence_describe - Dump fence describtion into seq_file - * @fence: the 6fence to describe + * DOC: deadline hints + * + * In an ideal world, it would be possible to pipeline a workload sufficiently + * that a utilization based device frequency governor could arrive at a minimum + * frequency that meets the requirements of the use-case, in order to minimize + * power consumption. But in the real world there are many workloads which + * defy this ideal. For example, but not limited to: + * + * * Workloads that ping-pong between device and CPU, with alternating periods + * of CPU waiting for device, and device waiting on CPU. This can result in + * devfreq and cpufreq seeing idle time in their respective domains and in + * result reduce frequency. + * + * * Workloads that interact with a periodic time based deadline, such as double + * buffered GPU rendering vs vblank sync'd page flipping. In this scenario, + * missing a vblank deadline results in an *increase* in idle time on the GPU + * (since it has to wait an additional vblank period), sending a signal to + * the GPU's devfreq to reduce frequency, when in fact the opposite is what is + * needed. + * + * To this end, deadline hint(s) can be set on a &dma_fence via &dma_fence_set_deadline + * (or indirectly via userspace facing ioctls like &sync_set_deadline). + * The deadline hint provides a way for the waiting driver, or userspace, to + * convey an appropriate sense of urgency to the signaling driver. + * + * A deadline hint is given in absolute ktime (CLOCK_MONOTONIC for userspace + * facing APIs). The time could either be some point in the future (such as + * the vblank based deadline for page-flipping, or the start of a compositor's + * composition cycle), or the current time to indicate an immediate deadline + * hint (Ie. forward progress cannot be made until this fence is signaled). + * + * Multiple deadlines may be set on a given fence, even in parallel. See the + * documentation for &dma_fence_ops.set_deadline. + * + * The deadline hint is just that, a hint. The driver that created the fence + * may react by increasing frequency, making different scheduling choices, etc. + * Or doing nothing at all. + */ + +/** + * dma_fence_set_deadline - set desired fence-wait deadline hint + * @fence: the fence that is to be waited on + * @deadline: the time by which the waiter hopes for the fence to be + * signaled + * + * Give the fence signaler a hint about an upcoming deadline, such as + * vblank, by which point the waiter would prefer the fence to be + * signaled by. This is intended to give feedback to the fence signaler + * to aid in power management decisions, such as boosting GPU frequency + * if a periodic vblank deadline is approaching but the fence is not + * yet signaled.. + */ +void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) +{ + if (fence->ops->set_deadline && !dma_fence_is_signaled(fence)) + fence->ops->set_deadline(fence, deadline); +} +EXPORT_SYMBOL(dma_fence_set_deadline); + +/** + * dma_fence_describe - Dump fence description into seq_file + * @fence: the fence to describe * @seq: the seq_file to put the textual description into * * Dump a textual description of the fence and it's state into the seq_file. */ void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq) { - seq_printf(seq, "%s %s seq %llu %ssignalled\n", - fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), fence->seqno, - dma_fence_is_signaled(fence) ? "" : "un"); + const char __rcu *timeline = ""; + const char __rcu *driver = ""; + const char *signaled = ""; + + rcu_read_lock(); + + if (!dma_fence_is_signaled(fence)) { + timeline = dma_fence_timeline_name(fence); + driver = dma_fence_driver_name(fence); + signaled = "un"; + } + + seq_printf(seq, "%llu:%llu %s %s %ssignalled\n", + fence->context, fence->seqno, timeline, driver, + signaled); + + rcu_read_unlock(); } EXPORT_SYMBOL(dma_fence_describe); +static void +__dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, + spinlock_t *lock, u64 context, u64 seqno, unsigned long flags) +{ + BUG_ON(!lock); + BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name); + + kref_init(&fence->refcount); + fence->ops = ops; + INIT_LIST_HEAD(&fence->cb_list); + fence->lock = lock; + fence->context = context; + fence->seqno = seqno; + fence->flags = flags; + fence->error = 0; + + trace_dma_fence_init(fence); +} + /** * dma_fence_init - Initialize a custom fence. * @fence: the fence to initialize @@ -947,18 +1055,94 @@ void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, spinlock_t *lock, u64 context, u64 seqno) { - BUG_ON(!lock); - BUG_ON(!ops || !ops->get_driver_name || !ops->get_timeline_name); + __dma_fence_init(fence, ops, lock, context, seqno, 0UL); +} +EXPORT_SYMBOL(dma_fence_init); - kref_init(&fence->refcount); - fence->ops = ops; - INIT_LIST_HEAD(&fence->cb_list); - fence->lock = lock; - fence->context = context; - fence->seqno = seqno; - fence->flags = 0UL; - fence->error = 0; +/** + * dma_fence_init64 - Initialize a custom fence with 64-bit seqno support. + * @fence: the fence to initialize + * @ops: the dma_fence_ops for operations on this fence + * @lock: the irqsafe spinlock to use for locking this fence + * @context: the execution context this fence is run on + * @seqno: a linear increasing sequence number for this context + * + * Initializes an allocated fence, the caller doesn't have to keep its + * refcount after committing with this fence, but it will need to hold a + * refcount again if &dma_fence_ops.enable_signaling gets called. + * + * Context and seqno are used for easy comparison between fences, allowing + * to check which fence is later by simply using dma_fence_later(). + */ +void +dma_fence_init64(struct dma_fence *fence, const struct dma_fence_ops *ops, + spinlock_t *lock, u64 context, u64 seqno) +{ + __dma_fence_init(fence, ops, lock, context, seqno, + BIT(DMA_FENCE_FLAG_SEQNO64_BIT)); +} +EXPORT_SYMBOL(dma_fence_init64); - trace_dma_fence_init(fence); +/** + * dma_fence_driver_name - Access the driver name + * @fence: the fence to query + * + * Returns a driver name backing the dma-fence implementation. + * + * IMPORTANT CONSIDERATION: + * Dma-fence contract stipulates that access to driver provided data (data not + * directly embedded into the object itself), such as the &dma_fence.lock and + * memory potentially accessed by the &dma_fence.ops functions, is forbidden + * after the fence has been signalled. Drivers are allowed to free that data, + * and some do. + * + * To allow safe access drivers are mandated to guarantee a RCU grace period + * between signalling the fence and freeing said data. + * + * As such access to the driver name is only valid inside a RCU locked section. + * The pointer MUST be both queried and USED ONLY WITHIN a SINGLE block guarded + * by the &rcu_read_lock and &rcu_read_unlock pair. + */ +const char __rcu *dma_fence_driver_name(struct dma_fence *fence) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "RCU protection is required for safe access to returned string"); + + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return fence->ops->get_driver_name(fence); + else + return "detached-driver"; } -EXPORT_SYMBOL(dma_fence_init); +EXPORT_SYMBOL(dma_fence_driver_name); + +/** + * dma_fence_timeline_name - Access the timeline name + * @fence: the fence to query + * + * Returns a timeline name provided by the dma-fence implementation. + * + * IMPORTANT CONSIDERATION: + * Dma-fence contract stipulates that access to driver provided data (data not + * directly embedded into the object itself), such as the &dma_fence.lock and + * memory potentially accessed by the &dma_fence.ops functions, is forbidden + * after the fence has been signalled. Drivers are allowed to free that data, + * and some do. + * + * To allow safe access drivers are mandated to guarantee a RCU grace period + * between signalling the fence and freeing said data. + * + * As such access to the driver name is only valid inside a RCU locked section. + * The pointer MUST be both queried and USED ONLY WITHIN a SINGLE block guarded + * by the &rcu_read_lock and &rcu_read_unlock pair. + */ +const char __rcu *dma_fence_timeline_name(struct dma_fence *fence) +{ + RCU_LOCKDEP_WARN(!rcu_read_lock_held(), + "RCU protection is required for safe access to returned string"); + + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) + return fence->ops->get_timeline_name(fence); + else + return "signaled-timeline"; +} +EXPORT_SYMBOL(dma_fence_timeline_name); diff --git a/drivers/dma-buf/dma-heap.c b/drivers/dma-buf/dma-heap.c index c9e41e8a9e27..8ab49924f8b7 100644 --- a/drivers/dma-buf/dma-heap.c +++ b/drivers/dma-buf/dma-heap.c @@ -7,17 +7,16 @@ */ #include <linux/cdev.h> -#include <linux/debugfs.h> #include <linux/device.h> #include <linux/dma-buf.h> +#include <linux/dma-heap.h> #include <linux/err.h> -#include <linux/xarray.h> +#include <linux/export.h> #include <linux/list.h> -#include <linux/slab.h> #include <linux/nospec.h> -#include <linux/uaccess.h> #include <linux/syscalls.h> -#include <linux/dma-heap.h> +#include <linux/uaccess.h> +#include <linux/xarray.h> #include <uapi/linux/dma-heap.h> #define DEVNAME "dma_heap" @@ -28,9 +27,10 @@ * struct dma_heap - represents a dmabuf heap in the system * @name: used for debugging/device-node name * @ops: ops struct for this heap - * @heap_devt heap device node - * @list list head connecting to list of heaps - * @heap_cdev heap char device + * @priv: private data for this heap + * @heap_devt: heap device node + * @list: list head connecting to list of heaps + * @heap_cdev: heap char device * * Represents a heap of memory from which buffers can be made. */ @@ -50,8 +50,8 @@ static struct class *dma_heap_class; static DEFINE_XARRAY_ALLOC(dma_heap_minors); static int dma_heap_buffer_alloc(struct dma_heap *heap, size_t len, - unsigned int fd_flags, - unsigned int heap_flags) + u32 fd_flags, + u64 heap_flags) { struct dma_buf *dmabuf; int fd; @@ -193,20 +193,21 @@ static const struct file_operations dma_heap_fops = { }; /** - * dma_heap_get_drvdata() - get per-subdriver data for the heap + * dma_heap_get_drvdata - get per-heap driver data * @heap: DMA-Heap to retrieve private data for * * Returns: - * The per-subdriver data for the heap. + * The per-heap data for the heap. */ void *dma_heap_get_drvdata(struct dma_heap *heap) { return heap->priv; } +EXPORT_SYMBOL_NS_GPL(dma_heap_get_drvdata, "DMA_BUF_HEAP"); /** - * dma_heap_get_name() - get heap name - * @heap: DMA-Heap to retrieve private data for + * dma_heap_get_name - get heap name + * @heap: DMA-Heap to retrieve the name of * * Returns: * The char* for the heap name. @@ -215,7 +216,12 @@ const char *dma_heap_get_name(struct dma_heap *heap) { return heap->name; } +EXPORT_SYMBOL_NS_GPL(dma_heap_get_name, "DMA_BUF_HEAP"); +/** + * dma_heap_add - adds a heap to dmabuf heaps + * @exp_info: information needed to register this heap + */ struct dma_heap *dma_heap_add(const struct dma_heap_export_info *exp_info) { struct dma_heap *heap, *h, *err_ret; @@ -300,6 +306,7 @@ err0: kfree(heap); return err_ret; } +EXPORT_SYMBOL_NS_GPL(dma_heap_add, "DMA_BUF_HEAP"); static char *dma_heap_devnode(const struct device *dev, umode_t *mode) { @@ -314,7 +321,7 @@ static int dma_heap_init(void) if (ret) return ret; - dma_heap_class = class_create(THIS_MODULE, DEVNAME); + dma_heap_class = class_create(DEVNAME); if (IS_ERR(dma_heap_class)) { unregister_chrdev_region(dma_heap_devt, NUM_HEAP_MINORS); return PTR_ERR(dma_heap_class); diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c index 1c76aed8e262..bea3e9858aca 100644 --- a/drivers/dma-buf/dma-resv.c +++ b/drivers/dma-buf/dma-resv.c @@ -186,6 +186,13 @@ int dma_resv_reserve_fences(struct dma_resv *obj, unsigned int num_fences) dma_resv_assert_held(obj); + /* Driver and component code should never call this function with + * num_fences=0. If they do it usually points to bugs when calculating + * the number of needed fences dynamically. + */ + if (WARN_ON(!num_fences)) + return -EINVAL; + old = dma_resv_fences_list(obj); if (old && old->max_fences) { if ((old->num_fences + num_fences) <= old->max_fences) @@ -301,7 +308,7 @@ void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence, dma_resv_list_entry(fobj, i, obj, &old, &old_usage); if ((old->context == fence->context && old_usage >= usage && - dma_fence_is_later(fence, old)) || + dma_fence_is_later_or_same(fence, old)) || dma_fence_is_signaled(old)) { dma_resv_list_set(fobj, i, fence, usage); dma_fence_put(old); @@ -313,8 +320,9 @@ void dma_resv_add_fence(struct dma_resv *obj, struct dma_fence *fence, count++; dma_resv_list_set(fobj, i, fence, usage); - /* pointer update must be visible before we extend the num_fences */ - smp_store_mb(fobj->num_fences, count); + /* fence update must be visible before we extend the num_fences */ + smp_wmb(); + fobj->num_fences = count; } EXPORT_SYMBOL(dma_resv_add_fence); @@ -405,7 +413,7 @@ static void dma_resv_iter_walk_unlocked(struct dma_resv_iter *cursor) * * Beware that the iterator can be restarted. Code which accumulates statistics * or similar needs to check for this with dma_resv_iter_is_restarted(). For - * this reason prefer the locked dma_resv_iter_first() whenver possible. + * this reason prefer the locked dma_resv_iter_first() whenever possible. * * Returns the first fence from an unlocked dma_resv obj. */ @@ -428,7 +436,7 @@ EXPORT_SYMBOL(dma_resv_iter_first_unlocked); * * Beware that the iterator can be restarted. Code which accumulates statistics * or similar needs to check for this with dma_resv_iter_is_restarted(). For - * this reason prefer the locked dma_resv_iter_next() whenver possible. + * this reason prefer the locked dma_resv_iter_next() whenever possible. * * Returns the next fence from an unlocked dma_resv obj. */ @@ -571,6 +579,7 @@ int dma_resv_get_fences(struct dma_resv *obj, enum dma_resv_usage usage, dma_resv_for_each_fence_unlocked(&cursor, fence) { if (dma_resv_iter_is_restarted(&cursor)) { + struct dma_fence **new_fences; unsigned int count; while (*num_fences) @@ -579,13 +588,17 @@ int dma_resv_get_fences(struct dma_resv *obj, enum dma_resv_usage usage, count = cursor.num_fences + 1; /* Eventually re-allocate the array */ - *fences = krealloc_array(*fences, count, - sizeof(void *), - GFP_KERNEL); - if (count && !*fences) { + new_fences = krealloc_array(*fences, count, + sizeof(void *), + GFP_KERNEL); + if (count && !new_fences) { + kfree(*fences); + *fences = NULL; + *num_fences = 0; dma_resv_iter_end(&cursor); return -ENOMEM; } + *fences = new_fences; } (*fences)[(*num_fences)++] = dma_fence_get(fence); @@ -660,7 +673,7 @@ EXPORT_SYMBOL_GPL(dma_resv_get_singleton); * dma_resv_lock() already * RETURNS * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or - * greater than zer on success. + * greater than zero on success. */ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, bool intr, unsigned long timeout) @@ -672,11 +685,13 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, dma_resv_iter_begin(&cursor, obj, usage); dma_resv_for_each_fence_unlocked(&cursor, fence) { - ret = dma_fence_wait_timeout(fence, intr, ret); - if (ret <= 0) { - dma_resv_iter_end(&cursor); - return ret; - } + ret = dma_fence_wait_timeout(fence, intr, timeout); + if (ret <= 0) + break; + + /* Even for zero timeout the return value is 1 */ + if (timeout) + timeout = ret; } dma_resv_iter_end(&cursor); @@ -684,6 +699,28 @@ long dma_resv_wait_timeout(struct dma_resv *obj, enum dma_resv_usage usage, } EXPORT_SYMBOL_GPL(dma_resv_wait_timeout); +/** + * dma_resv_set_deadline - Set a deadline on reservation's objects fences + * @obj: the reservation object + * @usage: controls which fences to include, see enum dma_resv_usage. + * @deadline: the requested deadline (MONOTONIC) + * + * May be called without holding the dma_resv lock. Sets @deadline on + * all fences filtered by @usage. + */ +void dma_resv_set_deadline(struct dma_resv *obj, enum dma_resv_usage usage, + ktime_t deadline) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, obj, usage); + dma_resv_for_each_fence_unlocked(&cursor, fence) { + dma_fence_set_deadline(fence, deadline); + } + dma_resv_iter_end(&cursor); +} +EXPORT_SYMBOL_GPL(dma_resv_set_deadline); /** * dma_resv_test_signaled - Test if a reservation object's fences have been diff --git a/drivers/dma-buf/heaps/cma_heap.c b/drivers/dma-buf/heaps/cma_heap.c index 1131fb943992..42f88193eab9 100644 --- a/drivers/dma-buf/heaps/cma_heap.c +++ b/drivers/dma-buf/heaps/cma_heap.c @@ -9,20 +9,39 @@ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ * Andrew F. Davis <afd@ti.com> */ + +#define pr_fmt(fmt) "cma_heap: " fmt + #include <linux/cma.h> #include <linux/dma-buf.h> +#include <linux/dma-buf/heaps/cma.h> #include <linux/dma-heap.h> #include <linux/dma-map-ops.h> -#include <linux/dma-resv.h> #include <linux/err.h> #include <linux/highmem.h> #include <linux/io.h> #include <linux/mm.h> #include <linux/module.h> +#include <linux/of.h> +#include <linux/of_reserved_mem.h> #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/vmalloc.h> +#define DEFAULT_CMA_NAME "default_cma_region" + +static struct cma *dma_areas[MAX_CMA_AREAS] __initdata; +static unsigned int dma_areas_num __initdata; + +int __init dma_heap_cma_register_heap(struct cma *cma) +{ + if (dma_areas_num >= ARRAY_SIZE(dma_areas)) + return -EINVAL; + + dma_areas[dma_areas_num++] = cma; + + return 0; +} struct cma_heap { struct dma_heap *heap; @@ -166,13 +185,10 @@ static vm_fault_t cma_heap_vm_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct cma_heap_buffer *buffer = vma->vm_private_data; - if (vmf->pgoff > buffer->pagecount) + if (vmf->pgoff >= buffer->pagecount) return VM_FAULT_SIGBUS; - vmf->page = buffer->pages[vmf->pgoff]; - get_page(vmf->page); - - return 0; + return vmf_insert_pfn(vma, vmf->address, page_to_pfn(buffer->pages[vmf->pgoff])); } static const struct vm_operations_struct dma_heap_vm_ops = { @@ -183,11 +199,11 @@ static int cma_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) { struct cma_heap_buffer *buffer = dmabuf->priv; - dma_resv_assert_held(dmabuf->resv); - if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) return -EINVAL; + vm_flags_set(vma, VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); + vma->vm_ops = &dma_heap_vm_ops; vma->vm_private_data = buffer; @@ -278,8 +294,8 @@ static const struct dma_buf_ops cma_heap_buf_ops = { static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) + u32 fd_flags, + u64 heap_flags) { struct cma_heap *cma_heap = dma_heap_get_drvdata(heap); struct cma_heap_buffer *buffer; @@ -313,13 +329,13 @@ static struct dma_buf *cma_heap_allocate(struct dma_heap *heap, struct page *page = cma_pages; while (nr_clear_pages > 0) { - void *vaddr = kmap_atomic(page); + void *vaddr = kmap_local_page(page); memset(vaddr, 0, PAGE_SIZE); - kunmap_atomic(vaddr); + kunmap_local(vaddr); /* * Avoid wasting time zeroing memory if the process - * has been killed by by SIGKILL + * has been killed by SIGKILL. */ if (fatal_signal_pending(current)) goto free_cma; @@ -370,17 +386,17 @@ static const struct dma_heap_ops cma_heap_ops = { .allocate = cma_heap_allocate, }; -static int __add_cma_heap(struct cma *cma, void *data) +static int __init __add_cma_heap(struct cma *cma, const char *name) { - struct cma_heap *cma_heap; struct dma_heap_export_info exp_info; + struct cma_heap *cma_heap; cma_heap = kzalloc(sizeof(*cma_heap), GFP_KERNEL); if (!cma_heap) return -ENOMEM; cma_heap->cma = cma; - exp_info.name = cma_get_name(cma); + exp_info.name = name; exp_info.ops = &cma_heap_ops; exp_info.priv = cma_heap; @@ -395,16 +411,30 @@ static int __add_cma_heap(struct cma *cma, void *data) return 0; } -static int add_default_cma_heap(void) +static int __init add_cma_heaps(void) { struct cma *default_cma = dev_get_cma_area(NULL); - int ret = 0; + unsigned int i; + int ret; - if (default_cma) - ret = __add_cma_heap(default_cma, NULL); + if (default_cma) { + ret = __add_cma_heap(default_cma, DEFAULT_CMA_NAME); + if (ret) + return ret; + } - return ret; + for (i = 0; i < dma_areas_num; i++) { + struct cma *cma = dma_areas[i]; + + ret = __add_cma_heap(cma, cma_get_name(cma)); + if (ret) { + pr_warn("Failed to add CMA heap %s", cma_get_name(cma)); + continue; + } + + } + + return 0; } -module_init(add_default_cma_heap); +module_init(add_cma_heaps); MODULE_DESCRIPTION("DMA-BUF CMA Heap"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/heaps/system_heap.c b/drivers/dma-buf/heaps/system_heap.c index e8bd10e60998..4c782fe33fd4 100644 --- a/drivers/dma-buf/heaps/system_heap.c +++ b/drivers/dma-buf/heaps/system_heap.c @@ -13,7 +13,6 @@ #include <linux/dma-buf.h> #include <linux/dma-mapping.h> #include <linux/dma-heap.h> -#include <linux/dma-resv.h> #include <linux/err.h> #include <linux/highmem.h> #include <linux/mm.h> @@ -22,8 +21,6 @@ #include <linux/slab.h> #include <linux/vmalloc.h> -static struct dma_heap *sys_heap; - struct system_heap_buffer { struct dma_heap *heap; struct list_head attachments; @@ -36,17 +33,16 @@ struct system_heap_buffer { struct dma_heap_attachment { struct device *dev; - struct sg_table *table; + struct sg_table table; struct list_head list; bool mapped; }; -#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO | __GFP_COMP) -#define MID_ORDER_GFP (LOW_ORDER_GFP | __GFP_NOWARN) +#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO) #define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ | __GFP_NORETRY) & ~__GFP_RECLAIM) \ | __GFP_COMP) -static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP}; +static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; /* * The selection of the orders used for allocation (1MB, 64K, 4K) is designed * to match with the sizes often found in IOMMUs. Using order 4 pages instead @@ -56,29 +52,22 @@ static gfp_t order_flags[] = {HIGH_ORDER_GFP, MID_ORDER_GFP, LOW_ORDER_GFP}; static const unsigned int orders[] = {8, 4, 0}; #define NUM_ORDERS ARRAY_SIZE(orders) -static struct sg_table *dup_sg_table(struct sg_table *table) +static int dup_sg_table(struct sg_table *from, struct sg_table *to) { - struct sg_table *new_table; - int ret, i; struct scatterlist *sg, *new_sg; + int ret, i; - new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); - if (!new_table) - return ERR_PTR(-ENOMEM); - - ret = sg_alloc_table(new_table, table->orig_nents, GFP_KERNEL); - if (ret) { - kfree(new_table); - return ERR_PTR(-ENOMEM); - } + ret = sg_alloc_table(to, from->orig_nents, GFP_KERNEL); + if (ret) + return ret; - new_sg = new_table->sgl; - for_each_sgtable_sg(table, sg, i) { + new_sg = to->sgl; + for_each_sgtable_sg(from, sg, i) { sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); new_sg = sg_next(new_sg); } - return new_table; + return 0; } static int system_heap_attach(struct dma_buf *dmabuf, @@ -86,19 +75,18 @@ static int system_heap_attach(struct dma_buf *dmabuf, { struct system_heap_buffer *buffer = dmabuf->priv; struct dma_heap_attachment *a; - struct sg_table *table; + int ret; a = kzalloc(sizeof(*a), GFP_KERNEL); if (!a) return -ENOMEM; - table = dup_sg_table(&buffer->sg_table); - if (IS_ERR(table)) { + ret = dup_sg_table(&buffer->sg_table, &a->table); + if (ret) { kfree(a); - return -ENOMEM; + return ret; } - a->table = table; a->dev = attachment->dev; INIT_LIST_HEAD(&a->list); a->mapped = false; @@ -122,8 +110,7 @@ static void system_heap_detach(struct dma_buf *dmabuf, list_del(&a->list); mutex_unlock(&buffer->lock); - sg_free_table(a->table); - kfree(a->table); + sg_free_table(&a->table); kfree(a); } @@ -131,7 +118,7 @@ static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attac enum dma_data_direction direction) { struct dma_heap_attachment *a = attachment->priv; - struct sg_table *table = a->table; + struct sg_table *table = &a->table; int ret; ret = dma_map_sgtable(attachment->dev, table, direction, 0); @@ -166,7 +153,7 @@ static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; - dma_sync_sgtable_for_cpu(a->dev, a->table, direction); + dma_sync_sgtable_for_cpu(a->dev, &a->table, direction); } mutex_unlock(&buffer->lock); @@ -187,7 +174,7 @@ static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, list_for_each_entry(a, &buffer->attachments, list) { if (!a->mapped) continue; - dma_sync_sgtable_for_device(a->dev, a->table, direction); + dma_sync_sgtable_for_device(a->dev, &a->table, direction); } mutex_unlock(&buffer->lock); @@ -199,22 +186,35 @@ static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) struct system_heap_buffer *buffer = dmabuf->priv; struct sg_table *table = &buffer->sg_table; unsigned long addr = vma->vm_start; - struct sg_page_iter piter; - int ret; + unsigned long pgoff = vma->vm_pgoff; + struct scatterlist *sg; + int i, ret; - dma_resv_assert_held(dmabuf->resv); + for_each_sgtable_sg(table, sg, i) { + unsigned long n = sg->length >> PAGE_SHIFT; + + if (pgoff < n) + break; + pgoff -= n; + } - for_each_sgtable_page(table, &piter, vma->vm_pgoff) { - struct page *page = sg_page_iter_page(&piter); + for (; sg && addr < vma->vm_end; sg = sg_next(sg)) { + unsigned long n = (sg->length >> PAGE_SHIFT) - pgoff; + struct page *page = sg_page(sg) + pgoff; + unsigned long size = n << PAGE_SHIFT; - ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, - vma->vm_page_prot); + if (addr + size > vma->vm_end) + size = vma->vm_end - addr; + + ret = remap_pfn_range(vma, addr, page_to_pfn(page), + size, vma->vm_page_prot); if (ret) return ret; - addr += PAGE_SIZE; - if (addr >= vma->vm_end) - return 0; + + addr += size; + pgoff = 0; } + return 0; } @@ -337,8 +337,8 @@ static struct page *alloc_largest_available(unsigned long size, static struct dma_buf *system_heap_allocate(struct dma_heap *heap, unsigned long len, - unsigned long fd_flags, - unsigned long heap_flags) + u32 fd_flags, + u64 heap_flags) { struct system_heap_buffer *buffer; DEFINE_DMA_BUF_EXPORT_INFO(exp_info); @@ -425,9 +425,10 @@ static const struct dma_heap_ops system_heap_ops = { .allocate = system_heap_allocate, }; -static int system_heap_create(void) +static int __init system_heap_create(void) { struct dma_heap_export_info exp_info; + struct dma_heap *sys_heap; exp_info.name = "system"; exp_info.ops = &system_heap_ops; @@ -440,4 +441,3 @@ static int system_heap_create(void) return 0; } module_init(system_heap_create); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma-buf/st-dma-fence-chain.c b/drivers/dma-buf/st-dma-fence-chain.c index c0979c8049b5..ed4b323886e4 100644 --- a/drivers/dma-buf/st-dma-fence-chain.c +++ b/drivers/dma-buf/st-dma-fence-chain.c @@ -84,11 +84,11 @@ static int sanitycheck(void *arg) return -ENOMEM; chain = mock_chain(NULL, f, 1); - if (!chain) + if (chain) + dma_fence_enable_sw_signaling(chain); + else err = -ENOMEM; - dma_fence_enable_sw_signaling(chain); - dma_fence_signal(f); dma_fence_put(f); @@ -476,10 +476,9 @@ static int find_race(void *arg) for (i = 0; i < ncpus; i++) { int ret; - ret = kthread_stop(threads[i]); + ret = kthread_stop_put(threads[i]); if (ret && !err) err = ret; - put_task_struct(threads[i]); } kfree(threads); @@ -591,8 +590,7 @@ static int wait_forward(void *arg) for (i = 0; i < fc.chain_length; i++) dma_fence_signal(fc.fences[i]); - err = kthread_stop(tsk); - put_task_struct(tsk); + err = kthread_stop_put(tsk); err: fence_chains_fini(&fc); @@ -621,8 +619,7 @@ static int wait_backward(void *arg) for (i = fc.chain_length; i--; ) dma_fence_signal(fc.fences[i]); - err = kthread_stop(tsk); - put_task_struct(tsk); + err = kthread_stop_put(tsk); err: fence_chains_fini(&fc); @@ -669,8 +666,7 @@ static int wait_random(void *arg) for (i = 0; i < fc.chain_length; i++) dma_fence_signal(fc.fences[i]); - err = kthread_stop(tsk); - put_task_struct(tsk); + err = kthread_stop_put(tsk); err: fence_chains_fini(&fc); diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c index f0cee984b6c7..a3be888ae2e8 100644 --- a/drivers/dma-buf/st-dma-fence-unwrap.c +++ b/drivers/dma-buf/st-dma-fence-unwrap.c @@ -28,7 +28,7 @@ static const struct dma_fence_ops mock_ops = { .get_timeline_name = mock_name, }; -static struct dma_fence *mock_fence(void) +static struct dma_fence *__mock_fence(u64 context, u64 seqno) { struct mock_fence *f; @@ -37,12 +37,16 @@ static struct dma_fence *mock_fence(void) return NULL; spin_lock_init(&f->lock); - dma_fence_init(&f->base, &mock_ops, &f->lock, - dma_fence_context_alloc(1), 1); + dma_fence_init(&f->base, &mock_ops, &f->lock, context, seqno); return &f->base; } +static struct dma_fence *mock_fence(void) +{ + return __mock_fence(dma_fence_context_alloc(1), 1); +} + static struct dma_fence *mock_array(unsigned int num_fences, ...) { struct dma_fence_array *array; @@ -304,6 +308,177 @@ error_put_f1: return err; } +static int unwrap_merge_duplicate(void *arg) +{ + struct dma_fence *fence, *f1, *f2; + struct dma_fence_unwrap iter; + int err = 0; + + f1 = mock_fence(); + if (!f1) + return -ENOMEM; + + dma_fence_enable_sw_signaling(f1); + + f2 = dma_fence_unwrap_merge(f1, f1); + if (!f2) { + err = -ENOMEM; + goto error_put_f1; + } + + dma_fence_unwrap_for_each(fence, &iter, f2) { + if (fence == f1) { + dma_fence_put(f1); + f1 = NULL; + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f1) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(f2); +error_put_f1: + dma_fence_put(f1); + return err; +} + +static int unwrap_merge_seqno(void *arg) +{ + struct dma_fence *fence, *f1, *f2, *f3, *f4; + struct dma_fence_unwrap iter; + int err = 0; + u64 ctx[2]; + + ctx[0] = dma_fence_context_alloc(1); + ctx[1] = dma_fence_context_alloc(1); + + f1 = __mock_fence(ctx[1], 1); + if (!f1) + return -ENOMEM; + + dma_fence_enable_sw_signaling(f1); + + f2 = __mock_fence(ctx[1], 2); + if (!f2) { + err = -ENOMEM; + goto error_put_f1; + } + + dma_fence_enable_sw_signaling(f2); + + f3 = __mock_fence(ctx[0], 1); + if (!f3) { + err = -ENOMEM; + goto error_put_f2; + } + + dma_fence_enable_sw_signaling(f3); + + f4 = dma_fence_unwrap_merge(f1, f2, f3); + if (!f4) { + err = -ENOMEM; + goto error_put_f3; + } + + dma_fence_unwrap_for_each(fence, &iter, f4) { + if (fence == f3 && f2) { + dma_fence_put(f3); + f3 = NULL; + } else if (fence == f2 && !f3) { + dma_fence_put(f2); + f2 = NULL; + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f2 || f3) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(f4); +error_put_f3: + dma_fence_put(f3); +error_put_f2: + dma_fence_put(f2); +error_put_f1: + dma_fence_put(f1); + return err; +} + +static int unwrap_merge_order(void *arg) +{ + struct dma_fence *fence, *f1, *f2, *a1, *a2, *c1, *c2; + struct dma_fence_unwrap iter; + int err = 0; + + f1 = mock_fence(); + if (!f1) + return -ENOMEM; + + dma_fence_enable_sw_signaling(f1); + + f2 = mock_fence(); + if (!f2) { + dma_fence_put(f1); + return -ENOMEM; + } + + dma_fence_enable_sw_signaling(f2); + + a1 = mock_array(2, f1, f2); + if (!a1) + return -ENOMEM; + + c1 = mock_chain(NULL, dma_fence_get(f1)); + if (!c1) + goto error_put_a1; + + c2 = mock_chain(c1, dma_fence_get(f2)); + if (!c2) + goto error_put_a1; + + /* + * The fences in the chain are the same as in a1 but in oposite order, + * the dma_fence_merge() function should be able to handle that. + */ + a2 = dma_fence_unwrap_merge(a1, c2); + + dma_fence_unwrap_for_each(fence, &iter, a2) { + if (fence == f1) { + f1 = NULL; + if (!f2) + pr_err("Unexpected order!\n"); + } else if (fence == f2) { + f2 = NULL; + if (f1) + pr_err("Unexpected order!\n"); + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f1 || f2) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(a2); + return err; + +error_put_a1: + dma_fence_put(a1); + return -ENOMEM; +} + static int unwrap_merge_complex(void *arg) { struct dma_fence *fence, *f1, *f2, *f3, *f4, *f5; @@ -327,7 +502,7 @@ static int unwrap_merge_complex(void *arg) goto error_put_f2; /* The resulting array has the fences in reverse */ - f4 = dma_fence_unwrap_merge(f2, f1); + f4 = mock_array(2, dma_fence_get(f2), dma_fence_get(f1)); if (!f4) goto error_put_f3; @@ -367,6 +542,87 @@ error_put_f1: return err; } +static int unwrap_merge_complex_seqno(void *arg) +{ + struct dma_fence *fence, *f1, *f2, *f3, *f4, *f5, *f6, *f7; + struct dma_fence_unwrap iter; + int err = -ENOMEM; + u64 ctx[2]; + + ctx[0] = dma_fence_context_alloc(1); + ctx[1] = dma_fence_context_alloc(1); + + f1 = __mock_fence(ctx[0], 2); + if (!f1) + return -ENOMEM; + + dma_fence_enable_sw_signaling(f1); + + f2 = __mock_fence(ctx[1], 1); + if (!f2) + goto error_put_f1; + + dma_fence_enable_sw_signaling(f2); + + f3 = __mock_fence(ctx[0], 1); + if (!f3) + goto error_put_f2; + + dma_fence_enable_sw_signaling(f3); + + f4 = __mock_fence(ctx[1], 2); + if (!f4) + goto error_put_f3; + + dma_fence_enable_sw_signaling(f4); + + f5 = mock_array(2, dma_fence_get(f1), dma_fence_get(f2)); + if (!f5) + goto error_put_f4; + + f6 = mock_array(2, dma_fence_get(f3), dma_fence_get(f4)); + if (!f6) + goto error_put_f5; + + f7 = dma_fence_unwrap_merge(f5, f6); + if (!f7) + goto error_put_f6; + + err = 0; + dma_fence_unwrap_for_each(fence, &iter, f7) { + if (fence == f1 && f4) { + dma_fence_put(f1); + f1 = NULL; + } else if (fence == f4 && !f1) { + dma_fence_put(f4); + f4 = NULL; + } else { + pr_err("Unexpected fence!\n"); + err = -EINVAL; + } + } + + if (f1 || f4) { + pr_err("Not all fences seen!\n"); + err = -EINVAL; + } + + dma_fence_put(f7); +error_put_f6: + dma_fence_put(f6); +error_put_f5: + dma_fence_put(f5); +error_put_f4: + dma_fence_put(f4); +error_put_f3: + dma_fence_put(f3); +error_put_f2: + dma_fence_put(f2); +error_put_f1: + dma_fence_put(f1); + return err; +} + int dma_fence_unwrap(void) { static const struct subtest tests[] = { @@ -375,7 +631,11 @@ int dma_fence_unwrap(void) SUBTEST(unwrap_chain), SUBTEST(unwrap_chain_array), SUBTEST(unwrap_merge), + SUBTEST(unwrap_merge_duplicate), + SUBTEST(unwrap_merge_seqno), + SUBTEST(unwrap_merge_order), SUBTEST(unwrap_merge_complex), + SUBTEST(unwrap_merge_complex_seqno), }; return subtests(tests, NULL); diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c index fb6e0a6ae2c9..27a36045410b 100644 --- a/drivers/dma-buf/st-dma-fence.c +++ b/drivers/dma-buf/st-dma-fence.c @@ -375,7 +375,7 @@ struct wait_timer { static void wait_timer(struct timer_list *timer) { - struct wait_timer *wt = from_timer(wt, timer, timer); + struct wait_timer *wt = timer_container_of(wt, timer, timer); dma_fence_signal(wt->f); } @@ -402,7 +402,7 @@ static int test_wait_timeout(void *arg) if (dma_fence_wait_timeout(wt.f, false, 2) == -ETIME) { if (timer_pending(&wt.timer)) { - pr_notice("Timer did not fire within the jiffie!\n"); + pr_notice("Timer did not fire within the jiffy!\n"); err = 0; /* not our fault! */ } else { pr_err("Wait reported incomplete after timeout\n"); @@ -412,8 +412,8 @@ static int test_wait_timeout(void *arg) err = 0; err_free: - del_timer_sync(&wt.timer); - destroy_timer_on_stack(&wt.timer); + timer_delete_sync(&wt.timer); + timer_destroy_on_stack(&wt.timer); dma_fence_signal(wt.f); dma_fence_put(wt.f); return err; @@ -540,6 +540,12 @@ static int race_signal_callback(void *arg) t[i].before = pass; t[i].task = kthread_run(thread_signal_callback, &t[i], "dma-fence:%d", i); + if (IS_ERR(t[i].task)) { + ret = PTR_ERR(t[i].task); + while (--i >= 0) + kthread_stop_put(t[i].task); + return ret; + } get_task_struct(t[i].task); } @@ -548,11 +554,9 @@ static int race_signal_callback(void *arg) for (i = 0; i < ARRAY_SIZE(t); i++) { int err; - err = kthread_stop(t[i].task); + err = kthread_stop_put(t[i].task); if (err && !ret) ret = err; - - put_task_struct(t[i].task); } } diff --git a/drivers/dma-buf/sw_sync.c b/drivers/dma-buf/sw_sync.c index 348b3a9170fa..6f09d13be6b6 100644 --- a/drivers/dma-buf/sw_sync.c +++ b/drivers/dma-buf/sw_sync.c @@ -8,6 +8,7 @@ #include <linux/file.h> #include <linux/fs.h> #include <linux/uaccess.h> +#include <linux/panic.h> #include <linux/slab.h> #include <linux/sync_file.h> @@ -52,12 +53,33 @@ struct sw_sync_create_fence_data { __s32 fence; /* fd of new fence */ }; +/** + * struct sw_sync_get_deadline - get the deadline hint of a sw_sync fence + * @deadline_ns: absolute time of the deadline + * @pad: must be zero + * @fence_fd: the sw_sync fence fd (in) + * + * Return the earliest deadline set on the fence. The timebase for the + * deadline is CLOCK_MONOTONIC (same as vblank). If there is no deadline + * set on the fence, this ioctl will return -ENOENT. + */ +struct sw_sync_get_deadline { + __u64 deadline_ns; + __u32 pad; + __s32 fence_fd; +}; + #define SW_SYNC_IOC_MAGIC 'W' #define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ struct sw_sync_create_fence_data) #define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) +#define SW_SYNC_GET_DEADLINE _IOWR(SW_SYNC_IOC_MAGIC, 2, \ + struct sw_sync_get_deadline) + + +#define SW_SYNC_HAS_DEADLINE_BIT DMA_FENCE_FLAG_USER_BITS static const struct dma_fence_ops timeline_fence_ops; @@ -85,7 +107,7 @@ static struct sync_timeline *sync_timeline_create(const char *name) kref_init(&obj->kref); obj->context = dma_fence_context_alloc(1); - strlcpy(obj->name, name, sizeof(obj->name)); + strscpy(obj->name, name, sizeof(obj->name)); obj->pt_tree = RB_ROOT; INIT_LIST_HEAD(&obj->pt_list); @@ -149,36 +171,31 @@ static bool timeline_fence_signaled(struct dma_fence *fence) { struct sync_timeline *parent = dma_fence_parent(fence); - return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops); -} - -static bool timeline_fence_enable_signaling(struct dma_fence *fence) -{ - return true; -} - -static void timeline_fence_value_str(struct dma_fence *fence, - char *str, int size) -{ - snprintf(str, size, "%lld", fence->seqno); + return !__dma_fence_is_later(fence, fence->seqno, parent->value); } -static void timeline_fence_timeline_value_str(struct dma_fence *fence, - char *str, int size) +static void timeline_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) { - struct sync_timeline *parent = dma_fence_parent(fence); + struct sync_pt *pt = dma_fence_to_sync_pt(fence); + unsigned long flags; - snprintf(str, size, "%d", parent->value); + spin_lock_irqsave(fence->lock, flags); + if (test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { + if (ktime_before(deadline, pt->deadline)) + pt->deadline = deadline; + } else { + pt->deadline = deadline; + __set_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags); + } + spin_unlock_irqrestore(fence->lock, flags); } static const struct dma_fence_ops timeline_fence_ops = { .get_driver_name = timeline_fence_get_driver_name, .get_timeline_name = timeline_fence_get_timeline_name, - .enable_signaling = timeline_fence_enable_signaling, .signaled = timeline_fence_signaled, .release = timeline_fence_release, - .fence_value_str = timeline_fence_value_str, - .timeline_value_str = timeline_fence_timeline_value_str, + .set_deadline = timeline_fence_set_deadline, }; /** @@ -191,6 +208,7 @@ static const struct dma_fence_ops timeline_fence_ops = { */ static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) { + LIST_HEAD(signalled); struct sync_pt *pt, *next; trace_sync_timeline(obj); @@ -203,21 +221,20 @@ static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) if (!timeline_fence_signaled(&pt->base)) break; - list_del_init(&pt->link); + dma_fence_get(&pt->base); + + list_move_tail(&pt->link, &signalled); rb_erase(&pt->node, &obj->pt_tree); - /* - * A signal callback may release the last reference to this - * fence, causing it to be freed. That operation has to be - * last to avoid a use after free inside this loop, and must - * be after we remove the fence from the timeline in order to - * prevent deadlocking on timeline->lock inside - * timeline_fence_release(). - */ dma_fence_signal_locked(&pt->base); } spin_unlock_irq(&obj->lock); + + list_for_each_entry_safe(pt, next, &signalled, link) { + list_del_init(&pt->link); + dma_fence_put(&pt->base); + } } /** @@ -333,6 +350,9 @@ static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, struct sync_file *sync_file; struct sw_sync_create_fence_data data; + /* SW sync fence are inherently unsafe and can deadlock the kernel */ + add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); + if (fd < 0) return fd; @@ -387,6 +407,56 @@ static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg) return 0; } +static int sw_sync_ioctl_get_deadline(struct sync_timeline *obj, unsigned long arg) +{ + struct sw_sync_get_deadline data; + struct dma_fence *fence; + unsigned long flags; + struct sync_pt *pt; + int ret = 0; + + if (copy_from_user(&data, (void __user *)arg, sizeof(data))) + return -EFAULT; + + if (data.deadline_ns || data.pad) + return -EINVAL; + + fence = sync_file_get_fence(data.fence_fd); + if (!fence) + return -EINVAL; + + pt = dma_fence_to_sync_pt(fence); + if (!pt) { + ret = -EINVAL; + goto put_fence; + } + + spin_lock_irqsave(fence->lock, flags); + if (!test_bit(SW_SYNC_HAS_DEADLINE_BIT, &fence->flags)) { + ret = -ENOENT; + goto unlock; + } + data.deadline_ns = ktime_to_ns(pt->deadline); + spin_unlock_irqrestore(fence->lock, flags); + + dma_fence_put(fence); + + if (ret) + return ret; + + if (copy_to_user((void __user *)arg, &data, sizeof(data))) + return -EFAULT; + + return 0; + +unlock: + spin_unlock_irqrestore(fence->lock, flags); +put_fence: + dma_fence_put(fence); + + return ret; +} + static long sw_sync_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -399,6 +469,9 @@ static long sw_sync_ioctl(struct file *file, unsigned int cmd, case SW_SYNC_IOC_INC: return sw_sync_ioctl_inc(obj, arg); + case SW_SYNC_GET_DEADLINE: + return sw_sync_ioctl_get_deadline(obj, arg); + default: return -ENOTTY; } diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c index 101394f16930..9e5d662cd4e8 100644 --- a/drivers/dma-buf/sync_debug.c +++ b/drivers/dma-buf/sync_debug.c @@ -12,8 +12,6 @@ static struct dentry *dbgfs; static LIST_HEAD(sync_timeline_list_head); static DEFINE_SPINLOCK(sync_timeline_list_lock); -static LIST_HEAD(sync_file_list_head); -static DEFINE_SPINLOCK(sync_file_list_lock); void sync_timeline_debug_add(struct sync_timeline *obj) { @@ -33,24 +31,6 @@ void sync_timeline_debug_remove(struct sync_timeline *obj) spin_unlock_irqrestore(&sync_timeline_list_lock, flags); } -void sync_file_debug_add(struct sync_file *sync_file) -{ - unsigned long flags; - - spin_lock_irqsave(&sync_file_list_lock, flags); - list_add_tail(&sync_file->sync_file_list, &sync_file_list_head); - spin_unlock_irqrestore(&sync_file_list_lock, flags); -} - -void sync_file_debug_remove(struct sync_file *sync_file) -{ - unsigned long flags; - - spin_lock_irqsave(&sync_file_list_lock, flags); - list_del(&sync_file->sync_file_list); - spin_unlock_irqrestore(&sync_file_list_lock, flags); -} - static const char *sync_status_str(int status) { if (status < 0) @@ -79,28 +59,11 @@ static void sync_print_fence(struct seq_file *s, struct timespec64 ts64 = ktime_to_timespec64(fence->timestamp); - seq_printf(s, "@%lld.%09ld", (s64)ts64.tv_sec, ts64.tv_nsec); - } - - if (fence->ops->timeline_value_str && - fence->ops->fence_value_str) { - char value[64]; - bool success; - - fence->ops->fence_value_str(fence, value, sizeof(value)); - success = strlen(value); - - if (success) { - seq_printf(s, ": %s", value); - - fence->ops->timeline_value_str(fence, value, - sizeof(value)); - - if (strlen(value)) - seq_printf(s, " / %s", value); - } + seq_printf(s, "@%ptSp", &ts64); } + seq_printf(s, ": %lld", fence->seqno); + seq_printf(s, " / %d", parent->value); seq_putc(s, '\n'); } @@ -110,32 +73,12 @@ static void sync_print_obj(struct seq_file *s, struct sync_timeline *obj) seq_printf(s, "%s: %d\n", obj->name, obj->value); - spin_lock_irq(&obj->lock); + spin_lock(&obj->lock); /* Caller already disabled IRQ. */ list_for_each(pos, &obj->pt_list) { struct sync_pt *pt = container_of(pos, struct sync_pt, link); sync_print_fence(s, &pt->base, false); } - spin_unlock_irq(&obj->lock); -} - -static void sync_print_sync_file(struct seq_file *s, - struct sync_file *sync_file) -{ - char buf[128]; - int i; - - seq_printf(s, "[%p] %s: %s\n", sync_file, - sync_file_get_name(sync_file, buf, sizeof(buf)), - sync_status_str(dma_fence_get_status(sync_file->fence))); - - if (dma_fence_is_array(sync_file->fence)) { - struct dma_fence_array *array = to_dma_fence_array(sync_file->fence); - - for (i = 0; i < array->num_fences; ++i) - sync_print_fence(s, array->fences[i], true); - } else { - sync_print_fence(s, sync_file->fence, true); - } + spin_unlock(&obj->lock); } static int sync_info_debugfs_show(struct seq_file *s, void *unused) @@ -157,15 +100,6 @@ static int sync_info_debugfs_show(struct seq_file *s, void *unused) seq_puts(s, "fences:\n--------------\n"); - spin_lock_irq(&sync_file_list_lock); - list_for_each(pos, &sync_file_list_head) { - struct sync_file *sync_file = - container_of(pos, struct sync_file, sync_file_list); - - sync_print_sync_file(s, sync_file); - seq_putc(s, '\n'); - } - spin_unlock_irq(&sync_file_list_lock); return 0; } diff --git a/drivers/dma-buf/sync_debug.h b/drivers/dma-buf/sync_debug.h index 6176e52ba2d7..02af347293d0 100644 --- a/drivers/dma-buf/sync_debug.h +++ b/drivers/dma-buf/sync_debug.h @@ -55,18 +55,18 @@ static inline struct sync_timeline *dma_fence_parent(struct dma_fence *fence) * @base: base fence object * @link: link on the sync timeline's list * @node: node in the sync timeline's tree + * @deadline: the earliest fence deadline hint */ struct sync_pt { struct dma_fence base; struct list_head link; struct rb_node node; + ktime_t deadline; }; extern const struct file_operations sw_sync_debugfs_fops; void sync_timeline_debug_add(struct sync_timeline *obj); void sync_timeline_debug_remove(struct sync_timeline *obj); -void sync_file_debug_add(struct sync_file *fence); -void sync_file_debug_remove(struct sync_file *fence); #endif /* _LINUX_SYNC_H */ diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c index af57799c86ce..747e377fb954 100644 --- a/drivers/dma-buf/sync_file.c +++ b/drivers/dma-buf/sync_file.c @@ -135,12 +135,18 @@ char *sync_file_get_name(struct sync_file *sync_file, char *buf, int len) strscpy(buf, sync_file->user_name, len); } else { struct dma_fence *fence = sync_file->fence; + const char __rcu *timeline; + const char __rcu *driver; + rcu_read_lock(); + driver = dma_fence_driver_name(fence); + timeline = dma_fence_timeline_name(fence); snprintf(buf, len, "%s-%s%llu-%lld", - fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), + rcu_dereference(driver), + rcu_dereference(timeline), fence->context, fence->seqno); + rcu_read_unlock(); } return buf; @@ -262,19 +268,26 @@ err_put_fd: static int sync_fill_fence_info(struct dma_fence *fence, struct sync_fence_info *info) { - strscpy(info->obj_name, fence->ops->get_timeline_name(fence), + const char __rcu *timeline; + const char __rcu *driver; + + rcu_read_lock(); + + driver = dma_fence_driver_name(fence); + timeline = dma_fence_timeline_name(fence); + + strscpy(info->obj_name, rcu_dereference(timeline), sizeof(info->obj_name)); - strscpy(info->driver_name, fence->ops->get_driver_name(fence), + strscpy(info->driver_name, rcu_dereference(driver), sizeof(info->driver_name)); info->status = dma_fence_get_status(fence); - while (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags) && - !test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags)) - cpu_relax(); info->timestamp_ns = - test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags) ? - ktime_to_ns(fence->timestamp) : - ktime_set(0, 0); + dma_fence_is_signaled(fence) ? + ktime_to_ns(dma_fence_timestamp(fence)) : + ktime_set(0, 0); + + rcu_read_unlock(); return info->status; } @@ -350,6 +363,22 @@ out: return ret; } +static int sync_file_ioctl_set_deadline(struct sync_file *sync_file, + unsigned long arg) +{ + struct sync_set_deadline ts; + + if (copy_from_user(&ts, (void __user *)arg, sizeof(ts))) + return -EFAULT; + + if (ts.pad) + return -EINVAL; + + dma_fence_set_deadline(sync_file->fence, ns_to_ktime(ts.deadline_ns)); + + return 0; +} + static long sync_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -362,6 +391,9 @@ static long sync_file_ioctl(struct file *file, unsigned int cmd, case SYNC_IOC_FILE_INFO: return sync_file_ioctl_fence_info(sync_file, arg); + case SYNC_IOC_SET_DEADLINE: + return sync_file_ioctl_set_deadline(sync_file, arg); + default: return -ENOTTY; } diff --git a/drivers/dma-buf/sync_trace.h b/drivers/dma-buf/sync_trace.h index 06e468a218ff..d71dcf954b8d 100644 --- a/drivers/dma-buf/sync_trace.h +++ b/drivers/dma-buf/sync_trace.h @@ -20,7 +20,7 @@ TRACE_EVENT(sync_timeline, ), TP_fast_assign( - __assign_str(name, timeline->name); + __assign_str(name); __entry->value = timeline->value; ), diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c index 740d6e426ee9..40399c26e6be 100644 --- a/drivers/dma-buf/udmabuf.c +++ b/drivers/dma-buf/udmabuf.c @@ -10,9 +10,9 @@ #include <linux/miscdevice.h> #include <linux/module.h> #include <linux/shmem_fs.h> +#include <linux/hugetlb.h> #include <linux/slab.h> #include <linux/udmabuf.h> -#include <linux/hugetlb.h> #include <linux/vmalloc.h> #include <linux/iosys-map.h> @@ -26,9 +26,22 @@ MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is struct udmabuf { pgoff_t pagecount; - struct page **pages; + struct folio **folios; + + /** + * Unlike folios, pinned_folios is only used for unpin. + * So, nr_pinned is not the same to pagecount, the pinned_folios + * only set each folio which already pinned when udmabuf_create. + * Note that, since a folio may be pinned multiple times, each folio + * can be added to pinned_folios multiple times, depending on how many + * times the folio has been pinned when create. + */ + pgoff_t nr_pinned; + struct folio **pinned_folios; + struct sg_table *sg; struct miscdevice *device; + pgoff_t *offsets; }; static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) @@ -36,12 +49,44 @@ static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) struct vm_area_struct *vma = vmf->vma; struct udmabuf *ubuf = vma->vm_private_data; pgoff_t pgoff = vmf->pgoff; + unsigned long addr, pfn; + vm_fault_t ret; if (pgoff >= ubuf->pagecount) return VM_FAULT_SIGBUS; - vmf->page = ubuf->pages[pgoff]; - get_page(vmf->page); - return 0; + + pfn = folio_pfn(ubuf->folios[pgoff]); + pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; + + ret = vmf_insert_pfn(vma, vmf->address, pfn); + if (ret & VM_FAULT_ERROR) + return ret; + + /* pre fault */ + pgoff = vma->vm_pgoff; + addr = vma->vm_start; + + for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { + if (addr == vmf->address) + continue; + + if (WARN_ON(pgoff >= ubuf->pagecount)) + break; + + pfn = folio_pfn(ubuf->folios[pgoff]); + pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; + + /** + * If the below vmf_insert_pfn() fails, we do not return an + * error here during this pre-fault step. However, an error + * will be returned if the failure occurs when the addr is + * truly accessed. + */ + if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) + break; + } + + return ret; } static const struct vm_operations_struct udmabuf_vm_ops = { @@ -52,24 +97,34 @@ static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) { struct udmabuf *ubuf = buf->priv; - dma_resv_assert_held(buf->resv); - if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) return -EINVAL; vma->vm_ops = &udmabuf_vm_ops; vma->vm_private_data = ubuf; + vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); return 0; } static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) { struct udmabuf *ubuf = buf->priv; + struct page **pages; void *vaddr; + pgoff_t pg; dma_resv_assert_held(buf->resv); - vaddr = vm_map_ram(ubuf->pages, ubuf->pagecount, -1); + pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + for (pg = 0; pg < ubuf->pagecount; pg++) + pages[pg] = folio_page(ubuf->folios[pg], + ubuf->offsets[pg] >> PAGE_SHIFT); + + vaddr = vm_map_ram(pages, ubuf->pagecount, -1); + kvfree(pages); if (!vaddr) return -EINVAL; @@ -91,23 +146,30 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, { struct udmabuf *ubuf = buf->priv; struct sg_table *sg; + struct scatterlist *sgl; + unsigned int i = 0; int ret; sg = kzalloc(sizeof(*sg), GFP_KERNEL); if (!sg) return ERR_PTR(-ENOMEM); - ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount, - 0, ubuf->pagecount << PAGE_SHIFT, - GFP_KERNEL); + + ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL); if (ret < 0) - goto err; + goto err_alloc; + + for_each_sg(sg->sgl, sgl, ubuf->pagecount, i) + sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE, + ubuf->offsets[i]); + ret = dma_map_sgtable(dev, sg, direction, 0); if (ret < 0) - goto err; + goto err_map; return sg; -err: +err_map: sg_free_table(sg); +err_alloc: kfree(sg); return ERR_PTR(ret); } @@ -133,18 +195,51 @@ static void unmap_udmabuf(struct dma_buf_attachment *at, return put_sg_table(at->dev, sg, direction); } +static void unpin_all_folios(struct udmabuf *ubuf) +{ + pgoff_t i; + + for (i = 0; i < ubuf->nr_pinned; ++i) + unpin_folio(ubuf->pinned_folios[i]); + + kvfree(ubuf->pinned_folios); +} + +static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt) +{ + ubuf->folios = kvmalloc_array(pgcnt, sizeof(*ubuf->folios), GFP_KERNEL); + if (!ubuf->folios) + return -ENOMEM; + + ubuf->offsets = kvcalloc(pgcnt, sizeof(*ubuf->offsets), GFP_KERNEL); + if (!ubuf->offsets) + return -ENOMEM; + + ubuf->pinned_folios = kvmalloc_array(pgcnt, + sizeof(*ubuf->pinned_folios), + GFP_KERNEL); + if (!ubuf->pinned_folios) + return -ENOMEM; + + return 0; +} + +static __always_inline void deinit_udmabuf(struct udmabuf *ubuf) +{ + unpin_all_folios(ubuf); + kvfree(ubuf->offsets); + kvfree(ubuf->folios); +} + static void release_udmabuf(struct dma_buf *buf) { struct udmabuf *ubuf = buf->priv; struct device *dev = ubuf->device->this_device; - pgoff_t pg; if (ubuf->sg) put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); - for (pg = 0; pg < ubuf->pagecount; pg++) - put_page(ubuf->pages[pg]); - kfree(ubuf->pages); + deinit_udmabuf(ubuf); kfree(ubuf); } @@ -162,8 +257,7 @@ static int begin_cpu_udmabuf(struct dma_buf *buf, ubuf->sg = NULL; } } else { - dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents, - direction); + dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); } return ret; @@ -178,12 +272,11 @@ static int end_cpu_udmabuf(struct dma_buf *buf, if (!ubuf->sg) return -EINVAL; - dma_sync_sg_for_device(dev, ubuf->sg->sgl, ubuf->sg->nents, direction); + dma_sync_sgtable_for_device(dev, ubuf->sg, direction); return 0; } static const struct dma_buf_ops udmabuf_ops = { - .cache_sgt_mapping = true, .map_dma_buf = map_udmabuf, .unmap_dma_buf = unmap_udmabuf, .release = release_udmabuf, @@ -195,135 +288,182 @@ static const struct dma_buf_ops udmabuf_ops = { }; #define SEALS_WANTED (F_SEAL_SHRINK) -#define SEALS_DENIED (F_SEAL_WRITE) +#define SEALS_DENIED (F_SEAL_WRITE|F_SEAL_FUTURE_WRITE) + +static int check_memfd_seals(struct file *memfd) +{ + int seals; + + if (!shmem_file(memfd) && !is_file_hugepages(memfd)) + return -EBADFD; + + seals = memfd_fcntl(memfd, F_GET_SEALS, 0); + if (seals == -EINVAL) + return -EBADFD; + + if ((seals & SEALS_WANTED) != SEALS_WANTED || + (seals & SEALS_DENIED) != 0) + return -EINVAL; + + return 0; +} + +static struct dma_buf *export_udmabuf(struct udmabuf *ubuf, + struct miscdevice *device) +{ + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + + ubuf->device = device; + exp_info.ops = &udmabuf_ops; + exp_info.size = ubuf->pagecount << PAGE_SHIFT; + exp_info.priv = ubuf; + exp_info.flags = O_RDWR; + + return dma_buf_export(&exp_info); +} + +static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd, + loff_t start, loff_t size, struct folio **folios) +{ + pgoff_t nr_pinned = ubuf->nr_pinned; + pgoff_t upgcnt = ubuf->pagecount; + u32 cur_folio, cur_pgcnt; + pgoff_t pgoff, pgcnt; + long nr_folios; + loff_t end; + + pgcnt = size >> PAGE_SHIFT; + end = start + (pgcnt << PAGE_SHIFT) - 1; + nr_folios = memfd_pin_folios(memfd, start, end, folios, pgcnt, &pgoff); + if (nr_folios <= 0) + return nr_folios ? nr_folios : -EINVAL; + + cur_pgcnt = 0; + for (cur_folio = 0; cur_folio < nr_folios; ++cur_folio) { + pgoff_t subpgoff = pgoff; + size_t fsize = folio_size(folios[cur_folio]); + + ubuf->pinned_folios[nr_pinned++] = folios[cur_folio]; + + for (; subpgoff < fsize; subpgoff += PAGE_SIZE) { + ubuf->folios[upgcnt] = folios[cur_folio]; + ubuf->offsets[upgcnt] = subpgoff; + ++upgcnt; + + if (++cur_pgcnt >= pgcnt) + goto end; + } + + /** + * In a given range, only the first subpage of the first folio + * has an offset, that is returned by memfd_pin_folios(). + * The first subpages of other folios (in the range) have an + * offset of 0. + */ + pgoff = 0; + } +end: + ubuf->pagecount = upgcnt; + ubuf->nr_pinned = nr_pinned; + return 0; +} static long udmabuf_create(struct miscdevice *device, struct udmabuf_create_list *head, struct udmabuf_create_item *list) { - DEFINE_DMA_BUF_EXPORT_INFO(exp_info); - struct file *memfd = NULL; - struct address_space *mapping = NULL; + unsigned long max_nr_folios = 0; + struct folio **folios = NULL; + pgoff_t pgcnt = 0, pglimit; struct udmabuf *ubuf; - struct dma_buf *buf; - pgoff_t pgoff, pgcnt, pgidx, pgbuf = 0, pglimit; - struct page *page, *hpage = NULL; - pgoff_t subpgoff, maxsubpgs; - struct hstate *hpstate; - int seals, ret = -EINVAL; + struct dma_buf *dmabuf; + long ret = -EINVAL; u32 i, flags; ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); if (!ubuf) return -ENOMEM; - pglimit = (size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; + pglimit = ((u64)size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; for (i = 0; i < head->count; i++) { - if (!IS_ALIGNED(list[i].offset, PAGE_SIZE)) - goto err; - if (!IS_ALIGNED(list[i].size, PAGE_SIZE)) - goto err; - ubuf->pagecount += list[i].size >> PAGE_SHIFT; - if (ubuf->pagecount > pglimit) - goto err; + pgoff_t subpgcnt; + + if (!PAGE_ALIGNED(list[i].offset)) + goto err_noinit; + if (!PAGE_ALIGNED(list[i].size)) + goto err_noinit; + + subpgcnt = list[i].size >> PAGE_SHIFT; + pgcnt += subpgcnt; + if (pgcnt > pglimit) + goto err_noinit; + + max_nr_folios = max_t(unsigned long, subpgcnt, max_nr_folios); } - if (!ubuf->pagecount) + if (!pgcnt) + goto err_noinit; + + ret = init_udmabuf(ubuf, pgcnt); + if (ret) goto err; - ubuf->pages = kmalloc_array(ubuf->pagecount, sizeof(*ubuf->pages), - GFP_KERNEL); - if (!ubuf->pages) { + folios = kvmalloc_array(max_nr_folios, sizeof(*folios), GFP_KERNEL); + if (!folios) { ret = -ENOMEM; goto err; } - pgbuf = 0; for (i = 0; i < head->count; i++) { - ret = -EBADFD; - memfd = fget(list[i].memfd); - if (!memfd) - goto err; - mapping = memfd->f_mapping; - if (!shmem_mapping(mapping) && !is_file_hugepages(memfd)) - goto err; - seals = memfd_fcntl(memfd, F_GET_SEALS, 0); - if (seals == -EINVAL) - goto err; - ret = -EINVAL; - if ((seals & SEALS_WANTED) != SEALS_WANTED || - (seals & SEALS_DENIED) != 0) + struct file *memfd = fget(list[i].memfd); + + if (!memfd) { + ret = -EBADFD; goto err; - pgoff = list[i].offset >> PAGE_SHIFT; - pgcnt = list[i].size >> PAGE_SHIFT; - if (is_file_hugepages(memfd)) { - hpstate = hstate_file(memfd); - pgoff = list[i].offset >> huge_page_shift(hpstate); - subpgoff = (list[i].offset & - ~huge_page_mask(hpstate)) >> PAGE_SHIFT; - maxsubpgs = huge_page_size(hpstate) >> PAGE_SHIFT; - } - for (pgidx = 0; pgidx < pgcnt; pgidx++) { - if (is_file_hugepages(memfd)) { - if (!hpage) { - hpage = find_get_page_flags(mapping, pgoff, - FGP_ACCESSED); - if (!hpage) { - ret = -EINVAL; - goto err; - } - } - page = hpage + subpgoff; - get_page(page); - subpgoff++; - if (subpgoff == maxsubpgs) { - put_page(hpage); - hpage = NULL; - subpgoff = 0; - pgoff++; - } - } else { - page = shmem_read_mapping_page(mapping, - pgoff + pgidx); - if (IS_ERR(page)) { - ret = PTR_ERR(page); - goto err; - } - } - ubuf->pages[pgbuf++] = page; } + + /* + * Take the inode lock to protect against concurrent + * memfd_add_seals(), which takes this lock in write mode. + */ + inode_lock_shared(file_inode(memfd)); + ret = check_memfd_seals(memfd); + if (ret) + goto out_unlock; + + ret = udmabuf_pin_folios(ubuf, memfd, list[i].offset, + list[i].size, folios); +out_unlock: + inode_unlock_shared(file_inode(memfd)); fput(memfd); - memfd = NULL; - if (hpage) { - put_page(hpage); - hpage = NULL; - } + if (ret) + goto err; } - exp_info.ops = &udmabuf_ops; - exp_info.size = ubuf->pagecount << PAGE_SHIFT; - exp_info.priv = ubuf; - exp_info.flags = O_RDWR; - - ubuf->device = device; - buf = dma_buf_export(&exp_info); - if (IS_ERR(buf)) { - ret = PTR_ERR(buf); + flags = head->flags & UDMABUF_FLAGS_CLOEXEC ? O_CLOEXEC : 0; + dmabuf = export_udmabuf(ubuf, device); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); goto err; } + /* + * Ownership of ubuf is held by the dmabuf from here. + * If the following dma_buf_fd() fails, dma_buf_put() cleans up both the + * dmabuf and the ubuf (through udmabuf_ops.release). + */ + + ret = dma_buf_fd(dmabuf, flags); + if (ret < 0) + dma_buf_put(dmabuf); - flags = 0; - if (head->flags & UDMABUF_FLAGS_CLOEXEC) - flags |= O_CLOEXEC; - return dma_buf_fd(buf, flags); + kvfree(folios); + return ret; err: - while (pgbuf > 0) - put_page(ubuf->pages[--pgbuf]); - if (memfd) - fput(memfd); - kfree(ubuf->pages); + deinit_udmabuf(ubuf); +err_noinit: kfree(ubuf); + kvfree(folios); return ret; } @@ -430,4 +570,3 @@ module_init(udmabuf_dev_init) module_exit(udmabuf_dev_exit) MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>"); -MODULE_LICENSE("GPL v2"); |