From 5ed390e5a9d4d84f220e012ee20ae97fc254a9d9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Apr 2022 09:40:17 +0200 Subject: ARM/dma-mapping: mark various dma-mapping routines static in dma-mapping.c With the dmabounce removal these aren't used outside of dma-mapping.c, so mark them static. Move the dma_map_ops declarations down a bit to avoid lots of forward declarations. Signed-off-by: Christoph Hellwig Reviewed-by: Arnd Bergmann Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 100 +++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 54 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 059cce018570..90142183d104 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -193,50 +193,6 @@ static int arm_dma_supported(struct device *dev, u64 mask) return dma_to_pfn(dev, mask) >= max_dma_pfn; } -const struct dma_map_ops arm_dma_ops = { - .alloc = arm_dma_alloc, - .free = arm_dma_free, - .alloc_pages = dma_direct_alloc_pages, - .free_pages = dma_direct_free_pages, - .mmap = arm_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_dma_map_page, - .unmap_page = arm_dma_unmap_page, - .map_sg = arm_dma_map_sg, - .unmap_sg = arm_dma_unmap_sg, - .map_resource = dma_direct_map_resource, - .sync_single_for_cpu = arm_dma_sync_single_for_cpu, - .sync_single_for_device = arm_dma_sync_single_for_device, - .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, - .sync_sg_for_device = arm_dma_sync_sg_for_device, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_dma_ops); - -static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs); -static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs); -static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs); - -const struct dma_map_ops arm_coherent_dma_ops = { - .alloc = arm_coherent_dma_alloc, - .free = arm_coherent_dma_free, - .alloc_pages = dma_direct_alloc_pages, - .free_pages = dma_direct_free_pages, - .mmap = arm_coherent_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_coherent_dma_map_page, - .map_sg = arm_dma_map_sg, - .map_resource = dma_direct_map_resource, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_coherent_dma_ops); - static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) { /* @@ -742,7 +698,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, * Allocate DMA-coherent memory space and return both the kernel remapped * virtual and bus address for that space. */ -void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, +static void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); @@ -791,7 +747,7 @@ static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); } -int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, +static int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { @@ -824,7 +780,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, kfree(buf); } -void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, +static void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, unsigned long attrs) { __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); @@ -836,7 +792,7 @@ static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_add __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); } -int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, +static int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t handle, size_t size, unsigned long attrs) { @@ -977,7 +933,7 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, * Device ownership issues as mentioned for dma_map_single are the same * here. */ -int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, +static int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -1013,8 +969,8 @@ int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, * Unmap a set of streaming mode DMA translations. Again, CPU access * rules concerning calls here are the same as for dma_unmap_single(). */ -void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) +static void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { const struct dma_map_ops *ops = get_dma_ops(dev); struct scatterlist *s; @@ -1032,7 +988,7 @@ void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, * @nents: number of buffers to map (returned from dma_map_sg) * @dir: DMA transfer direction (same as was passed to dma_map_sg) */ -void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, +static void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); @@ -1051,8 +1007,8 @@ void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, * @nents: number of buffers to map (returned from dma_map_sg) * @dir: DMA transfer direction (same as was passed to dma_map_sg) */ -void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) +static void arm_dma_sync_sg_for_device(struct device *dev, + struct scatterlist *sg, int nents, enum dma_data_direction dir) { const struct dma_map_ops *ops = get_dma_ops(dev); struct scatterlist *s; @@ -1063,6 +1019,42 @@ void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, dir); } +const struct dma_map_ops arm_dma_ops = { + .alloc = arm_dma_alloc, + .free = arm_dma_free, + .alloc_pages = dma_direct_alloc_pages, + .free_pages = dma_direct_free_pages, + .mmap = arm_dma_mmap, + .get_sgtable = arm_dma_get_sgtable, + .map_page = arm_dma_map_page, + .unmap_page = arm_dma_unmap_page, + .map_sg = arm_dma_map_sg, + .unmap_sg = arm_dma_unmap_sg, + .map_resource = dma_direct_map_resource, + .sync_single_for_cpu = arm_dma_sync_single_for_cpu, + .sync_single_for_device = arm_dma_sync_single_for_device, + .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, + .sync_sg_for_device = arm_dma_sync_sg_for_device, + .dma_supported = arm_dma_supported, + .get_required_mask = dma_direct_get_required_mask, +}; +EXPORT_SYMBOL(arm_dma_ops); + +const struct dma_map_ops arm_coherent_dma_ops = { + .alloc = arm_coherent_dma_alloc, + .free = arm_coherent_dma_free, + .alloc_pages = dma_direct_alloc_pages, + .free_pages = dma_direct_free_pages, + .mmap = arm_coherent_dma_mmap, + .get_sgtable = arm_dma_get_sgtable, + .map_page = arm_coherent_dma_map_page, + .map_sg = arm_dma_map_sg, + .map_resource = dma_direct_map_resource, + .dma_supported = arm_dma_supported, + .get_required_mask = dma_direct_get_required_mask, +}; +EXPORT_SYMBOL(arm_coherent_dma_ops); + static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent) { /* -- cgit From f9774cfd5062d69dc8ba522369638f919c35391e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Apr 2022 09:48:53 +0200 Subject: ARM/dma-mapping: use dma_to_phys/phys_to_dma in the dma-mapping code Use the helpers as expected by the dma-direct code in the old arm dma-mapping code to ease a gradual switch to the common DMA code. Signed-off-by: Christoph Hellwig Reviewed-by: Arnd Bergmann Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 90142183d104..a09ce16c7ddb 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -128,14 +128,14 @@ static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, { if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) __dma_page_cpu_to_dev(page, offset, size, dir); - return pfn_to_dma(dev, page_to_pfn(page)) + offset; + return phys_to_dma(dev, page_to_phys(page) + offset); } static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { - return pfn_to_dma(dev, page_to_pfn(page)) + offset; + return phys_to_dma(dev, page_to_phys(page) + offset); } /** @@ -156,7 +156,7 @@ static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), + __dma_page_dev_to_cpu(phys_to_page(dma_to_phys(dev, handle)), handle & ~PAGE_MASK, size, dir); } @@ -164,7 +164,7 @@ static void arm_dma_sync_single_for_cpu(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); + struct page *page = phys_to_page(dma_to_phys(dev, handle-offset)); __dma_page_dev_to_cpu(page, offset, size, dir); } @@ -172,7 +172,7 @@ static void arm_dma_sync_single_for_device(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); + struct page *page = phys_to_page(dma_to_phys(dev, handle-offset)); __dma_page_cpu_to_dev(page, offset, size, dir); } @@ -190,7 +190,7 @@ static int arm_dma_supported(struct device *dev, u64 mask) * Translate the device's DMA mask to a PFN limit. This * PFN number includes the page which we can DMA to. */ - return dma_to_pfn(dev, mask) >= max_dma_pfn; + return PHYS_PFN(dma_to_phys(dev, mask)) >= max_dma_pfn; } static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) @@ -681,7 +681,7 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, if (page) { unsigned long flags; - *handle = pfn_to_dma(dev, page_to_pfn(page)); + *handle = phys_to_dma(dev, page_to_phys(page)); buf->virt = args.want_vaddr ? addr : page; spin_lock_irqsave(&arm_dma_bufs_lock, flags); @@ -721,7 +721,7 @@ static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, int ret = -ENXIO; unsigned long nr_vma_pages = vma_pages(vma); unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long pfn = dma_to_pfn(dev, dma_addr); + unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); unsigned long off = vma->vm_pgoff; if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) @@ -762,7 +762,7 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, unsigned long attrs, bool is_coherent) { - struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); + struct page *page = phys_to_page(dma_to_phys(dev, handle)); struct arm_dma_buffer *buf; struct arm_dma_free_args args = { .dev = dev, @@ -796,15 +796,15 @@ static int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t handle, size_t size, unsigned long attrs) { - unsigned long pfn = dma_to_pfn(dev, handle); + phys_addr_t paddr = dma_to_phys(dev, handle); struct page *page; int ret; /* If the PFN is not valid, we do not have a struct page */ - if (!pfn_valid(pfn)) + if (!pfn_valid(PHYS_PFN(paddr))) return -ENXIO; - page = pfn_to_page(pfn); + page = phys_to_page(paddr); ret = sg_alloc_table(sgt, 1, GFP_KERNEL); if (unlikely(ret)) -- cgit From ae626eb97376148bb63c3f3ca9517fde0f39bec3 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 19 Apr 2022 10:28:28 +0200 Subject: ARM/dma-mapping: use dma-direct unconditionally Use dma-direct unconditionally on arm. It has already been used for some time for LPAE and nommu configurations. This mostly changes the streaming mapping implementation and the (simple) coherent allocator for device that are DMA coherent. The existing complex allocator for uncached mappings for non-coherent devices is still used as is using the arch_dma_alloc/arch_dma_free hooks. Signed-off-by: Christoph Hellwig Reviewed-by: Arnd Bergmann Acked-by: Andre Przywara [highbank] Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 365 ++-------------------------------------------- 1 file changed, 15 insertions(+), 350 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index a09ce16c7ddb..ceb56928d01e 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -103,79 +103,8 @@ static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) * before transfers and delay cache invalidation until transfer completion. * */ -static void __dma_page_cpu_to_dev(struct page *, unsigned long, - size_t, enum dma_data_direction); -static void __dma_page_dev_to_cpu(struct page *, unsigned long, - size_t, enum dma_data_direction); - -/** - * arm_dma_map_page - map a portion of a page for streaming DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @page: page that buffer resides in - * @offset: offset into page for start of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Ensure that any data held in the cache is appropriately discarded - * or written back. - * - * The device owns this memory once this call has completed. The CPU - * can regain ownership by calling dma_unmap_page(). - */ -static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_cpu_to_dev(page, offset, size, dir); - return phys_to_dma(dev, page_to_phys(page) + offset); -} - -static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - return phys_to_dma(dev, page_to_phys(page) + offset); -} - -/** - * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @handle: DMA address of buffer - * @size: size of buffer (same as passed to dma_map_page) - * @dir: DMA transfer direction (same as passed to dma_map_page) - * - * Unmap a page streaming mode DMA translation. The handle and size - * must match what was provided in the previous dma_map_page() call. - * All other usages are undefined. - * - * After this call, reads by the CPU to the buffer are guaranteed to see - * whatever the device wrote there. - */ -static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_dev_to_cpu(phys_to_page(dma_to_phys(dev, handle)), - handle & ~PAGE_MASK, size, dir); -} - -static void arm_dma_sync_single_for_cpu(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = phys_to_page(dma_to_phys(dev, handle-offset)); - __dma_page_dev_to_cpu(page, offset, size, dir); -} - -static void arm_dma_sync_single_for_device(struct device *dev, - dma_addr_t handle, size_t size, enum dma_data_direction dir) -{ - unsigned int offset = handle & (PAGE_SIZE - 1); - struct page *page = phys_to_page(dma_to_phys(dev, handle-offset)); - __dma_page_cpu_to_dev(page, offset, size, dir); -} +#ifdef CONFIG_ARM_DMA_USE_IOMMU /* * Return whether the given device DMA address mask can be supported * properly. For example, if your device can only drive the low 24-bits @@ -192,6 +121,7 @@ static int arm_dma_supported(struct device *dev, u64 mask) */ return PHYS_PFN(dma_to_phys(dev, mask)) >= max_dma_pfn; } +#endif static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) { @@ -694,67 +624,6 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, return args.want_vaddr ? addr : page; } -/* - * Allocate DMA-coherent memory space and return both the kernel remapped - * virtual and bus address for that space. - */ -static void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, - gfp_t gfp, unsigned long attrs) -{ - pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); - - return __dma_alloc(dev, size, handle, gfp, prot, false, - attrs, __builtin_return_address(0)); -} - -static void *arm_coherent_dma_alloc(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, - attrs, __builtin_return_address(0)); -} - -static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - int ret = -ENXIO; - unsigned long nr_vma_pages = vma_pages(vma); - unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; - unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); - unsigned long off = vma->vm_pgoff; - - if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) - return ret; - - if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { - ret = remap_pfn_range(vma, vma->vm_start, - pfn + off, - vma->vm_end - vma->vm_start, - vma->vm_page_prot); - } - - return ret; -} - -/* - * Create userspace mapping for the DMA-coherent memory. - */ -static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); -} - -static int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t dma_addr, size_t size, - unsigned long attrs) -{ - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); - return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); -} - /* * Free a buffer as defined by the above mapping. */ @@ -780,40 +649,6 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, kfree(buf); } -static void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); -} - -static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs) -{ - __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); -} - -static int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, - void *cpu_addr, dma_addr_t handle, size_t size, - unsigned long attrs) -{ - phys_addr_t paddr = dma_to_phys(dev, handle); - struct page *page; - int ret; - - /* If the PFN is not valid, we do not have a struct page */ - if (!pfn_valid(PHYS_PFN(paddr))) - return -ENXIO; - - page = phys_to_page(paddr); - - ret = sg_alloc_table(sgt, 1, GFP_KERNEL); - if (unlikely(ret)) - return ret; - - sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); - return 0; -} - static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, void (*op)(const void *, size_t, int)) @@ -917,158 +752,6 @@ static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, } } -/** - * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Map a set of buffers described by scatterlist in streaming mode for DMA. - * This is the scatter-gather version of the dma_map_single interface. - * Here the scatter gather list elements are each tagged with the - * appropriate dma address and length. They are obtained via - * sg_dma_{address,length}. - * - * Device ownership issues as mentioned for dma_map_single are the same - * here. - */ -static int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i, j, ret; - - for_each_sg(sg, s, nents, i) { -#ifdef CONFIG_NEED_SG_DMA_LENGTH - s->dma_length = s->length; -#endif - s->dma_address = ops->map_page(dev, sg_page(s), s->offset, - s->length, dir, attrs); - if (dma_mapping_error(dev, s->dma_address)) { - ret = -EIO; - goto bad_mapping; - } - } - return nents; - - bad_mapping: - for_each_sg(sg, s, i, j) - ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); - return ret; -} - -/** - * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -static void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - - int i; - - for_each_sg(sg, s, nents, i) - ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); -} - -/** - * arm_dma_sync_sg_for_cpu - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map (returned from dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - */ -static void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i; - - for_each_sg(sg, s, nents, i) - ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, - dir); -} - -/** - * arm_dma_sync_sg_for_device - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map (returned from dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - */ -static void arm_dma_sync_sg_for_device(struct device *dev, - struct scatterlist *sg, int nents, enum dma_data_direction dir) -{ - const struct dma_map_ops *ops = get_dma_ops(dev); - struct scatterlist *s; - int i; - - for_each_sg(sg, s, nents, i) - ops->sync_single_for_device(dev, sg_dma_address(s), s->length, - dir); -} - -const struct dma_map_ops arm_dma_ops = { - .alloc = arm_dma_alloc, - .free = arm_dma_free, - .alloc_pages = dma_direct_alloc_pages, - .free_pages = dma_direct_free_pages, - .mmap = arm_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_dma_map_page, - .unmap_page = arm_dma_unmap_page, - .map_sg = arm_dma_map_sg, - .unmap_sg = arm_dma_unmap_sg, - .map_resource = dma_direct_map_resource, - .sync_single_for_cpu = arm_dma_sync_single_for_cpu, - .sync_single_for_device = arm_dma_sync_single_for_device, - .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, - .sync_sg_for_device = arm_dma_sync_sg_for_device, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_dma_ops); - -const struct dma_map_ops arm_coherent_dma_ops = { - .alloc = arm_coherent_dma_alloc, - .free = arm_coherent_dma_free, - .alloc_pages = dma_direct_alloc_pages, - .free_pages = dma_direct_free_pages, - .mmap = arm_coherent_dma_mmap, - .get_sgtable = arm_dma_get_sgtable, - .map_page = arm_coherent_dma_map_page, - .map_sg = arm_dma_map_sg, - .map_resource = dma_direct_map_resource, - .dma_supported = arm_dma_supported, - .get_required_mask = dma_direct_get_required_mask, -}; -EXPORT_SYMBOL(arm_coherent_dma_ops); - -static const struct dma_map_ops *arm_get_dma_map_ops(bool coherent) -{ - /* - * When CONFIG_ARM_LPAE is set, physical address can extend above - * 32-bits, which then can't be addressed by devices that only support - * 32-bit DMA. - * Use the generic dma-direct / swiotlb ops code in that case, as that - * handles bounce buffering for us. - */ - if (IS_ENABLED(CONFIG_ARM_LPAE)) - return NULL; - return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; -} - #ifdef CONFIG_ARM_DMA_USE_IOMMU static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs) @@ -2193,40 +1876,35 @@ void arm_iommu_detach_device(struct device *dev) iommu_detach_device(mapping->domain, dev); kref_put(&mapping->kref, release_iommu_mapping); to_dma_iommu_mapping(dev) = NULL; - set_dma_ops(dev, arm_get_dma_map_ops(dev->archdata.dma_coherent)); + set_dma_ops(dev, NULL); pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); } EXPORT_SYMBOL_GPL(arm_iommu_detach_device); -static const struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) -{ - return coherent ? &iommu_coherent_ops : &iommu_ops; -} - -static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu) +static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, + const struct iommu_ops *iommu, bool coherent) { struct dma_iommu_mapping *mapping; - if (!iommu) - return false; - mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); if (IS_ERR(mapping)) { pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", size, dev_name(dev)); - return false; + return; } if (__arm_iommu_attach_device(dev, mapping)) { pr_warn("Failed to attached device %s to IOMMU_mapping\n", dev_name(dev)); arm_iommu_release_mapping(mapping); - return false; + return; } - return true; + if (coherent) + set_dma_ops(dev, &iommu_coherent_ops); + else + set_dma_ops(dev, &iommu_ops); } static void arm_teardown_iommu_dma_ops(struct device *dev) @@ -2242,27 +1920,20 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) #else -static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, - const struct iommu_ops *iommu) +static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, + const struct iommu_ops *iommu, bool coherent) { - return false; } static void arm_teardown_iommu_dma_ops(struct device *dev) { } -#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops - #endif /* CONFIG_ARM_DMA_USE_IOMMU */ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, const struct iommu_ops *iommu, bool coherent) { - const struct dma_map_ops *dma_ops; - dev->archdata.dma_coherent = coherent; -#ifdef CONFIG_SWIOTLB dev->dma_coherent = coherent; -#endif /* * Don't override the dma_ops if they have already been set. Ideally @@ -2272,12 +1943,8 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, if (dev->dma_ops) return; - if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) - dma_ops = arm_get_iommu_dma_map_ops(coherent); - else - dma_ops = arm_get_dma_map_ops(coherent); - - set_dma_ops(dev, dma_ops); + if (iommu) + arm_setup_iommu_dma_ops(dev, dma_base, size, iommu, coherent); xen_setup_dma_ops(dev); dev->archdata.dma_ops_setup = true; @@ -2293,7 +1960,6 @@ void arch_teardown_dma_ops(struct device *dev) set_dma_ops(dev, NULL); } -#ifdef CONFIG_SWIOTLB void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, enum dma_data_direction dir) { @@ -2321,4 +1987,3 @@ void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, { __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false); } -#endif /* CONFIG_SWIOTLB */ -- cgit From 42998ef08aba01643d99f65718903b4414a8922f Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Apr 2022 12:36:57 +0100 Subject: ARM/dma-mapping: drop .dma_supported for IOMMU ops When an IOMMU is present, we trust that it should be capable of remapping any physical memory, and since the device masks represent the input (virtual) addresses to the IOMMU it makes no sense to validate them against physical PFNs anyway. Signed-off-by: Robin Murphy Signed-off-by: Christoph Hellwig Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 23 ----------------------- 1 file changed, 23 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index ceb56928d01e..4055f2dc2859 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -104,25 +104,6 @@ static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) * */ -#ifdef CONFIG_ARM_DMA_USE_IOMMU -/* - * Return whether the given device DMA address mask can be supported - * properly. For example, if your device can only drive the low 24-bits - * during bus mastering, then you would pass 0x00ffffff as the mask - * to this function. - */ -static int arm_dma_supported(struct device *dev, u64 mask) -{ - unsigned long max_dma_pfn = min(max_pfn - 1, arm_dma_pfn_limit); - - /* - * Translate the device's DMA mask to a PFN limit. This - * PFN number includes the page which we can DMA to. - */ - return PHYS_PFN(dma_to_phys(dev, mask)) >= max_dma_pfn; -} -#endif - static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) { /* @@ -1681,8 +1662,6 @@ static const struct dma_map_ops iommu_ops = { .map_resource = arm_iommu_map_resource, .unmap_resource = arm_iommu_unmap_resource, - - .dma_supported = arm_dma_supported, }; static const struct dma_map_ops iommu_coherent_ops = { @@ -1699,8 +1678,6 @@ static const struct dma_map_ops iommu_coherent_ops = { .map_resource = arm_iommu_map_resource, .unmap_resource = arm_iommu_unmap_resource, - - .dma_supported = arm_dma_supported, }; /** -- cgit From d563bccfa35b59c2cdb982b181cf9de0ddc6c0be Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Apr 2022 12:36:58 +0100 Subject: ARM/dma-mapping: consolidate IOMMU ops callbacks Merge the coherent and non-coherent callbacks down to a single implementation each, relying on the generic dev->dma_coherent flag at the points where the difference matters. Signed-off-by: Robin Murphy Signed-off-by: Christoph Hellwig Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 238 +++++++++++----------------------------------- 1 file changed, 55 insertions(+), 183 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 4055f2dc2859..e7ccf7c82e02 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1079,13 +1079,13 @@ static void __iommu_free_atomic(struct device *dev, void *cpu_addr, __free_from_pool(cpu_addr, size); } -static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs, - int coherent_flag) +static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp, unsigned long attrs) { pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); struct page **pages; void *addr = NULL; + int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; *handle = DMA_MAPPING_ERROR; size = PAGE_ALIGN(size); @@ -1128,19 +1128,7 @@ err_buffer: return NULL; } -static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL); -} - -static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size, - dma_addr_t *handle, gfp_t gfp, unsigned long attrs) -{ - return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT); -} - -static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, +static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) { @@ -1154,35 +1142,24 @@ static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma if (vma->vm_pgoff >= nr_pages) return -ENXIO; + if (!dev->dma_coherent) + vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); + err = vm_map_pages(vma, pages, nr_pages); if (err) pr_err("Remapping memory failed: %d\n", err); return err; } -static int arm_iommu_mmap_attrs(struct device *dev, - struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, unsigned long attrs) -{ - vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); - - return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); -} - -static int arm_coherent_iommu_mmap_attrs(struct device *dev, - struct vm_area_struct *vma, void *cpu_addr, - dma_addr_t dma_addr, size_t size, unsigned long attrs) -{ - return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); -} /* * free a page as defined by the above mapping. * Must not be called with IRQs disabled. */ -static void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle, unsigned long attrs, int coherent_flag) +static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, unsigned long attrs) { + int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; struct page **pages; size = PAGE_ALIGN(size); @@ -1204,19 +1181,6 @@ static void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_ad __iommu_free_buffer(dev, pages, size, attrs); } -static void arm_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, - unsigned long attrs) -{ - __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL); -} - -static void arm_coherent_iommu_free_attrs(struct device *dev, size_t size, - void *cpu_addr, dma_addr_t handle, unsigned long attrs) -{ - __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT); -} - static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs) @@ -1236,8 +1200,7 @@ static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, */ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, size_t size, dma_addr_t *handle, - enum dma_data_direction dir, unsigned long attrs, - bool is_coherent) + enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova, iova_base; @@ -1257,7 +1220,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, phys_addr_t phys = page_to_phys(sg_page(s)); unsigned int len = PAGE_ALIGN(s->offset + s->length); - if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); prot = __dma_info_to_prot(dir, attrs); @@ -1277,9 +1240,20 @@ fail: return ret; } -static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir, unsigned long attrs, - bool is_coherent) +/** + * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA + * @dev: valid struct device pointer + * @sg: list of buffers + * @nents: number of buffers to map + * @dir: DMA transfer direction + * + * Map a set of buffers described by scatterlist in streaming mode for DMA. + * The scatter gather list elements are merged together (if possible) and + * tagged with the appropriate dma address and length. They are obtained via + * sg_dma_{address,length}. + */ +static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, + int nents, enum dma_data_direction dir, unsigned long attrs) { struct scatterlist *s = sg, *dma = sg, *start = sg; int i, count = 0, ret; @@ -1294,8 +1268,7 @@ static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { ret = __map_sg_chunk(dev, start, size, - &dma->dma_address, dir, attrs, - is_coherent); + &dma->dma_address, dir, attrs); if (ret < 0) goto bad_mapping; @@ -1309,8 +1282,7 @@ static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, } size += s->length; } - ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, - is_coherent); + ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs); if (ret < 0) goto bad_mapping; @@ -1328,44 +1300,19 @@ bad_mapping: } /** - * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Map a set of i/o coherent buffers described by scatterlist in streaming - * mode for DMA. The scatter gather list elements are merged together (if - * possible) and tagged with the appropriate dma address and length. They are - * obtained via sg_dma_{address,length}. - */ -static int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - return __iommu_map_sg(dev, sg, nents, dir, attrs, true); -} - -/** - * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA + * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg * @dev: valid struct device pointer * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction + * @nents: number of buffers to unmap (same as was passed to dma_map_sg) + * @dir: DMA transfer direction (same as was passed to dma_map_sg) * - * Map a set of buffers described by scatterlist in streaming mode for DMA. - * The scatter gather list elements are merged together (if possible) and - * tagged with the appropriate dma address and length. They are obtained via - * sg_dma_{address,length}. + * Unmap a set of streaming mode DMA translations. Again, CPU access + * rules concerning calls here are the same as for dma_unmap_single(). */ -static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, unsigned long attrs) -{ - return __iommu_map_sg(dev, sg, nents, dir, attrs, false); -} - -static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, - int nents, enum dma_data_direction dir, - unsigned long attrs, bool is_coherent) +static void arm_iommu_unmap_sg(struct device *dev, + struct scatterlist *sg, int nents, + enum dma_data_direction dir, + unsigned long attrs) { struct scatterlist *s; int i; @@ -1374,47 +1321,12 @@ static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, if (sg_dma_len(s)) __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); - if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); } } -/** - * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -static void arm_coherent_iommu_unmap_sg(struct device *dev, - struct scatterlist *sg, int nents, enum dma_data_direction dir, - unsigned long attrs) -{ - __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); -} - -/** - * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer - * @sg: list of buffers - * @nents: number of buffers to unmap (same as was passed to dma_map_sg) - * @dir: DMA transfer direction (same as was passed to dma_map_sg) - * - * Unmap a set of streaming mode DMA translations. Again, CPU access - * rules concerning calls here are the same as for dma_unmap_single(). - */ -static void arm_iommu_unmap_sg(struct device *dev, - struct scatterlist *sg, int nents, - enum dma_data_direction dir, - unsigned long attrs) -{ - __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); -} - /** * arm_iommu_sync_sg_for_cpu * @dev: valid struct device pointer @@ -1452,18 +1364,17 @@ static void arm_iommu_sync_sg_for_device(struct device *dev, __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); } - /** - * arm_coherent_iommu_map_page + * arm_iommu_map_page * @dev: valid struct device pointer * @page: page that buffer resides in * @offset: offset into page for start of buffer * @size: size of buffer to map * @dir: DMA transfer direction * - * Coherent IOMMU aware version of arm_dma_map_page() + * IOMMU aware version of arm_dma_map_page() */ -static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, +static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -1471,6 +1382,9 @@ static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *p dma_addr_t dma_addr; int ret, prot, len = PAGE_ALIGN(size + offset); + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + __dma_page_cpu_to_dev(page, offset, size, dir); + dma_addr = __alloc_iova(mapping, len); if (dma_addr == DMA_MAPPING_ERROR) return dma_addr; @@ -1487,50 +1401,6 @@ fail: return DMA_MAPPING_ERROR; } -/** - * arm_iommu_map_page - * @dev: valid struct device pointer - * @page: page that buffer resides in - * @offset: offset into page for start of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * IOMMU aware version of arm_dma_map_page() - */ -static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) - __dma_page_cpu_to_dev(page, offset, size, dir); - - return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); -} - -/** - * arm_coherent_iommu_unmap_page - * @dev: valid struct device pointer - * @handle: DMA address of buffer - * @size: size of buffer (same as passed to dma_map_page) - * @dir: DMA transfer direction (same as passed to dma_map_page) - * - * Coherent IOMMU aware version of arm_dma_unmap_page() - */ -static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, - size_t size, enum dma_data_direction dir, unsigned long attrs) -{ - struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); - dma_addr_t iova = handle & PAGE_MASK; - int offset = handle & ~PAGE_MASK; - int len = PAGE_ALIGN(size + offset); - - if (!iova) - return; - - iommu_unmap(mapping->domain, iova, len); - __free_iova(mapping, iova, len); -} - /** * arm_iommu_unmap_page * @dev: valid struct device pointer @@ -1545,15 +1415,17 @@ static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; int offset = handle & ~PAGE_MASK; int len = PAGE_ALIGN(size + offset); if (!iova) return; - if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) + if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_dev_to_cpu(page, offset, size, dir); + } iommu_unmap(mapping->domain, iova, len); __free_iova(mapping, iova, len); @@ -1665,16 +1537,16 @@ static const struct dma_map_ops iommu_ops = { }; static const struct dma_map_ops iommu_coherent_ops = { - .alloc = arm_coherent_iommu_alloc_attrs, - .free = arm_coherent_iommu_free_attrs, - .mmap = arm_coherent_iommu_mmap_attrs, + .alloc = arm_iommu_alloc_attrs, + .free = arm_iommu_free_attrs, + .mmap = arm_iommu_mmap_attrs, .get_sgtable = arm_iommu_get_sgtable, - .map_page = arm_coherent_iommu_map_page, - .unmap_page = arm_coherent_iommu_unmap_page, + .map_page = arm_iommu_map_page, + .unmap_page = arm_iommu_unmap_page, - .map_sg = arm_coherent_iommu_map_sg, - .unmap_sg = arm_coherent_iommu_unmap_sg, + .map_sg = arm_iommu_map_sg, + .unmap_sg = arm_iommu_unmap_sg, .map_resource = arm_iommu_map_resource, .unmap_resource = arm_iommu_unmap_resource, -- cgit From 4136ce90f079e812fec2c5e10732b2265f61f1bd Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 21 Apr 2022 12:36:59 +0100 Subject: ARM/dma-mapping: merge IOMMU ops The dma_sync_* operations are now the only difference between the coherent and non-coherent IOMMU ops. Some minor tweaks to make those safe for coherent devices with minimal overhead, and we can condense down to a single set of DMA ops. Signed-off-by: Robin Murphy Signed-off-by: Christoph Hellwig Tested-by: Marc Zyngier --- arch/arm/mm/dma-mapping.c | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index e7ccf7c82e02..e68d1d2ac4be 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -1341,6 +1341,9 @@ static void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *s; int i; + if (dev->dma_coherent) + return; + for_each_sg(sg, s, nents, i) __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); @@ -1360,6 +1363,9 @@ static void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *s; int i; + if (dev->dma_coherent) + return; + for_each_sg(sg, s, nents, i) __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); } @@ -1493,12 +1499,13 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; unsigned int offset = handle & ~PAGE_MASK; - if (!iova) + if (dev->dma_coherent || !iova) return; + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_dev_to_cpu(page, offset, size, dir); } @@ -1507,12 +1514,13 @@ static void arm_iommu_sync_single_for_device(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); + struct page *page; unsigned int offset = handle & ~PAGE_MASK; - if (!iova) + if (dev->dma_coherent || !iova) return; + page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); __dma_page_cpu_to_dev(page, offset, size, dir); } @@ -1536,22 +1544,6 @@ static const struct dma_map_ops iommu_ops = { .unmap_resource = arm_iommu_unmap_resource, }; -static const struct dma_map_ops iommu_coherent_ops = { - .alloc = arm_iommu_alloc_attrs, - .free = arm_iommu_free_attrs, - .mmap = arm_iommu_mmap_attrs, - .get_sgtable = arm_iommu_get_sgtable, - - .map_page = arm_iommu_map_page, - .unmap_page = arm_iommu_unmap_page, - - .map_sg = arm_iommu_map_sg, - .unmap_sg = arm_iommu_unmap_sg, - - .map_resource = arm_iommu_map_resource, - .unmap_resource = arm_iommu_unmap_resource, -}; - /** * arm_iommu_create_mapping * @bus: pointer to the bus holding the client device (for IOMMU calls) @@ -1750,10 +1742,7 @@ static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, return; } - if (coherent) - set_dma_ops(dev, &iommu_coherent_ops); - else - set_dma_ops(dev, &iommu_ops); + set_dma_ops(dev, &iommu_ops); } static void arm_teardown_iommu_dma_ops(struct device *dev) -- cgit From a45e52bf1eca9bf8a7d075ce12b1e306497ce393 Mon Sep 17 00:00:00 2001 From: Lukas Bulwahn Date: Wed, 20 Jul 2022 21:02:08 +0200 Subject: dma-mapping: update comment after dmabounce removal Commit e3217540c271 ("ARM/dma-mapping: remove dmabounce") removes the config DMABOUNCE. A comment to the function __dma_page_cpu_to_dev() refers to this removed config DMABOUNCE. Remove the obsolete explanation, but keep the recommendation not to use __dma_page_cpu_to_dev() and use dma_sync_* functions instead. Signed-off-by: Lukas Bulwahn Acked-by: Arnd Bergmann Signed-off-by: Christoph Hellwig --- arch/arm/mm/dma-mapping.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'arch/arm/mm/dma-mapping.c') diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index e68d1d2ac4be..16c2cdd04639 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -679,8 +679,7 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, /* * Make an area consistent for devices. - * Note: Drivers should NOT use this function directly, as it will break - * platforms with CONFIG_DMABOUNCE. + * Note: Drivers should NOT use this function directly. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, -- cgit