summaryrefslogtreecommitdiff
path: root/include/linux
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2024-07-19 10:20:26 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2024-07-19 10:20:26 -0700
commitafd81d914f6fb3e74a46bf5d0dd0b028591ea22e (patch)
treed442b5de397de72b365d263e6a9c577fcb7aa0a6 /include/linux
parentebcfbf02abfbecc144440ff797419cc95cb047fe (diff)
parentb69bdba5a37eb6224039e9572e0e98fc3a931fee (diff)
Merge tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping
Pull dma-mapping updates from Christoph Hellwig: - reduce duplicate swiotlb pool lookups (Michael Kelley) - minor small fixes (Yicong Yang, Yang Li) * tag 'dma-mapping-6.11-2024-07-19' of git://git.infradead.org/users/hch/dma-mapping: swiotlb: fix kernel-doc description for swiotlb_del_transient swiotlb: reduce swiotlb pool lookups dma-mapping: benchmark: Don't starve others when doing the test
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/scatterlist.h2
-rw-r--r--include/linux/swiotlb.h105
2 files changed, 63 insertions, 44 deletions
diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h
index 77df3d7b18a6..e61d164622db 100644
--- a/include/linux/scatterlist.h
+++ b/include/linux/scatterlist.h
@@ -332,7 +332,7 @@ static inline void sg_dma_unmark_bus_address(struct scatterlist *sg)
* Description:
* Returns true if the scatterlist was marked for SWIOTLB bouncing. Not all
* elements may have been bounced, so the caller would have to check
- * individual SG entries with is_swiotlb_buffer().
+ * individual SG entries with swiotlb_find_pool().
*/
static inline bool sg_dma_is_swiotlb(struct scatterlist *sg)
{
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 14bc10c1bb23..3dae0f592063 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -42,24 +42,6 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
int (*remap)(void *tlb, unsigned long nslabs));
extern void __init swiotlb_update_mem_attributes(void);
-phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
- size_t mapping_size,
- unsigned int alloc_aligned_mask, enum dma_data_direction dir,
- unsigned long attrs);
-
-extern void swiotlb_tbl_unmap_single(struct device *hwdev,
- phys_addr_t tlb_addr,
- size_t mapping_size,
- enum dma_data_direction dir,
- unsigned long attrs);
-
-void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
- size_t size, enum dma_data_direction dir);
-void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
- size_t size, enum dma_data_direction dir);
-dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
- size_t size, enum dma_data_direction dir, unsigned long attrs);
-
#ifdef CONFIG_SWIOTLB
/**
@@ -143,37 +125,27 @@ struct io_tlb_mem {
#endif
};
-#ifdef CONFIG_SWIOTLB_DYNAMIC
-
-struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
-
-#else
-
-static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
- phys_addr_t paddr)
-{
- return &dev->dma_io_tlb_mem->defpool;
-}
-
-#endif
+struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
/**
- * is_swiotlb_buffer() - check if a physical address belongs to a swiotlb
+ * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs
* @dev: Device which has mapped the buffer.
* @paddr: Physical address within the DMA buffer.
*
- * Check if @paddr points into a bounce buffer.
+ * Find the swiotlb pool that @paddr points into.
*
* Return:
- * * %true if @paddr points into a bounce buffer
- * * %false otherwise
+ * * pool address if @paddr points into a bounce buffer
+ * * NULL if @paddr does not point into a bounce buffer. As such, this function
+ * can be used to determine if @paddr denotes a swiotlb bounce buffer.
*/
-static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
+static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
+ phys_addr_t paddr)
{
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
if (!mem)
- return false;
+ return NULL;
#ifdef CONFIG_SWIOTLB_DYNAMIC
/*
@@ -182,16 +154,19 @@ static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
* If a SWIOTLB address is checked on another CPU, then it was
* presumably loaded by the device driver from an unspecified private
* data structure. Make sure that this load is ordered before reading
- * dev->dma_uses_io_tlb here and mem->pools in swiotlb_find_pool().
+ * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool().
*
* This barrier pairs with smp_mb() in swiotlb_find_slots().
*/
smp_rmb();
- return READ_ONCE(dev->dma_uses_io_tlb) &&
- swiotlb_find_pool(dev, paddr);
+ if (READ_ONCE(dev->dma_uses_io_tlb))
+ return __swiotlb_find_pool(dev, paddr);
#else
- return paddr >= mem->defpool.start && paddr < mem->defpool.end;
+ if (paddr >= mem->defpool.start && paddr < mem->defpool.end)
+ return &mem->defpool;
#endif
+
+ return NULL;
}
static inline bool is_swiotlb_force_bounce(struct device *dev)
@@ -219,9 +194,10 @@ static inline void swiotlb_dev_init(struct device *dev)
{
}
-static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
+static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
+ phys_addr_t paddr)
{
- return false;
+ return NULL;
}
static inline bool is_swiotlb_force_bounce(struct device *dev)
{
@@ -260,6 +236,49 @@ static inline phys_addr_t default_swiotlb_limit(void)
}
#endif /* CONFIG_SWIOTLB */
+phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
+ size_t mapping_size, unsigned int alloc_aligned_mask,
+ enum dma_data_direction dir, unsigned long attrs);
+dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
+ size_t size, enum dma_data_direction dir, unsigned long attrs);
+
+void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
+ size_t mapping_size, enum dma_data_direction dir,
+ unsigned long attrs, struct io_tlb_pool *pool);
+static inline void swiotlb_tbl_unmap_single(struct device *dev,
+ phys_addr_t addr, size_t size, enum dma_data_direction dir,
+ unsigned long attrs)
+{
+ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
+
+ if (unlikely(pool))
+ __swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);
+}
+
+void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
+ size_t size, enum dma_data_direction dir,
+ struct io_tlb_pool *pool);
+static inline void swiotlb_sync_single_for_device(struct device *dev,
+ phys_addr_t addr, size_t size, enum dma_data_direction dir)
+{
+ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
+
+ if (unlikely(pool))
+ __swiotlb_sync_single_for_device(dev, addr, size, dir, pool);
+}
+
+void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
+ size_t size, enum dma_data_direction dir,
+ struct io_tlb_pool *pool);
+static inline void swiotlb_sync_single_for_cpu(struct device *dev,
+ phys_addr_t addr, size_t size, enum dma_data_direction dir)
+{
+ struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
+
+ if (unlikely(pool))
+ __swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);
+}
+
extern void swiotlb_print_info(void);
#ifdef CONFIG_DMA_RESTRICTED_POOL