From 2973073a80b46daebc352c31d09d95d16cf6876e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:24 +0100 Subject: swiotlb: remove the alloc_size parameter to swiotlb_tbl_unmap_single Now that swiotlb remembers the allocation size there is no need to pass it back to swiotlb_tbl_unmap_single. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/iommu/dma-iommu.c | 11 +++-------- drivers/xen/swiotlb-xen.c | 4 ++-- 2 files changed, 5 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index af765c813cc8..9149597410e2 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -499,8 +499,6 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, unsigned long attrs) { struct iommu_domain *domain = iommu_get_dma_domain(dev); - struct iommu_dma_cookie *cookie = domain->iova_cookie; - struct iova_domain *iovad = &cookie->iovad; phys_addr_t phys; phys = iommu_iova_to_phys(domain, dma_addr); @@ -510,8 +508,7 @@ static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr, __iommu_dma_unmap(dev, dma_addr, size); if (unlikely(is_swiotlb_buffer(phys))) - swiotlb_tbl_unmap_single(dev, phys, size, - iova_align(iovad, size), dir, attrs); + swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs); } static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys, @@ -581,10 +578,8 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys, } iova = __iommu_dma_map(dev, phys, aligned_size, prot, dma_mask); - if ((iova == DMA_MAPPING_ERROR) && is_swiotlb_buffer(phys)) - swiotlb_tbl_unmap_single(dev, phys, org_size, - aligned_size, dir, attrs); - + if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(phys)) + swiotlb_tbl_unmap_single(dev, phys, org_size, dir, attrs); return iova; } diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 2b385c1b4a99..d47f1b311caa 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -406,7 +406,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, * Ensure that the address returned is DMA'ble */ if (unlikely(!dma_capable(dev, dev_addr, size, true))) { - swiotlb_tbl_unmap_single(dev, map, size, size, dir, + swiotlb_tbl_unmap_single(dev, map, size, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); return DMA_MAPPING_ERROR; } @@ -445,7 +445,7 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, /* NOTE: We use dev_addr here, not paddr! */ if (is_xen_swiotlb_buffer(hwdev, dev_addr)) - swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs); + swiotlb_tbl_unmap_single(hwdev, paddr, size, dir, attrs); } static void -- cgit From 80808d273a3f075196d1a26463f65d4c9d2891c8 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:26 +0100 Subject: swiotlb: split swiotlb_tbl_sync_single Split swiotlb_tbl_sync_single into two separate funtions for the to device and to cpu synchronization. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/iommu/dma-iommu.c | 12 ++++++------ drivers/xen/swiotlb-xen.c | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9149597410e2..3087d9fa6065 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -750,7 +750,7 @@ static void iommu_dma_sync_single_for_cpu(struct device *dev, arch_sync_dma_for_cpu(phys, size, dir); if (is_swiotlb_buffer(phys)) - swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_CPU); + swiotlb_sync_single_for_cpu(dev, phys, size, dir); } static void iommu_dma_sync_single_for_device(struct device *dev, @@ -763,7 +763,7 @@ static void iommu_dma_sync_single_for_device(struct device *dev, phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle); if (is_swiotlb_buffer(phys)) - swiotlb_tbl_sync_single(dev, phys, size, dir, SYNC_FOR_DEVICE); + swiotlb_sync_single_for_device(dev, phys, size, dir); if (!dev_is_dma_coherent(dev)) arch_sync_dma_for_device(phys, size, dir); @@ -784,8 +784,8 @@ static void iommu_dma_sync_sg_for_cpu(struct device *dev, arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir); if (is_swiotlb_buffer(sg_phys(sg))) - swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, - dir, SYNC_FOR_CPU); + swiotlb_sync_single_for_cpu(dev, sg_phys(sg), + sg->length, dir); } } @@ -801,8 +801,8 @@ static void iommu_dma_sync_sg_for_device(struct device *dev, for_each_sg(sgl, sg, nelems, i) { if (is_swiotlb_buffer(sg_phys(sg))) - swiotlb_tbl_sync_single(dev, sg_phys(sg), sg->length, - dir, SYNC_FOR_DEVICE); + swiotlb_sync_single_for_device(dev, sg_phys(sg), + sg->length, dir); if (!dev_is_dma_coherent(dev)) arch_sync_dma_for_device(sg_phys(sg), sg->length, dir); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index d47f1b311caa..4e8a4e14942a 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -462,7 +462,7 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, } if (is_xen_swiotlb_buffer(dev, dma_addr)) - swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); + swiotlb_sync_single_for_cpu(dev, paddr, size, dir); } static void @@ -472,7 +472,7 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr); if (is_xen_swiotlb_buffer(dev, dma_addr)) - swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); + swiotlb_sync_single_for_device(dev, paddr, size, dir); if (!dev_is_dma_coherent(dev)) { if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr)))) -- cgit From 16bc75f3aa3963588b13b01f6dad589f85d9f733 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:27 +0100 Subject: xen-swiotlb: use is_swiotlb_buffer in is_xen_swiotlb_buffer Use the is_swiotlb_buffer to check if a physical address is a swiotlb buffer. This works because xen-swiotlb does use the same buffer as the main swiotlb code, and xen_io_tlb_{start,end} are just the addresses for it that went through phys_to_virt. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 4e8a4e14942a..bffb35993c9d 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -111,10 +111,8 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) * have the same virtual address as another address * in our domain. Therefore _only_ check address within our domain. */ - if (pfn_valid(PFN_DOWN(paddr))) { - return paddr >= virt_to_phys(xen_io_tlb_start) && - paddr < virt_to_phys(xen_io_tlb_end); - } + if (pfn_valid(PFN_DOWN(paddr))) + return is_swiotlb_buffer(paddr); return 0; } -- cgit From 6223d1cef7b462de9aeede5846e6dcdbbd1c4d60 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:28 +0100 Subject: xen-swiotlb: use io_tlb_end in xen_swiotlb_dma_supported Use the existing variable that holds the physical address for xen_io_tlb_end to simplify xen_swiotlb_dma_supported a bit, and remove the otherwise unused xen_io_tlb_end variable and the xen_virt_to_bus helper. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index bffb35993c9d..e99f0614dcb9 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -46,7 +46,7 @@ * API. */ -static char *xen_io_tlb_start, *xen_io_tlb_end; +static char *xen_io_tlb_start; static unsigned long xen_io_tlb_nslabs; /* * Quick lookup value of the bus address of the IOTLB. @@ -82,11 +82,6 @@ static inline phys_addr_t xen_dma_to_phys(struct device *dev, return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr)); } -static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address) -{ - return xen_phys_to_dma(dev, virt_to_phys(address)); -} - static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) { unsigned long next_bfn, xen_pfn = XEN_PFN_DOWN(p); @@ -250,7 +245,6 @@ retry: rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs); end: - xen_io_tlb_end = xen_io_tlb_start + bytes; if (!rc) swiotlb_set_max_segment(PAGE_SIZE); @@ -558,7 +552,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, static int xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask; + return xen_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; } const struct dma_map_ops xen_swiotlb_dma_ops = { -- cgit From 4035b43da6daa51668830becfe8537841414946b Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:29 +0100 Subject: xen-swiotlb: remove xen_set_nslabs The xen_set_nslabs function is a little weird, as it has just one caller, that caller passes a global variable as the argument, which is then overriden in the function and a derivative of it returned. Just add a cpp symbol for the default size using a readable constant and open code the remaining three lines in the caller. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index e99f0614dcb9..5352655432e7 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -138,16 +138,6 @@ xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) } while (i < nslabs); return 0; } -static unsigned long xen_set_nslabs(unsigned long nr_tbl) -{ - if (!nr_tbl) { - xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT); - xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE); - } else - xen_io_tlb_nslabs = nr_tbl; - - return xen_io_tlb_nslabs << IO_TLB_SHIFT; -} enum xen_swiotlb_err { XEN_SWIOTLB_UNKNOWN = 0, @@ -170,6 +160,9 @@ static const char *xen_swiotlb_error(enum xen_swiotlb_err err) } return ""; } + +#define DEFAULT_NSLABS ALIGN(SZ_64M >> IO_TLB_SHIFT, IO_TLB_SEGSIZE) + int __ref xen_swiotlb_init(int verbose, bool early) { unsigned long bytes, order; @@ -179,8 +172,10 @@ int __ref xen_swiotlb_init(int verbose, bool early) xen_io_tlb_nslabs = swiotlb_nr_tbl(); retry: - bytes = xen_set_nslabs(xen_io_tlb_nslabs); - order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT); + if (!xen_io_tlb_nslabs) + xen_io_tlb_nslabs = DEFAULT_NSLABS; + bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; + order = get_order(bytes); /* * IO TLB memory already allocated. Just use it. -- cgit From cbce99527ca7c4e98db9890651d5e9dfc2fb89ac Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:30 +0100 Subject: xen-swiotlb: remove xen_io_tlb_start and xen_io_tlb_nslabs The xen_io_tlb_start and xen_io_tlb_nslabs variables are now only used in xen_swiotlb_init, so replace them with local variables. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 57 +++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 32 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 5352655432e7..1a31ddf71397 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -40,14 +40,7 @@ #include #define MAX_DMA_BITS 32 -/* - * Used to do a quick range check in swiotlb_tbl_unmap_single and - * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this - * API. - */ -static char *xen_io_tlb_start; -static unsigned long xen_io_tlb_nslabs; /* * Quick lookup value of the bus address of the IOTLB. */ @@ -169,75 +162,75 @@ int __ref xen_swiotlb_init(int verbose, bool early) int rc = -ENOMEM; enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN; unsigned int repeat = 3; + char *start; + unsigned long nslabs; - xen_io_tlb_nslabs = swiotlb_nr_tbl(); + nslabs = swiotlb_nr_tbl(); retry: - if (!xen_io_tlb_nslabs) - xen_io_tlb_nslabs = DEFAULT_NSLABS; - bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; + if (!nslabs) + nslabs = DEFAULT_NSLABS; + bytes = nslabs << IO_TLB_SHIFT; order = get_order(bytes); /* * IO TLB memory already allocated. Just use it. */ - if (io_tlb_start != 0) { - xen_io_tlb_start = phys_to_virt(io_tlb_start); + if (io_tlb_start != 0) goto end; - } /* * Get IO TLB memory from any location. */ if (early) { - xen_io_tlb_start = memblock_alloc(PAGE_ALIGN(bytes), + start = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE); - if (!xen_io_tlb_start) + if (!start) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", __func__, PAGE_ALIGN(bytes), PAGE_SIZE); } else { #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { - xen_io_tlb_start = (void *)xen_get_swiotlb_free_pages(order); - if (xen_io_tlb_start) + start = (void *)xen_get_swiotlb_free_pages(order); + if (start) break; order--; } if (order != get_order(bytes)) { pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n", (PAGE_SIZE << order) >> 20); - xen_io_tlb_nslabs = SLABS_PER_PAGE << order; - bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT; + nslabs = SLABS_PER_PAGE << order; + bytes = nslabs << IO_TLB_SHIFT; } } - if (!xen_io_tlb_start) { + if (!start) { m_ret = XEN_SWIOTLB_ENOMEM; goto error; } /* * And replace that memory with pages under 4GB. */ - rc = xen_swiotlb_fixup(xen_io_tlb_start, + rc = xen_swiotlb_fixup(start, bytes, - xen_io_tlb_nslabs); + nslabs); if (rc) { if (early) - memblock_free(__pa(xen_io_tlb_start), + memblock_free(__pa(start), PAGE_ALIGN(bytes)); else { - free_pages((unsigned long)xen_io_tlb_start, order); - xen_io_tlb_start = NULL; + free_pages((unsigned long)start, order); + start = NULL; } m_ret = XEN_SWIOTLB_EFIXUP; goto error; } if (early) { - if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, + if (swiotlb_init_with_tbl(start, nslabs, verbose)) panic("Cannot allocate SWIOTLB buffer"); rc = 0; } else - rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs); + rc = swiotlb_late_init_with_tbl(start, nslabs); end: if (!rc) @@ -246,17 +239,17 @@ end: return rc; error: if (repeat--) { - xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */ - (xen_io_tlb_nslabs >> 1)); + nslabs = max(1024UL, /* Min is 2MB */ + (nslabs >> 1)); pr_info("Lowering to %luMB\n", - (xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20); + (nslabs << IO_TLB_SHIFT) >> 20); goto retry; } pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc); if (early) panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc); else - free_pages((unsigned long)xen_io_tlb_start, order); + free_pages((unsigned long)start, order); return rc; } -- cgit From 5d0538b2b884f7fd239f6ab3b667148dc57123f1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:31 +0100 Subject: swiotlb: lift the double initialization protection from xen-swiotlb Lift the double initialization protection from xen-swiotlb to the core code to avoid exposing too many swiotlb internals. Also upgrade the check to a warning as it should not happen. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 7 ------- 1 file changed, 7 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 1a31ddf71397..060eeb056486 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -172,12 +172,6 @@ retry: bytes = nslabs << IO_TLB_SHIFT; order = get_order(bytes); - /* - * IO TLB memory already allocated. Just use it. - */ - if (io_tlb_start != 0) - goto end; - /* * Get IO TLB memory from any location. */ @@ -232,7 +226,6 @@ retry: } else rc = swiotlb_late_init_with_tbl(start, nslabs); -end: if (!rc) swiotlb_set_max_segment(PAGE_SIZE); -- cgit From a98f565462f0fca9096e8f53933364dc2a74bc90 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:32 +0100 Subject: xen-swiotlb: split xen_swiotlb_init Split xen_swiotlb_init into a normal an an early case. That makes both much simpler and more readable, and also allows marking the early code as __init and x86-only. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 124 ++++++++++++++++++++++++++-------------------- 1 file changed, 70 insertions(+), 54 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 060eeb056486..00adeb95ebb9 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -156,96 +156,112 @@ static const char *xen_swiotlb_error(enum xen_swiotlb_err err) #define DEFAULT_NSLABS ALIGN(SZ_64M >> IO_TLB_SHIFT, IO_TLB_SEGSIZE) -int __ref xen_swiotlb_init(int verbose, bool early) +int __ref xen_swiotlb_init(void) { - unsigned long bytes, order; - int rc = -ENOMEM; enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN; + unsigned long nslabs, bytes, order; unsigned int repeat = 3; + int rc = -ENOMEM; char *start; - unsigned long nslabs; nslabs = swiotlb_nr_tbl(); -retry: if (!nslabs) nslabs = DEFAULT_NSLABS; +retry: + m_ret = XEN_SWIOTLB_ENOMEM; bytes = nslabs << IO_TLB_SHIFT; order = get_order(bytes); /* * Get IO TLB memory from any location. */ - if (early) { - start = memblock_alloc(PAGE_ALIGN(bytes), - PAGE_SIZE); - if (!start) - panic("%s: Failed to allocate %lu bytes align=0x%lx\n", - __func__, PAGE_ALIGN(bytes), PAGE_SIZE); - } else { #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT)) #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT) - while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { - start = (void *)xen_get_swiotlb_free_pages(order); - if (start) - break; - order--; - } - if (order != get_order(bytes)) { - pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n", - (PAGE_SIZE << order) >> 20); - nslabs = SLABS_PER_PAGE << order; - bytes = nslabs << IO_TLB_SHIFT; - } + while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) { + start = (void *)xen_get_swiotlb_free_pages(order); + if (start) + break; + order--; } - if (!start) { - m_ret = XEN_SWIOTLB_ENOMEM; + if (!start) goto error; + if (order != get_order(bytes)) { + pr_warn("Warning: only able to allocate %ld MB for software IO TLB\n", + (PAGE_SIZE << order) >> 20); + nslabs = SLABS_PER_PAGE << order; + bytes = nslabs << IO_TLB_SHIFT; } + /* * And replace that memory with pages under 4GB. */ - rc = xen_swiotlb_fixup(start, - bytes, - nslabs); + rc = xen_swiotlb_fixup(start, bytes, nslabs); if (rc) { - if (early) - memblock_free(__pa(start), - PAGE_ALIGN(bytes)); - else { - free_pages((unsigned long)start, order); - start = NULL; - } + free_pages((unsigned long)start, order); m_ret = XEN_SWIOTLB_EFIXUP; goto error; } - if (early) { - if (swiotlb_init_with_tbl(start, nslabs, - verbose)) - panic("Cannot allocate SWIOTLB buffer"); - rc = 0; - } else - rc = swiotlb_late_init_with_tbl(start, nslabs); - - if (!rc) - swiotlb_set_max_segment(PAGE_SIZE); - - return rc; + rc = swiotlb_late_init_with_tbl(start, nslabs); + if (rc) + return rc; + swiotlb_set_max_segment(PAGE_SIZE); + return 0; error: if (repeat--) { - nslabs = max(1024UL, /* Min is 2MB */ - (nslabs >> 1)); + /* Min is 2MB */ + nslabs = max(1024UL, (nslabs >> 1)); pr_info("Lowering to %luMB\n", (nslabs << IO_TLB_SHIFT) >> 20); goto retry; } pr_err("%s (rc:%d)\n", xen_swiotlb_error(m_ret), rc); - if (early) - panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc); - else - free_pages((unsigned long)start, order); + free_pages((unsigned long)start, order); return rc; } +#ifdef CONFIG_X86 +void __init xen_swiotlb_init_early(void) +{ + unsigned long nslabs, bytes; + unsigned int repeat = 3; + char *start; + int rc; + + nslabs = swiotlb_nr_tbl(); + if (!nslabs) + nslabs = DEFAULT_NSLABS; +retry: + /* + * Get IO TLB memory from any location. + */ + bytes = nslabs << IO_TLB_SHIFT; + start = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE); + if (!start) + panic("%s: Failed to allocate %lu bytes align=0x%lx\n", + __func__, PAGE_ALIGN(bytes), PAGE_SIZE); + + /* + * And replace that memory with pages under 4GB. + */ + rc = xen_swiotlb_fixup(start, bytes, nslabs); + if (rc) { + memblock_free(__pa(start), PAGE_ALIGN(bytes)); + if (repeat--) { + /* Min is 2MB */ + nslabs = max(1024UL, (nslabs >> 1)); + pr_info("Lowering to %luMB\n", + (nslabs << IO_TLB_SHIFT) >> 20); + goto retry; + } + panic("%s (rc:%d)", xen_swiotlb_error(XEN_SWIOTLB_EFIXUP), rc); + } + + if (swiotlb_init_with_tbl(start, nslabs, false)) + panic("Cannot allocate SWIOTLB buffer"); + swiotlb_set_max_segment(PAGE_SIZE); +} +#endif /* CONFIG_X86 */ + static void * xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, dma_addr_t *dma_handle, gfp_t flags, -- cgit From 6bcd4ea717f3d26edf3da397c82fc9c2236f4f27 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 1 Mar 2021 08:44:33 +0100 Subject: xen-swiotlb: remove the unused size argument from xen_swiotlb_fixup Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 00adeb95ebb9..4ecfce2c6f72 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -104,8 +104,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) return 0; } -static int -xen_swiotlb_fixup(void *buf, size_t size, unsigned long nslabs) +static int xen_swiotlb_fixup(void *buf, unsigned long nslabs) { int i, rc; int dma_bits; @@ -195,7 +194,7 @@ retry: /* * And replace that memory with pages under 4GB. */ - rc = xen_swiotlb_fixup(start, bytes, nslabs); + rc = xen_swiotlb_fixup(start, nslabs); if (rc) { free_pages((unsigned long)start, order); m_ret = XEN_SWIOTLB_EFIXUP; @@ -243,7 +242,7 @@ retry: /* * And replace that memory with pages under 4GB. */ - rc = xen_swiotlb_fixup(start, bytes, nslabs); + rc = xen_swiotlb_fixup(start, nslabs); if (rc) { memblock_free(__pa(start), PAGE_ALIGN(bytes)); if (repeat--) { -- cgit From 73f620951b2b594bdc38722c0d647c3b3312af7a Mon Sep 17 00:00:00 2001 From: Claire Chang Date: Thu, 18 Mar 2021 17:14:22 +0100 Subject: swiotlb: move global variables into a new io_tlb_mem structure Added a new struct, io_tlb_mem, as the IO TLB memory pool descriptor and moved relevant global variables into that struct. This will be useful later to allow for restricted DMA pool. Signed-off-by: Claire Chang [hch: rebased] Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 4ecfce2c6f72..5329ad54a5f3 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -548,7 +548,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, static int xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return xen_phys_to_dma(hwdev, io_tlb_end - 1) <= mask; + return xen_phys_to_dma(hwdev, io_tlb_default_mem.end - 1) <= mask; } const struct dma_map_ops xen_swiotlb_dma_ops = { -- cgit From 2d29960af0bee8cc6731b9bd3964850c9e7a6840 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 18 Mar 2021 17:14:23 +0100 Subject: swiotlb: dynamically allocate io_tlb_default_mem Instead of allocating ->list and ->orig_addr separately just do one dynamic allocation for the actual io_tlb_mem structure. This simplifies a lot of the initialization code, and also allows to just check io_tlb_default_mem to see if swiotlb is in use. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/xen/swiotlb-xen.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 5329ad54a5f3..4c89afc0df62 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -158,17 +158,14 @@ static const char *xen_swiotlb_error(enum xen_swiotlb_err err) int __ref xen_swiotlb_init(void) { enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN; - unsigned long nslabs, bytes, order; - unsigned int repeat = 3; + unsigned long bytes = swiotlb_size_or_default(); + unsigned long nslabs = bytes >> IO_TLB_SHIFT; + unsigned int order, repeat = 3; int rc = -ENOMEM; char *start; - nslabs = swiotlb_nr_tbl(); - if (!nslabs) - nslabs = DEFAULT_NSLABS; retry: m_ret = XEN_SWIOTLB_ENOMEM; - bytes = nslabs << IO_TLB_SHIFT; order = get_order(bytes); /* @@ -221,19 +218,16 @@ error: #ifdef CONFIG_X86 void __init xen_swiotlb_init_early(void) { - unsigned long nslabs, bytes; + unsigned long bytes = swiotlb_size_or_default(); + unsigned long nslabs = bytes >> IO_TLB_SHIFT; unsigned int repeat = 3; char *start; int rc; - nslabs = swiotlb_nr_tbl(); - if (!nslabs) - nslabs = DEFAULT_NSLABS; retry: /* * Get IO TLB memory from any location. */ - bytes = nslabs << IO_TLB_SHIFT; start = memblock_alloc(PAGE_ALIGN(bytes), PAGE_SIZE); if (!start) panic("%s: Failed to allocate %lu bytes align=0x%lx\n", @@ -248,8 +242,8 @@ retry: if (repeat--) { /* Min is 2MB */ nslabs = max(1024UL, (nslabs >> 1)); - pr_info("Lowering to %luMB\n", - (nslabs << IO_TLB_SHIFT) >> 20); + bytes = nslabs << IO_TLB_SHIFT; + pr_info("Lowering to %luMB\n", bytes >> 20); goto retry; } panic("%s (rc:%d)", xen_swiotlb_error(XEN_SWIOTLB_EFIXUP), rc); @@ -548,7 +542,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, static int xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return xen_phys_to_dma(hwdev, io_tlb_default_mem.end - 1) <= mask; + return xen_phys_to_dma(hwdev, io_tlb_default_mem->end - 1) <= mask; } const struct dma_map_ops xen_swiotlb_dma_ops = { -- cgit From 2cbc2776efe4faed0e17c48ae076aa03a0fcc61f Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 18 Mar 2021 17:14:24 +0100 Subject: swiotlb: remove swiotlb_nr_tbl All callers just use it to check if swiotlb is active at all, for which they can just use is_swiotlb_active. In the longer run drivers need to stop using is_swiotlb_active as well, but let's do the simple step first. Signed-off-by: Christoph Hellwig Signed-off-by: Konrad Rzeszutek Wilk --- drivers/gpu/drm/i915/gem/i915_gem_internal.c | 2 +- drivers/gpu/drm/nouveau/nouveau_ttm.c | 2 +- drivers/pci/xen-pcifront.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/gpu/drm/i915/gem/i915_gem_internal.c b/drivers/gpu/drm/i915/gem/i915_gem_internal.c index ad22f42541bd..a9d65fc8aa0e 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_internal.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_internal.c @@ -42,7 +42,7 @@ static int i915_gem_object_get_pages_internal(struct drm_i915_gem_object *obj) max_order = MAX_ORDER; #ifdef CONFIG_SWIOTLB - if (swiotlb_nr_tbl()) { + if (is_swiotlb_active()) { unsigned int max_segment; max_segment = swiotlb_max_segment(); diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c index a37bc3d7b38b..9662522aa066 100644 --- a/drivers/gpu/drm/nouveau/nouveau_ttm.c +++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c @@ -321,7 +321,7 @@ nouveau_ttm_init(struct nouveau_drm *drm) } #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86) - need_swiotlb = !!swiotlb_nr_tbl(); + need_swiotlb = is_swiotlb_active(); #endif ret = ttm_bo_device_init(&drm->ttm.bdev, &nouveau_bo_driver, diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index 2d7502648219..b7a8f3a1921f 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -693,7 +693,7 @@ static int pcifront_connect_and_init_dma(struct pcifront_device *pdev) spin_unlock(&pcifront_dev_lock); - if (!err && !swiotlb_nr_tbl()) { + if (!err && !is_swiotlb_active()) { err = pci_xen_swiotlb_init_late(); if (err) dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n"); -- cgit