diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c | 369 |
1 files changed, 229 insertions, 140 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c index 6d11e1721147..d2237ce9da70 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c @@ -25,12 +25,18 @@ * Alex Deucher * Jerome Glisse */ -#include <drm/drmP.h> + +#include <linux/pci.h> +#include <linux/vmalloc.h> + #include <drm/amdgpu_drm.h> #ifdef CONFIG_X86 #include <asm/set_memory.h> #endif #include "amdgpu.h" +#include "amdgpu_reset.h" +#include <drm/drm_drv.h> +#include <drm/ttm/ttm_tt.h> /* * GART @@ -57,7 +63,7 @@ */ /** - * amdgpu_dummy_page_init - init dummy page used by the driver + * amdgpu_gart_dummy_page_init - init dummy page used by the driver * * @adev: amdgpu_device pointer * @@ -68,13 +74,14 @@ */ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) { - struct page *dummy_page = adev->mman.bdev.glob->dummy_read_page; + struct page *dummy_page = ttm_glob.dummy_read_page; if (adev->dummy_page_addr) return 0; - adev->dummy_page_addr = pci_map_page(adev->pdev, dummy_page, 0, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(adev->pdev, adev->dummy_page_addr)) { + adev->dummy_page_addr = dma_map_page_attrs(&adev->pdev->dev, dummy_page, 0, + PAGE_SIZE, DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (dma_mapping_error(&adev->pdev->dev, adev->dummy_page_addr)) { dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n"); adev->dummy_page_addr = 0; return -ENOMEM; @@ -83,105 +90,184 @@ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) } /** - * amdgpu_dummy_page_fini - free dummy page used by the driver + * amdgpu_gart_dummy_page_fini - free dummy page used by the driver * * @adev: amdgpu_device pointer * * Frees the dummy page used by the driver (all asics). */ -static void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) +void amdgpu_gart_dummy_page_fini(struct amdgpu_device *adev) { if (!adev->dummy_page_addr) return; - pci_unmap_page(adev->pdev, adev->dummy_page_addr, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); + dma_unmap_page_attrs(&adev->pdev->dev, adev->dummy_page_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); adev->dummy_page_addr = 0; } /** - * amdgpu_gart_table_vram_alloc - allocate vram for gart page table + * amdgpu_gart_table_ram_alloc - allocate system ram for gart page table * * @adev: amdgpu_device pointer * - * Allocate video memory for GART page table - * (pcie r4xx, r5xx+). These asics require the - * gart table to be in video memory. + * Allocate system memory for GART page table for ASICs that don't have + * dedicated VRAM. * Returns 0 for success, error for failure. */ -int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev) +int amdgpu_gart_table_ram_alloc(struct amdgpu_device *adev) { - int r; + unsigned int order = get_order(adev->gart.table_size); + gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; + struct amdgpu_bo *bo = NULL; + struct sg_table *sg = NULL; + struct amdgpu_bo_param bp; + dma_addr_t dma_addr; + struct page *p; + unsigned long x; + int ret; + + if (adev->gart.bo != NULL) + return 0; - if (adev->gart.bo == NULL) { - struct amdgpu_bo_param bp; - - memset(&bp, 0, sizeof(bp)); - bp.size = adev->gart.table_size; - bp.byte_align = PAGE_SIZE; - bp.domain = AMDGPU_GEM_DOMAIN_VRAM; - bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | - AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; - bp.type = ttm_bo_type_kernel; - bp.resv = NULL; - r = amdgpu_bo_create(adev, &bp, &adev->gart.bo); - if (r) { - return r; - } + p = alloc_pages(gfp_flags, order); + if (!p) + return -ENOMEM; + + /* assign pages to this device */ + for (x = 0; x < (1UL << order); x++) + p[x].mapping = adev->mman.bdev.dev_mapping; + + /* If the hardware does not support UTCL2 snooping of the CPU caches + * then set_memory_wc() could be used as a workaround to mark the pages + * as write combine memory. + */ + dma_addr = dma_map_page(&adev->pdev->dev, p, 0, adev->gart.table_size, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(&adev->pdev->dev, dma_addr)) { + dev_err(&adev->pdev->dev, "Failed to DMA MAP the GART BO page\n"); + __free_pages(p, order); + p = NULL; + return -EFAULT; } + + dev_info(adev->dev, "%s dma_addr:%pad\n", __func__, &dma_addr); + /* Create SG table */ + sg = kmalloc(sizeof(*sg), GFP_KERNEL); + if (!sg) { + ret = -ENOMEM; + goto error; + } + ret = sg_alloc_table(sg, 1, GFP_KERNEL); + if (ret) + goto error; + + sg_dma_address(sg->sgl) = dma_addr; + sg->sgl->length = adev->gart.table_size; +#ifdef CONFIG_NEED_SG_DMA_LENGTH + sg->sgl->dma_length = adev->gart.table_size; +#endif + /* Create SG BO */ + memset(&bp, 0, sizeof(bp)); + bp.size = adev->gart.table_size; + bp.byte_align = PAGE_SIZE; + bp.domain = AMDGPU_GEM_DOMAIN_CPU; + bp.type = ttm_bo_type_sg; + bp.resv = NULL; + bp.bo_ptr_size = sizeof(struct amdgpu_bo); + bp.flags = 0; + ret = amdgpu_bo_create(adev, &bp, &bo); + if (ret) + goto error; + + bo->tbo.sg = sg; + bo->tbo.ttm->sg = sg; + bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; + bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; + + ret = amdgpu_bo_reserve(bo, true); + if (ret) { + dev_err(adev->dev, "(%d) failed to reserve bo for GART system bo\n", ret); + goto error; + } + + ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); + WARN(ret, "Pinning the GART table failed"); + if (ret) + goto error_resv; + + adev->gart.bo = bo; + adev->gart.ptr = page_to_virt(p); + /* Make GART table accessible in VMID0 */ + ret = amdgpu_ttm_alloc_gart(&adev->gart.bo->tbo); + if (ret) + amdgpu_gart_table_ram_free(adev); + amdgpu_bo_unreserve(bo); + return 0; + +error_resv: + amdgpu_bo_unreserve(bo); +error: + amdgpu_bo_unref(&bo); + if (sg) { + sg_free_table(sg); + kfree(sg); + } + __free_pages(p, order); + return ret; } /** - * amdgpu_gart_table_vram_pin - pin gart page table in vram + * amdgpu_gart_table_ram_free - free gart page table system ram * * @adev: amdgpu_device pointer * - * Pin the GART page table in vram so it will not be moved - * by the memory manager (pcie r4xx, r5xx+). These asics require the - * gart table to be in video memory. - * Returns 0 for success, error for failure. + * Free the system memory used for the GART page tableon ASICs that don't + * have dedicated VRAM. */ -int amdgpu_gart_table_vram_pin(struct amdgpu_device *adev) +void amdgpu_gart_table_ram_free(struct amdgpu_device *adev) { - int r; - - r = amdgpu_bo_reserve(adev->gart.bo, false); - if (unlikely(r != 0)) - return r; - r = amdgpu_bo_pin(adev->gart.bo, AMDGPU_GEM_DOMAIN_VRAM); - if (r) { + unsigned int order = get_order(adev->gart.table_size); + struct sg_table *sg = adev->gart.bo->tbo.sg; + struct page *p; + unsigned long x; + int ret; + + ret = amdgpu_bo_reserve(adev->gart.bo, false); + if (!ret) { + amdgpu_bo_unpin(adev->gart.bo); amdgpu_bo_unreserve(adev->gart.bo); - return r; } - r = amdgpu_bo_kmap(adev->gart.bo, &adev->gart.ptr); - if (r) - amdgpu_bo_unpin(adev->gart.bo); - amdgpu_bo_unreserve(adev->gart.bo); - return r; + amdgpu_bo_unref(&adev->gart.bo); + sg_free_table(sg); + kfree(sg); + p = virt_to_page(adev->gart.ptr); + for (x = 0; x < (1UL << order); x++) + p[x].mapping = NULL; + __free_pages(p, order); + + adev->gart.ptr = NULL; } /** - * amdgpu_gart_table_vram_unpin - unpin gart page table in vram + * amdgpu_gart_table_vram_alloc - allocate vram for gart page table * * @adev: amdgpu_device pointer * - * Unpin the GART page table in vram (pcie r4xx, r5xx+). - * These asics require the gart table to be in video memory. + * Allocate video memory for GART page table + * (pcie r4xx, r5xx+). These asics require the + * gart table to be in video memory. + * Returns 0 for success, error for failure. */ -void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev) +int amdgpu_gart_table_vram_alloc(struct amdgpu_device *adev) { - int r; + if (adev->gart.bo != NULL) + return 0; - if (adev->gart.bo == NULL) { - return; - } - r = amdgpu_bo_reserve(adev->gart.bo, true); - if (likely(r == 0)) { - amdgpu_bo_kunmap(adev->gart.bo); - amdgpu_bo_unpin(adev->gart.bo); - amdgpu_bo_unreserve(adev->gart.bo); - adev->gart.ptr = NULL; - } + return amdgpu_bo_create_kernel(adev, adev->gart.table_size, PAGE_SIZE, + AMDGPU_GEM_DOMAIN_VRAM, &adev->gart.bo, + NULL, (void *)&adev->gart.ptr); } /** @@ -195,10 +281,7 @@ void amdgpu_gart_table_vram_unpin(struct amdgpu_device *adev) */ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev) { - if (adev->gart.bo == NULL) { - return; - } - amdgpu_bo_unref(&adev->gart.bo); + amdgpu_bo_free_kernel(&adev->gart.bo, NULL, (void *)&adev->gart.ptr); } /* @@ -215,27 +298,24 @@ void amdgpu_gart_table_vram_free(struct amdgpu_device *adev) * replaces them with the dummy page (all asics). * Returns 0 for success, -EINVAL for failure. */ -int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, +void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, int pages) { unsigned t; - unsigned p; int i, j; u64 page_base; /* Starting from VEGA10, system bit must be 0 to mean invalid. */ uint64_t flags = 0; + int idx; - if (!adev->gart.ready) { - WARN(1, "trying to unbind memory from uninitialized GART !\n"); - return -EINVAL; - } + if (!adev->gart.ptr) + return; + + if (!drm_dev_enter(adev_to_drm(adev), &idx)) + return; t = offset / AMDGPU_GPU_PAGE_SIZE; - p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE; - for (i = 0; i < pages; i++, p++) { -#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS - adev->gart.pages[p] = NULL; -#endif + for (i = 0; i < pages; i++) { page_base = adev->dummy_page_addr; if (!adev->gart.ptr) continue; @@ -246,10 +326,9 @@ int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, page_base += AMDGPU_GPU_PAGE_SIZE; } } - mb(); - amdgpu_asic_flush_hdp(adev, NULL); - amdgpu_gmc_flush_gpu_tlb(adev, 0, 0); - return 0; + amdgpu_gart_invalidate_tlb(adev); + + drm_dev_exit(idx); } /** @@ -265,17 +344,16 @@ int amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset, * Map the dma_addresses into GART entries (all asics). * Returns 0 for success, -EINVAL for failure. */ -int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, +void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, int pages, dma_addr_t *dma_addr, uint64_t flags, void *dst) { uint64_t page_base; unsigned i, j, t; + int idx; - if (!adev->gart.ready) { - WARN(1, "trying to bind memory to uninitialized GART !\n"); - return -EINVAL; - } + if (!drm_dev_enter(adev_to_drm(adev), &idx)) + return; t = offset / AMDGPU_GPU_PAGE_SIZE; @@ -286,7 +364,43 @@ int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, page_base += AMDGPU_GPU_PAGE_SIZE; } } - return 0; + drm_dev_exit(idx); +} + +/** + * amdgpu_gart_map_vram_range - map VRAM pages into the GART page table + * + * @adev: amdgpu_device pointer + * @pa: physical address of the first page to be mapped + * @start_page: first page to map in the GART aperture + * @num_pages: number of pages to be mapped + * @flags: page table entry flags + * @dst: CPU address of the GART table + * + * Binds a BO that is allocated in VRAM to the GART page table + * (all ASICs). + * + * Useful when a kernel BO is located in VRAM but + * needs to be accessed from the GART address space. + */ +void amdgpu_gart_map_vram_range(struct amdgpu_device *adev, uint64_t pa, + uint64_t start_page, uint64_t num_pages, + uint64_t flags, void *dst) +{ + u32 i, idx; + + /* The SYSTEM flag indicates the pages aren't in VRAM. */ + WARN_ON_ONCE(flags & AMDGPU_PTE_SYSTEM); + + if (!drm_dev_enter(adev_to_drm(adev), &idx)) + return; + + for (i = 0; i < num_pages; ++i) { + amdgpu_gmc_set_pte_pde(adev, adev->gart.ptr, + start_page + i, pa + AMDGPU_GPU_PAGE_SIZE * i, flags); + } + + drm_dev_exit(idx); } /** @@ -295,46 +409,45 @@ int amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset, * @adev: amdgpu_device pointer * @offset: offset into the GPU's gart aperture * @pages: number of pages to bind - * @pagelist: pages to bind * @dma_addr: DMA addresses of pages + * @flags: page table entry flags * * Binds the requested pages to the gart page table * (all asics). * Returns 0 for success, -EINVAL for failure. */ -int amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, - int pages, struct page **pagelist, dma_addr_t *dma_addr, +void amdgpu_gart_bind(struct amdgpu_device *adev, uint64_t offset, + int pages, dma_addr_t *dma_addr, uint64_t flags) { -#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS - unsigned i,t,p; -#endif - int r; + if (!adev->gart.ptr) + return; - if (!adev->gart.ready) { - WARN(1, "trying to bind memory to uninitialized GART !\n"); - return -EINVAL; - } + amdgpu_gart_map(adev, offset, pages, dma_addr, flags, adev->gart.ptr); +} -#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS - t = offset / AMDGPU_GPU_PAGE_SIZE; - p = t / AMDGPU_GPU_PAGES_IN_CPU_PAGE; - for (i = 0; i < pages; i++, p++) - adev->gart.pages[p] = pagelist ? pagelist[i] : NULL; -#endif +/** + * amdgpu_gart_invalidate_tlb - invalidate gart TLB + * + * @adev: amdgpu device driver pointer + * + * Invalidate gart TLB which can be use as a way to flush gart changes + * + */ +void amdgpu_gart_invalidate_tlb(struct amdgpu_device *adev) +{ + int i; if (!adev->gart.ptr) - return 0; - - r = amdgpu_gart_map(adev, offset, pages, dma_addr, flags, - adev->gart.ptr); - if (r) - return r; + return; mb(); - amdgpu_asic_flush_hdp(adev, NULL); - amdgpu_gmc_flush_gpu_tlb(adev, 0, 0); - return 0; + if (down_read_trylock(&adev->reset_domain->sem)) { + amdgpu_device_flush_hdp(adev, NULL); + up_read(&adev->reset_domain->sem); + } + for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) + amdgpu_gmc_flush_gpu_tlb(adev, 0, i, 0); } /** @@ -366,29 +479,5 @@ int amdgpu_gart_init(struct amdgpu_device *adev) DRM_INFO("GART: num cpu pages %u, num gpu pages %u\n", adev->gart.num_cpu_pages, adev->gart.num_gpu_pages); -#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS - /* Allocate pages table */ - adev->gart.pages = vzalloc(array_size(sizeof(void *), - adev->gart.num_cpu_pages)); - if (adev->gart.pages == NULL) - return -ENOMEM; -#endif - return 0; } - -/** - * amdgpu_gart_fini - tear down the driver info for managing the gart - * - * @adev: amdgpu_device pointer - * - * Tear down the gart driver info and free the dummy page (all asics). - */ -void amdgpu_gart_fini(struct amdgpu_device *adev) -{ -#ifdef CONFIG_DRM_AMDGPU_GART_DEBUGFS - vfree(adev->gart.pages); - adev->gart.pages = NULL; -#endif - amdgpu_gart_dummy_page_fini(adev); -} |