diff options
Diffstat (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c')
| -rw-r--r-- | drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 972 |
1 files changed, 800 insertions, 172 deletions
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c index 3f9d5d00c9b3..9d934c07fa6b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c @@ -22,72 +22,242 @@ * Authors: Christian König */ -#include <drm/drmP.h> +#include <linux/dma-mapping.h> +#include <drm/ttm/ttm_range_manager.h> +#include <drm/drm_drv.h> + #include "amdgpu.h" +#include "amdgpu_vm.h" +#include "amdgpu_res_cursor.h" +#include "atom.h" + +#define AMDGPU_MAX_SG_SEGMENT_SIZE (2UL << 30) -struct amdgpu_vram_mgr { - struct drm_mm mm; - spinlock_t lock; - atomic64_t usage; - atomic64_t vis_usage; +struct amdgpu_vram_reservation { + u64 start; + u64 size; + struct list_head allocated; + struct list_head blocks; }; +static inline struct amdgpu_vram_mgr * +to_vram_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct amdgpu_vram_mgr, manager); +} + +static inline struct amdgpu_device * +to_amdgpu_device(struct amdgpu_vram_mgr *mgr) +{ + return container_of(mgr, struct amdgpu_device, mman.vram_mgr); +} + +static inline struct drm_buddy_block * +amdgpu_vram_mgr_first_block(struct list_head *list) +{ + return list_first_entry_or_null(list, struct drm_buddy_block, link); +} + +static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) +{ + struct drm_buddy_block *block; + u64 start, size; + + block = amdgpu_vram_mgr_first_block(head); + if (!block) + return false; + + while (head != block->link.next) { + start = amdgpu_vram_mgr_block_start(block); + size = amdgpu_vram_mgr_block_size(block); + + block = list_entry(block->link.next, struct drm_buddy_block, link); + if (start + size != amdgpu_vram_mgr_block_start(block)) + return false; + } + + return true; +} + +static inline u64 amdgpu_vram_mgr_blocks_size(struct list_head *head) +{ + struct drm_buddy_block *block; + u64 size = 0; + + list_for_each_entry(block, head, link) + size += amdgpu_vram_mgr_block_size(block); + + return size; +} + /** - * amdgpu_vram_mgr_init - init VRAM manager and DRM MM + * DOC: mem_info_vram_total * - * @man: TTM memory type manager - * @p_size: maximum size of VRAM + * The amdgpu driver provides a sysfs API for reporting current total VRAM + * available on the device + * The file mem_info_vram_total is used for this and returns the total + * amount of VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + + return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); +} + +/** + * DOC: mem_info_vis_vram_total * - * Allocate and initialize the VRAM manager. + * The amdgpu driver provides a sysfs API for reporting current total + * visible VRAM available on the device + * The file mem_info_vis_vram_total is used for this and returns the total + * amount of visible VRAM in bytes */ -static int amdgpu_vram_mgr_init(struct ttm_mem_type_manager *man, - unsigned long p_size) +static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, + struct device_attribute *attr, char *buf) { - struct amdgpu_vram_mgr *mgr; + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); - mgr = kzalloc(sizeof(*mgr), GFP_KERNEL); - if (!mgr) - return -ENOMEM; + return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); +} - drm_mm_init(&mgr->mm, 0, p_size); - spin_lock_init(&mgr->lock); - man->priv = mgr; - return 0; +/** + * DOC: mem_info_vram_used + * + * The amdgpu driver provides a sysfs API for reporting current total VRAM + * available on the device + * The file mem_info_vram_used is used for this and returns the total + * amount of currently used VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; + + return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man)); } /** - * amdgpu_vram_mgr_fini - free and destroy VRAM manager + * DOC: mem_info_vis_vram_used * - * @man: TTM memory type manager + * The amdgpu driver provides a sysfs API for reporting current total of + * used visible VRAM + * The file mem_info_vis_vram_used is used for this and returns the total + * amount of currently used visible VRAM in bytes + */ +static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + + return sysfs_emit(buf, "%llu\n", + amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr)); +} + +/** + * DOC: mem_info_vram_vendor * - * Destroy and free the VRAM manager, returns -EBUSY if ranges are still - * allocated inside it. + * The amdgpu driver provides a sysfs API for reporting the vendor of the + * installed VRAM + * The file mem_info_vram_vendor is used for this and returns the name of the + * vendor. */ -static int amdgpu_vram_mgr_fini(struct ttm_mem_type_manager *man) +static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, + struct device_attribute *attr, + char *buf) { - struct amdgpu_vram_mgr *mgr = man->priv; + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + + switch (adev->gmc.vram_vendor) { + case SAMSUNG: + return sysfs_emit(buf, "samsung\n"); + case INFINEON: + return sysfs_emit(buf, "infineon\n"); + case ELPIDA: + return sysfs_emit(buf, "elpida\n"); + case ETRON: + return sysfs_emit(buf, "etron\n"); + case NANYA: + return sysfs_emit(buf, "nanya\n"); + case HYNIX: + return sysfs_emit(buf, "hynix\n"); + case MOSEL: + return sysfs_emit(buf, "mosel\n"); + case WINBOND: + return sysfs_emit(buf, "winbond\n"); + case ESMT: + return sysfs_emit(buf, "esmt\n"); + case MICRON: + return sysfs_emit(buf, "micron\n"); + default: + return sysfs_emit(buf, "unknown\n"); + } +} - spin_lock(&mgr->lock); - drm_mm_takedown(&mgr->mm); - spin_unlock(&mgr->lock); - kfree(mgr); - man->priv = NULL; - return 0; +static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, + amdgpu_mem_info_vram_total_show, NULL); +static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, + amdgpu_mem_info_vis_vram_total_show,NULL); +static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, + amdgpu_mem_info_vram_used_show, NULL); +static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, + amdgpu_mem_info_vis_vram_used_show, NULL); +static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, + amdgpu_mem_info_vram_vendor, NULL); + +static struct attribute *amdgpu_vram_mgr_attributes[] = { + &dev_attr_mem_info_vram_total.attr, + &dev_attr_mem_info_vis_vram_total.attr, + &dev_attr_mem_info_vram_used.attr, + &dev_attr_mem_info_vis_vram_used.attr, + &dev_attr_mem_info_vram_vendor.attr, + NULL +}; + +static umode_t amdgpu_vram_attrs_is_visible(struct kobject *kobj, + struct attribute *attr, int i) +{ + struct device *dev = kobj_to_dev(kobj); + struct drm_device *ddev = dev_get_drvdata(dev); + struct amdgpu_device *adev = drm_to_adev(ddev); + + if (attr == &dev_attr_mem_info_vram_vendor.attr && + !adev->gmc.vram_vendor) + return 0; + + if (!ttm_resource_manager_used(&adev->mman.vram_mgr.manager)) + return 0; + + return attr->mode; } +const struct attribute_group amdgpu_vram_mgr_attr_group = { + .attrs = amdgpu_vram_mgr_attributes, + .is_visible = amdgpu_vram_attrs_is_visible +}; + /** - * amdgpu_vram_mgr_vis_size - Calculate visible node size + * amdgpu_vram_mgr_vis_size - Calculate visible block size * - * @adev: amdgpu device structure - * @node: MM node structure + * @adev: amdgpu_device pointer + * @block: DRM BUDDY block structure * - * Calculate how many bytes of the MM node are inside visible VRAM + * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM */ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, - struct drm_mm_node *node) + struct drm_buddy_block *block) { - uint64_t start = node->start << PAGE_SHIFT; - uint64_t end = (node->size + node->start) << PAGE_SHIFT; + u64 start = amdgpu_vram_mgr_block_start(block); + u64 end = start + amdgpu_vram_mgr_block_size(block); if (start >= adev->gmc.visible_vram_size) return 0; @@ -107,43 +277,155 @@ static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); - struct ttm_mem_reg *mem = &bo->tbo.mem; - struct drm_mm_node *nodes = mem->mm_node; - unsigned pages = mem->num_pages; - u64 usage; + struct ttm_resource *res = bo->tbo.resource; + struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); + struct drm_buddy_block *block; + u64 usage = 0; if (amdgpu_gmc_vram_full_visible(&adev->gmc)) return amdgpu_bo_size(bo); - if (mem->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) + if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) return 0; - for (usage = 0; nodes && pages; pages -= nodes->size, nodes++) - usage += amdgpu_vram_mgr_vis_size(adev, nodes); + list_for_each_entry(block, &vres->blocks, link) + usage += amdgpu_vram_mgr_vis_size(adev, block); return usage; } +/* Commit the reservation of VRAM pages */ +static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) +{ + struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); + struct amdgpu_device *adev = to_amdgpu_device(mgr); + struct drm_buddy *mm = &mgr->mm; + struct amdgpu_vram_reservation *rsv, *temp; + struct drm_buddy_block *block; + uint64_t vis_usage; + + list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { + if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, + rsv->size, mm->chunk_size, &rsv->allocated, + DRM_BUDDY_RANGE_ALLOCATION)) + continue; + + block = amdgpu_vram_mgr_first_block(&rsv->allocated); + if (!block) + continue; + + dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", + rsv->start, rsv->size); + + vis_usage = amdgpu_vram_mgr_vis_size(adev, block); + atomic64_add(vis_usage, &mgr->vis_usage); + spin_lock(&man->bdev->lru_lock); + man->usage += rsv->size; + spin_unlock(&man->bdev->lru_lock); + list_move(&rsv->blocks, &mgr->reserved_pages); + } +} + /** - * amdgpu_vram_mgr_virt_start - update virtual start address + * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM * - * @mem: ttm_mem_reg to update - * @node: just allocated node + * @mgr: amdgpu_vram_mgr pointer + * @start: start address of the range in VRAM + * @size: size of the range * - * Calculate a virtual BO start address to easily check if everything is CPU - * accessible. + * Reserve memory from start address with the specified size in VRAM */ -static void amdgpu_vram_mgr_virt_start(struct ttm_mem_reg *mem, - struct drm_mm_node *node) +int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, + uint64_t start, uint64_t size) { - unsigned long start; + struct amdgpu_vram_reservation *rsv; - start = node->start + node->size; - if (start > mem->num_pages) - start -= mem->num_pages; - else - start = 0; - mem->start = max(mem->start, start); + rsv = kzalloc(sizeof(*rsv), GFP_KERNEL); + if (!rsv) + return -ENOMEM; + + INIT_LIST_HEAD(&rsv->allocated); + INIT_LIST_HEAD(&rsv->blocks); + + rsv->start = start; + rsv->size = size; + + mutex_lock(&mgr->lock); + list_add_tail(&rsv->blocks, &mgr->reservations_pending); + amdgpu_vram_mgr_do_reserve(&mgr->manager); + mutex_unlock(&mgr->lock); + + return 0; +} + +/** + * amdgpu_vram_mgr_query_page_status - query the reservation status + * + * @mgr: amdgpu_vram_mgr pointer + * @start: start address of a page in VRAM + * + * Returns: + * -EBUSY: the page is still hold and in pending list + * 0: the page has been reserved + * -ENOENT: the input page is not a reservation + */ +int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, + uint64_t start) +{ + struct amdgpu_vram_reservation *rsv; + int ret; + + mutex_lock(&mgr->lock); + + list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { + if (rsv->start <= start && + (start < (rsv->start + rsv->size))) { + ret = -EBUSY; + goto out; + } + } + + list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { + if (rsv->start <= start && + (start < (rsv->start + rsv->size))) { + ret = 0; + goto out; + } + } + + ret = -ENOENT; +out: + mutex_unlock(&mgr->lock); + return ret; +} + +int amdgpu_vram_mgr_query_address_block_info(struct amdgpu_vram_mgr *mgr, + uint64_t address, struct amdgpu_vram_block_info *info) +{ + struct amdgpu_vram_mgr_resource *vres; + struct drm_buddy_block *block; + u64 start, size; + int ret = -ENOENT; + + mutex_lock(&mgr->lock); + list_for_each_entry(vres, &mgr->allocated_vres_list, vres_node) { + list_for_each_entry(block, &vres->blocks, link) { + start = amdgpu_vram_mgr_block_start(block); + size = amdgpu_vram_mgr_block_size(block); + if ((start <= address) && (address < (start + size))) { + info->start = start; + info->size = size; + memcpy(&info->task, &vres->task, sizeof(vres->task)); + ret = 0; + goto out; + } + } + } + +out: + mutex_unlock(&mgr->lock); + + return ret; } /** @@ -152,173 +434,438 @@ static void amdgpu_vram_mgr_virt_start(struct ttm_mem_reg *mem, * @man: TTM memory type manager * @tbo: TTM BO we need this range for * @place: placement flags and restrictions - * @mem: the resulting mem object + * @res: the resulting mem object * * Allocate VRAM for the given BO. */ -static int amdgpu_vram_mgr_new(struct ttm_mem_type_manager *man, +static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, struct ttm_buffer_object *tbo, const struct ttm_place *place, - struct ttm_mem_reg *mem) + struct ttm_resource **res) { - struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); - struct amdgpu_vram_mgr *mgr = man->priv; - struct drm_mm *mm = &mgr->mm; - struct drm_mm_node *nodes; - enum drm_mm_insert_mode mode; - unsigned long lpfn, num_nodes, pages_per_node, pages_left; - uint64_t usage = 0, vis_usage = 0; - unsigned i; + struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); + struct amdgpu_device *adev = to_amdgpu_device(mgr); + struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); + u64 vis_usage = 0, max_bytes, min_block_size; + struct amdgpu_vram_mgr_resource *vres; + u64 size, remaining_size, lpfn, fpfn; + unsigned int adjust_dcc_size = 0; + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; + unsigned long pages_per_block; int r; - lpfn = place->lpfn; - if (!lpfn) + lpfn = (u64)place->lpfn << PAGE_SHIFT; + if (!lpfn || lpfn > man->size) lpfn = man->size; - if (place->flags & TTM_PL_FLAG_CONTIGUOUS || - amdgpu_vram_page_split == -1) { - pages_per_node = ~0ul; - num_nodes = 1; + fpfn = (u64)place->fpfn << PAGE_SHIFT; + + max_bytes = adev->gmc.mc_vram_size; + if (tbo->type != ttm_bo_type_kernel) + max_bytes -= AMDGPU_VM_RESERVED_VRAM; + + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) { + pages_per_block = ~0ul; } else { - pages_per_node = max((uint32_t)amdgpu_vram_page_split, - mem->page_alignment); - num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pages_per_block = HPAGE_PMD_NR; +#else + /* default to 2MB */ + pages_per_block = 2UL << (20UL - PAGE_SHIFT); +#endif + pages_per_block = max_t(u32, pages_per_block, + tbo->page_alignment); } - nodes = kvmalloc_array(num_nodes, sizeof(*nodes), - GFP_KERNEL | __GFP_ZERO); - if (!nodes) + vres = kzalloc(sizeof(*vres), GFP_KERNEL); + if (!vres) return -ENOMEM; - mode = DRM_MM_INSERT_BEST; + ttm_resource_init(tbo, place, &vres->base); + + /* bail out quickly if there's likely not enough VRAM for this BO */ + if (ttm_resource_manager_usage(man) > max_bytes) { + r = -ENOSPC; + goto error_fini; + } + + INIT_LIST_HEAD(&vres->blocks); + if (place->flags & TTM_PL_FLAG_TOPDOWN) - mode = DRM_MM_INSERT_HIGH; + vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; - mem->start = 0; - pages_left = mem->num_pages; + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) + vres->flags |= DRM_BUDDY_CONTIGUOUS_ALLOCATION; - spin_lock(&mgr->lock); - for (i = 0; pages_left >= pages_per_node; ++i) { - unsigned long pages = rounddown_pow_of_two(pages_left); + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED) + vres->flags |= DRM_BUDDY_CLEAR_ALLOCATION; - r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, - pages_per_node, 0, - place->fpfn, lpfn, - mode); - if (unlikely(r)) - break; + if (fpfn || lpfn != mgr->mm.size) + /* Allocate blocks in desired range */ + vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; - usage += nodes[i].size << PAGE_SHIFT; - vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); - amdgpu_vram_mgr_virt_start(mem, &nodes[i]); - pages_left -= pages; + if (bo->flags & AMDGPU_GEM_CREATE_GFX12_DCC && + adev->gmc.gmc_funcs->get_dcc_alignment) + adjust_dcc_size = amdgpu_gmc_get_dcc_alignment(adev); + + remaining_size = (u64)vres->base.size; + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) { + unsigned int dcc_size; + + dcc_size = roundup_pow_of_two(vres->base.size + adjust_dcc_size); + remaining_size = (u64)dcc_size; + + vres->flags |= DRM_BUDDY_TRIM_DISABLE; } - for (; pages_left; ++i) { - unsigned long pages = min(pages_left, pages_per_node); - uint32_t alignment = mem->page_alignment; + mutex_lock(&mgr->lock); + while (remaining_size) { + if (tbo->page_alignment) + min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT; + else + min_block_size = mgr->default_page_size; + + size = remaining_size; + + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) + min_block_size = size; + else if ((size >= (u64)pages_per_block << PAGE_SHIFT) && + !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1))) + min_block_size = (u64)pages_per_block << PAGE_SHIFT; + + BUG_ON(min_block_size < mm->chunk_size); + + r = drm_buddy_alloc_blocks(mm, fpfn, + lpfn, + size, + min_block_size, + &vres->blocks, + vres->flags); - if (pages == pages_per_node) - alignment = pages_per_node; + if (unlikely(r == -ENOSPC) && pages_per_block == ~0ul && + !(place->flags & TTM_PL_FLAG_CONTIGUOUS)) { + vres->flags &= ~DRM_BUDDY_CONTIGUOUS_ALLOCATION; + pages_per_block = max_t(u32, 2UL << (20UL - PAGE_SHIFT), + tbo->page_alignment); + + continue; + } - r = drm_mm_insert_node_in_range(mm, &nodes[i], - pages, alignment, 0, - place->fpfn, lpfn, - mode); if (unlikely(r)) - goto error; + goto error_free_blocks; - usage += nodes[i].size << PAGE_SHIFT; - vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]); - amdgpu_vram_mgr_virt_start(mem, &nodes[i]); - pages_left -= pages; + if (size > remaining_size) + remaining_size = 0; + else + remaining_size -= size; } - spin_unlock(&mgr->lock); - atomic64_add(usage, &mgr->usage); - atomic64_add(vis_usage, &mgr->vis_usage); + vres->task.pid = task_pid_nr(current); + get_task_comm(vres->task.comm, current); + list_add_tail(&vres->vres_node, &mgr->allocated_vres_list); + + if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && adjust_dcc_size) { + struct drm_buddy_block *dcc_block; + unsigned long dcc_start; + u64 trim_start; + + dcc_block = amdgpu_vram_mgr_first_block(&vres->blocks); + /* Adjust the start address for DCC buffers only */ + dcc_start = + roundup((unsigned long)amdgpu_vram_mgr_block_start(dcc_block), + adjust_dcc_size); + trim_start = (u64)dcc_start; + drm_buddy_block_trim(mm, &trim_start, + (u64)vres->base.size, + &vres->blocks); + } + mutex_unlock(&mgr->lock); + + vres->base.start = 0; + size = max_t(u64, amdgpu_vram_mgr_blocks_size(&vres->blocks), + vres->base.size); + list_for_each_entry(block, &vres->blocks, link) { + unsigned long start; + + start = amdgpu_vram_mgr_block_start(block) + + amdgpu_vram_mgr_block_size(block); + start >>= PAGE_SHIFT; + + if (start > PFN_UP(size)) + start -= PFN_UP(size); + else + start = 0; + vres->base.start = max(vres->base.start, start); - mem->mm_node = nodes; + vis_usage += amdgpu_vram_mgr_vis_size(adev, block); + } + + if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) + vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; + + if (adev->gmc.xgmi.connected_to_cpu) + vres->base.bus.caching = ttm_cached; + else + vres->base.bus.caching = ttm_write_combined; + atomic64_add(vis_usage, &mgr->vis_usage); + *res = &vres->base; return 0; -error: - while (i--) - drm_mm_remove_node(&nodes[i]); - spin_unlock(&mgr->lock); +error_free_blocks: + drm_buddy_free_list(mm, &vres->blocks, 0); + mutex_unlock(&mgr->lock); +error_fini: + ttm_resource_fini(man, &vres->base); + kfree(vres); - kvfree(nodes); - return r == -ENOSPC ? 0 : r; + return r; } /** * amdgpu_vram_mgr_del - free ranges * * @man: TTM memory type manager - * @tbo: TTM BO we need this range for - * @place: placement flags and restrictions - * @mem: TTM memory object + * @res: TTM memory object * * Free the allocated VRAM again. */ -static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man, - struct ttm_mem_reg *mem) +static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, + struct ttm_resource *res) { - struct amdgpu_device *adev = amdgpu_ttm_adev(man->bdev); - struct amdgpu_vram_mgr *mgr = man->priv; - struct drm_mm_node *nodes = mem->mm_node; - uint64_t usage = 0, vis_usage = 0; - unsigned pages = mem->num_pages; + struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); + struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); + struct amdgpu_device *adev = to_amdgpu_device(mgr); + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; + uint64_t vis_usage = 0; - if (!mem->mm_node) - return; + mutex_lock(&mgr->lock); - spin_lock(&mgr->lock); - while (pages) { - pages -= nodes->size; - drm_mm_remove_node(nodes); - usage += nodes->size << PAGE_SHIFT; - vis_usage += amdgpu_vram_mgr_vis_size(adev, nodes); - ++nodes; - } - spin_unlock(&mgr->lock); + list_del(&vres->vres_node); + memset(&vres->task, 0, sizeof(vres->task)); + + list_for_each_entry(block, &vres->blocks, link) + vis_usage += amdgpu_vram_mgr_vis_size(adev, block); + + drm_buddy_free_list(mm, &vres->blocks, vres->flags); + amdgpu_vram_mgr_do_reserve(man); + mutex_unlock(&mgr->lock); - atomic64_sub(usage, &mgr->usage); atomic64_sub(vis_usage, &mgr->vis_usage); - kvfree(mem->mm_node); - mem->mm_node = NULL; + ttm_resource_fini(man, res); + kfree(vres); } /** - * amdgpu_vram_mgr_usage - how many bytes are used in this domain + * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table * - * @man: TTM memory type manager + * @adev: amdgpu device pointer + * @res: TTM memory object + * @offset: byte offset from the base of VRAM BO + * @length: number of bytes to export in sg_table + * @dev: the other device + * @dir: dma direction + * @sgt: resulting sg table * - * Returns how many bytes are used in this domain. + * Allocate and fill a sg table from a VRAM allocation. */ -uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man) +int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, + struct ttm_resource *res, + u64 offset, u64 length, + struct device *dev, + enum dma_data_direction dir, + struct sg_table **sgt) { - struct amdgpu_vram_mgr *mgr = man->priv; + struct amdgpu_res_cursor cursor; + struct scatterlist *sg; + int num_entries = 0; + int i, r; - return atomic64_read(&mgr->usage); + *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); + if (!*sgt) + return -ENOMEM; + + /* Determine the number of DRM_BUDDY blocks to export */ + amdgpu_res_first(res, offset, length, &cursor); + while (cursor.remaining) { + num_entries++; + amdgpu_res_next(&cursor, min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE)); + } + + r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); + if (r) + goto error_free; + + /* Initialize scatterlist nodes of sg_table */ + for_each_sgtable_sg((*sgt), sg, i) + sg->length = 0; + + /* + * Walk down DRM_BUDDY blocks to populate scatterlist nodes + * @note: Use iterator api to get first the DRM_BUDDY block + * and the number of bytes from it. Access the following + * DRM_BUDDY block(s) if more buffer needs to exported + */ + amdgpu_res_first(res, offset, length, &cursor); + for_each_sgtable_sg((*sgt), sg, i) { + phys_addr_t phys = cursor.start + adev->gmc.aper_base; + unsigned long size = min(cursor.size, AMDGPU_MAX_SG_SEGMENT_SIZE); + dma_addr_t addr; + + addr = dma_map_resource(dev, phys, size, dir, + DMA_ATTR_SKIP_CPU_SYNC); + r = dma_mapping_error(dev, addr); + if (r) + goto error_unmap; + + sg_set_page(sg, NULL, size, 0); + sg_dma_address(sg) = addr; + sg_dma_len(sg) = size; + + amdgpu_res_next(&cursor, size); + } + + return 0; + +error_unmap: + for_each_sgtable_sg((*sgt), sg, i) { + if (!sg->length) + continue; + + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + } + sg_free_table(*sgt); + +error_free: + kfree(*sgt); + return r; +} + +/** + * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table + * + * @dev: device pointer + * @dir: data direction of resource to unmap + * @sgt: sg table to free + * + * Free a previously allocate sg table. + */ +void amdgpu_vram_mgr_free_sgt(struct device *dev, + enum dma_data_direction dir, + struct sg_table *sgt) +{ + struct scatterlist *sg; + int i; + + for_each_sgtable_sg(sgt, sg, i) + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + sg_free_table(sgt); + kfree(sgt); } /** * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part * - * @man: TTM memory type manager + * @mgr: amdgpu_vram_mgr pointer * * Returns how many bytes are used in the visible part of VRAM */ -uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man) +uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) { - struct amdgpu_vram_mgr *mgr = man->priv; - return atomic64_read(&mgr->vis_usage); } /** + * amdgpu_vram_mgr_clear_reset_blocks - reset clear blocks + * + * @adev: amdgpu device pointer + * + * Reset the cleared drm buddy blocks. + */ +void amdgpu_vram_mgr_clear_reset_blocks(struct amdgpu_device *adev) +{ + struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; + struct drm_buddy *mm = &mgr->mm; + + mutex_lock(&mgr->lock); + drm_buddy_reset_clear(mm, false); + mutex_unlock(&mgr->lock); +} + +/** + * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection + * + * @man: TTM memory type manager + * @res: The resource to test + * @place: The place to test against + * @size: Size of the new allocation + * + * Test each drm buddy block for intersection for eviction decision. + */ +static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); + struct drm_buddy_block *block; + + /* Check each drm buddy block individually */ + list_for_each_entry(block, &mgr->blocks, link) { + unsigned long fpfn = + amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; + unsigned long lpfn = fpfn + + (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); + + if (place->fpfn < lpfn && + (!place->lpfn || place->lpfn > fpfn)) + return true; + } + + return false; +} + +/** + * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility + * + * @man: TTM memory type manager + * @res: The resource to test + * @place: The place to test against + * @size: Size of the new allocation + * + * Test each drm buddy block for placement compatibility. + */ +static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, + struct ttm_resource *res, + const struct ttm_place *place, + size_t size) +{ + struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); + struct drm_buddy_block *block; + + /* Check each drm buddy block individually */ + list_for_each_entry(block, &mgr->blocks, link) { + unsigned long fpfn = + amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; + unsigned long lpfn = fpfn + + (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); + + if (fpfn < place->fpfn || + (place->lpfn && lpfn > place->lpfn)) + return false; + } + + return true; +} + +/** * amdgpu_vram_mgr_debug - dump VRAM table * * @man: TTM memory type manager @@ -326,24 +873,105 @@ uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man) * * Dump the table content using printk. */ -static void amdgpu_vram_mgr_debug(struct ttm_mem_type_manager *man, +static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, struct drm_printer *printer) { - struct amdgpu_vram_mgr *mgr = man->priv; + struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); + struct drm_buddy *mm = &mgr->mm; + struct amdgpu_vram_reservation *rsv; + + drm_printf(printer, " vis usage:%llu\n", + amdgpu_vram_mgr_vis_usage(mgr)); - spin_lock(&mgr->lock); - drm_mm_print(&mgr->mm, printer); - spin_unlock(&mgr->lock); + mutex_lock(&mgr->lock); + drm_printf(printer, "default_page_size: %lluKiB\n", + mgr->default_page_size >> 10); - drm_printf(printer, "man size:%llu pages, ram usage:%lluMB, vis usage:%lluMB\n", - man->size, amdgpu_vram_mgr_usage(man) >> 20, - amdgpu_vram_mgr_vis_usage(man) >> 20); + drm_buddy_print(mm, printer); + + drm_printf(printer, "reserved:\n"); + list_for_each_entry(rsv, &mgr->reserved_pages, blocks) + drm_printf(printer, "%#018llx-%#018llx: %llu\n", + rsv->start, rsv->start + rsv->size, rsv->size); + mutex_unlock(&mgr->lock); } -const struct ttm_mem_type_manager_func amdgpu_vram_mgr_func = { - .init = amdgpu_vram_mgr_init, - .takedown = amdgpu_vram_mgr_fini, - .get_node = amdgpu_vram_mgr_new, - .put_node = amdgpu_vram_mgr_del, - .debug = amdgpu_vram_mgr_debug +static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { + .alloc = amdgpu_vram_mgr_new, + .free = amdgpu_vram_mgr_del, + .intersects = amdgpu_vram_mgr_intersects, + .compatible = amdgpu_vram_mgr_compatible, + .debug = amdgpu_vram_mgr_debug }; + +/** + * amdgpu_vram_mgr_init - init VRAM manager and DRM MM + * + * @adev: amdgpu_device pointer + * + * Allocate and initialize the VRAM manager. + */ +int amdgpu_vram_mgr_init(struct amdgpu_device *adev) +{ + struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; + struct ttm_resource_manager *man = &mgr->manager; + int err; + + man->cg = drmm_cgroup_register_region(adev_to_drm(adev), "vram", adev->gmc.real_vram_size); + if (IS_ERR(man->cg)) + return PTR_ERR(man->cg); + ttm_resource_manager_init(man, &adev->mman.bdev, + adev->gmc.real_vram_size); + + mutex_init(&mgr->lock); + INIT_LIST_HEAD(&mgr->reservations_pending); + INIT_LIST_HEAD(&mgr->reserved_pages); + INIT_LIST_HEAD(&mgr->allocated_vres_list); + mgr->default_page_size = PAGE_SIZE; + + man->func = &amdgpu_vram_mgr_func; + err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); + if (err) + return err; + + ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); + ttm_resource_manager_set_used(man, true); + return 0; +} + +/** + * amdgpu_vram_mgr_fini - free and destroy VRAM manager + * + * @adev: amdgpu_device pointer + * + * Destroy and free the VRAM manager, returns -EBUSY if ranges are still + * allocated inside it. + */ +void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) +{ + struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; + struct ttm_resource_manager *man = &mgr->manager; + int ret; + struct amdgpu_vram_reservation *rsv, *temp; + + ttm_resource_manager_set_used(man, false); + + ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); + if (ret) + return; + + mutex_lock(&mgr->lock); + list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) + kfree(rsv); + + list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { + drm_buddy_free_list(&mgr->mm, &rsv->allocated, 0); + kfree(rsv); + } + if (!adev->gmc.is_app_apu) + drm_buddy_fini(&mgr->mm); + mutex_unlock(&mgr->lock); + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); +} |