diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 2307 |
1 files changed, 860 insertions, 1447 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index f73b81c2576e..bd27607f8076 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -31,9 +31,13 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_module.h> -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/drm_print.h> +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -41,328 +45,133 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> +#include <linux/cgroup_dmem.h> #include <linux/dma-resv.h> -static void ttm_bo_global_kobj_release(struct kobject *kobj); - -/** - * ttm_global_mutex - protecting the global BO state - */ -DEFINE_MUTEX(ttm_global_mutex); -unsigned ttm_bo_glob_use_count; -struct ttm_bo_global ttm_bo_glob; -EXPORT_SYMBOL(ttm_bo_glob); - -static struct attribute ttm_bo_count = { - .name = "bo_count", - .mode = S_IRUGO -}; - -/* default destructor */ -static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) -{ - kfree(bo); -} - -static inline int ttm_mem_type_from_place(const struct ttm_place *place, - uint32_t *mem_type) -{ - int pos; - - pos = ffs(place->flags & TTM_PL_MASK_MEM); - if (unlikely(!pos)) - return -EINVAL; - - *mem_type = pos - 1; - return 0; -} - -static void ttm_mem_type_debug(struct ttm_bo_device *bdev, struct drm_printer *p, - int mem_type) -{ - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - - drm_printf(p, " has_type: %d\n", man->has_type); - drm_printf(p, " use_type: %d\n", man->use_type); - drm_printf(p, " flags: 0x%08X\n", man->flags); - drm_printf(p, " gpu_offset: 0x%08llX\n", man->gpu_offset); - drm_printf(p, " size: %llu\n", man->size); - drm_printf(p, " available_caching: 0x%08X\n", man->available_caching); - drm_printf(p, " default_caching: 0x%08X\n", man->default_caching); - if (mem_type != TTM_PL_SYSTEM) - (*man->func->debug)(man, p); -} +#include "ttm_module.h" +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) { - struct drm_printer p = drm_debug_printer(TTM_PFX); - int i, ret, mem_type; + struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); + struct ttm_resource_manager *man; + int i, mem_type; - drm_printf(&p, "No space for %p (%lu pages, %luK, %luM)\n", - bo, bo->mem.num_pages, bo->mem.size >> 10, - bo->mem.size >> 20); for (i = 0; i < placement->num_placement; i++) { - ret = ttm_mem_type_from_place(&placement->placement[i], - &mem_type); - if (ret) - return; + mem_type = placement->placement[i].mem_type; drm_printf(&p, " placement[%d]=0x%08X (%d)\n", i, placement->placement[i].flags, mem_type); - ttm_mem_type_debug(bo->bdev, &p, mem_type); - } -} - -static ssize_t ttm_bo_global_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - return snprintf(buffer, PAGE_SIZE, "%d\n", - atomic_read(&glob->bo_count)); -} - -static struct attribute *ttm_bo_global_attrs[] = { - &ttm_bo_count, - NULL -}; - -static const struct sysfs_ops ttm_bo_global_ops = { - .show = &ttm_bo_global_show -}; - -static struct kobj_type ttm_bo_glob_kobj_type = { - .release = &ttm_bo_global_kobj_release, - .sysfs_ops = &ttm_bo_global_ops, - .default_attrs = ttm_bo_global_attrs -}; - - -static inline uint32_t ttm_bo_type_flags(unsigned type) -{ - return 1 << (type); -} - -static void ttm_bo_add_mem_to_lru(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man; - - if (!list_empty(&bo->lru)) - return; - - if (mem->placement & TTM_PL_FLAG_NO_EVICT) - return; - - man = &bdev->man[mem->mem_type]; - list_add_tail(&bo->lru, &man->lru[bo->priority]); - - if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm && - !(bo->ttm->page_flags & (TTM_PAGE_FLAG_SG | - TTM_PAGE_FLAG_SWAPPED))) { - list_add_tail(&bo->swap, &ttm_bo_glob.swap_lru[bo->priority]); + man = ttm_manager_type(bo->bdev, mem_type); + ttm_resource_manager_debug(man, &p); } } -static void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - bool notify = false; - - if (!list_empty(&bo->swap)) { - list_del_init(&bo->swap); - notify = true; - } - if (!list_empty(&bo->lru)) { - list_del_init(&bo->lru); - notify = true; - } - - if (notify && bdev->driver->del_from_lru_notify) - bdev->driver->del_from_lru_notify(bo); -} - -static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, - struct ttm_buffer_object *bo) -{ - if (!pos->first) - pos->first = bo; - pos->last = bo; -} - -void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, - struct ttm_lru_bulk_move *bulk) +/** + * ttm_bo_move_to_lru_tail + * + * @bo: The buffer object. + * + * Move this BO to the tail of all lru lists used to lookup and reserve an + * object. This function must be called with struct ttm_global::lru_lock + * held, and is used to make a BO less likely to be considered for eviction. + */ +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { dma_resv_assert_held(bo->base.resv); - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, &bo->mem); - - if (bulk && !(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { - switch (bo->mem.mem_type) { - case TTM_PL_TT: - ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); - break; - - case TTM_PL_VRAM: - ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); - break; - } - if (bo->ttm && !(bo->ttm->page_flags & - (TTM_PAGE_FLAG_SG | TTM_PAGE_FLAG_SWAPPED))) - ttm_bo_bulk_move_set_pos(&bulk->swap[bo->priority], bo); - } + if (bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); -void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) +/** + * ttm_bo_set_bulk_move - update BOs bulk move object + * + * @bo: The buffer object. + * @bulk: bulk move structure + * + * Update the BOs bulk move object, making sure that resources are added/removed + * as well. A bulk move allows to move many resource on the LRU at once, + * resulting in much less overhead of maintaining the LRU. + * The only requirement is that the resources stay together on the LRU and are + * never separated. This is enforces by setting the bulk_move structure on a BO. + * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of + * their LRU list. + */ +void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) { - unsigned i; - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; - struct ttm_mem_type_manager *man; - - if (!pos->first) - continue; - - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); - - man = &pos->first->bdev->man[TTM_PL_TT]; - list_bulk_move_tail(&man->lru[i], &pos->first->lru, - &pos->last->lru); - } - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; - struct ttm_mem_type_manager *man; - - if (!pos->first) - continue; - - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); - - man = &pos->first->bdev->man[TTM_PL_VRAM]; - list_bulk_move_tail(&man->lru[i], &pos->first->lru, - &pos->last->lru); - } - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->swap[i]; - struct list_head *lru; - - if (!pos->first) - continue; + dma_resv_assert_held(bo->base.resv); - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); + if (bo->bulk_move == bulk) + return; - lru = &ttm_bo_glob.swap_lru[i]; - list_bulk_move_tail(lru, &pos->first->swap, &pos->last->swap); - } + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + bo->bulk_move = bulk; + if (bo->resource) + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); } -EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); +EXPORT_SYMBOL(ttm_bo_set_bulk_move); static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem, bool evict, - struct ttm_operation_ctx *ctx) + struct ttm_resource *mem, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) { - struct ttm_bo_device *bdev = bo->bdev; - bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); - bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); - struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; - struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; - int ret = 0; + struct ttm_device *bdev = bo->bdev; + bool old_use_tt, new_use_tt; + int ret; - if (old_is_pci || new_is_pci || - ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { - ret = ttm_mem_io_lock(old_man, true); - if (unlikely(ret != 0)) - goto out_err; - ttm_bo_unmap_virtual_locked(bo); - ttm_mem_io_unlock(old_man); - } + old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; + new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; + + ttm_bo_unmap_virtual(bo); /* * Create and bind a ttm if required. */ - if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (bo->ttm == NULL) { - bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); - ret = ttm_tt_create(bo, zero); - if (ret) - goto out_err; - } - - ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); + if (new_use_tt) { + /* Zero init the new TTM structure if the old location should + * have used one as well. + */ + ret = ttm_tt_create(bo, old_use_tt); if (ret) goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(bo->ttm, mem, ctx); + ret = ttm_bo_populate(bo, ctx); if (ret) goto out_err; } - - if (bo->mem.mem_type == TTM_PL_SYSTEM) { - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, evict, mem); - bo->mem = *mem; - mem->mm_node = NULL; - goto moved; - } } - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, evict, mem); - - if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && - !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, ctx, mem); - else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, ctx, mem); - else - ret = ttm_bo_move_memcpy(bo, ctx, mem); + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto out_err; + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); if (ret) { - if (bdev->driver->move_notify) { - swap(*mem, bo->mem); - bdev->driver->move_notify(bo, false, mem); - swap(*mem, bo->mem); - } - + if (ret == -EMULTIHOP) + return ret; goto out_err; } -moved: - bo->evicted = false; - - if (bo->mem.mm_node) - bo->offset = (bo->mem.start << PAGE_SHIFT) + - bdev->man[bo->mem.mem_type].gpu_offset; - else - bo->offset = 0; - - ctx->bytes_moved += bo->num_pages << PAGE_SHIFT; + ctx->bytes_moved += bo->base.size; return 0; out_err: - new_man = &bdev->man[bo->mem.mem_type]; - if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) { - ttm_tt_destroy(bo->ttm); - bo->ttm = NULL; - } + if (!old_use_tt) + ttm_bo_tt_destroy(bo); return ret; } -/** +/* * Call bo::reserved. * Will release GPU memory type usage on destruction. * This is the place to put in driver specific hooks to release @@ -372,12 +181,11 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { - if (bo->bdev->driver->move_notify) - bo->bdev->driver->move_notify(bo, false, NULL); + if (bo->bdev->funcs->delete_mem_notify) + bo->bdev->funcs->delete_mem_notify(bo); - ttm_tt_destroy(bo->ttm); - bo->ttm = NULL; - ttm_bo_mem_put(bo, &bo->mem); + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); } static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) @@ -399,9 +207,9 @@ static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) * reference it any more. The only tricky case is the trylock on * the resv object while holding the lru_lock. */ - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); bo->base.resv = &bo->base._resv; - spin_unlock(&ttm_bo_glob.lru_lock); + spin_unlock(&bo->bdev->lru_lock); } return r; @@ -410,276 +218,172 @@ static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) { struct dma_resv *resv = &bo->base._resv; - struct dma_resv_list *fobj; + struct dma_resv_iter cursor; struct dma_fence *fence; - int i; - - rcu_read_lock(); - fobj = rcu_dereference(resv->fence); - fence = rcu_dereference(resv->fence_excl); - if (fence && !fence->ops->signaled) - dma_fence_enable_sw_signaling(fence); - - for (i = 0; fobj && i < fobj->shared_count; ++i) { - fence = rcu_dereference(fobj->shared[i]); + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) { if (!fence->ops->signaled) dma_fence_enable_sw_signaling(fence); } - rcu_read_unlock(); + dma_resv_iter_end(&cursor); } -/** - * function ttm_bo_cleanup_refs - * If bo idle, remove from lru lists, and unref. - * If not idle, block if possible. - * - * Must be called with lru_lock and reservation held, this function - * will drop the lru lock and optionally the reservation lock before returning. - * - * @interruptible Any sleeps should occur interruptibly. - * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. - * @unlock_resv Unlock the reservation lock as well. +/* + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. */ - -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, - bool unlock_resv) +static void ttm_bo_delayed_delete(struct work_struct *work) { - struct dma_resv *resv = &bo->base._resv; - int ret; - - if (dma_resv_test_signaled_rcu(resv, true)) - ret = 0; - else - ret = -EBUSY; - - if (ret && !no_wait_gpu) { - long lret; - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&ttm_bo_glob.lru_lock); - - lret = dma_resv_wait_timeout_rcu(resv, true, interruptible, - 30 * HZ); - - if (lret < 0) - return lret; - else if (lret == 0) - return -EBUSY; - - spin_lock(&ttm_bo_glob.lru_lock); - if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - spin_unlock(&ttm_bo_glob.lru_lock); - return 0; - } - ret = 0; - } + struct ttm_buffer_object *bo; - if (ret || unlikely(list_empty(&bo->ddestroy))) { - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&ttm_bo_glob.lru_lock); - return ret; - } + bo = container_of(work, typeof(*bo), delayed_delete); - ttm_bo_del_from_lru(bo); - list_del_init(&bo->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); ttm_bo_cleanup_memtype_use(bo); - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - + dma_resv_unlock(bo->base.resv); ttm_bo_put(bo); - - return 0; -} - -/** - * Traverse the delayed list, and call ttm_bo_cleanup_refs on all - * encountered buffers. - */ -static bool ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - struct list_head removed; - bool empty; - - INIT_LIST_HEAD(&removed); - - spin_lock(&glob->lru_lock); - while (!list_empty(&bdev->ddestroy)) { - struct ttm_buffer_object *bo; - - bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, - ddestroy); - list_move_tail(&bo->ddestroy, &removed); - if (!ttm_bo_get_unless_zero(bo)) - continue; - - if (remove_all || bo->base.resv != &bo->base._resv) { - spin_unlock(&glob->lru_lock); - dma_resv_lock(bo->base.resv, NULL); - - spin_lock(&glob->lru_lock); - ttm_bo_cleanup_refs(bo, false, !remove_all, true); - - } else if (dma_resv_trylock(bo->base.resv)) { - ttm_bo_cleanup_refs(bo, false, !remove_all, true); - } else { - spin_unlock(&glob->lru_lock); - } - - ttm_bo_put(bo); - spin_lock(&glob->lru_lock); - } - list_splice_tail(&removed, &bdev->ddestroy); - empty = list_empty(&bdev->ddestroy); - spin_unlock(&glob->lru_lock); - - return empty; -} - -static void ttm_bo_delayed_workqueue(struct work_struct *work) -{ - struct ttm_bo_device *bdev = - container_of(work, struct ttm_bo_device, wq.work); - - if (!ttm_bo_delayed_delete(bdev, false)) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); } static void ttm_bo_release(struct kref *kref) { struct ttm_buffer_object *bo = container_of(kref, struct ttm_buffer_object, kref); - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - size_t acc_size = bo->acc_size; + struct ttm_device *bdev = bo->bdev; int ret; + WARN_ON_ONCE(bo->pin_count); + WARN_ON_ONCE(bo->bulk_move); + if (!bo->deleted) { ret = ttm_bo_individualize_resv(bo); if (ret) { /* Last resort, if we fail to allocate memory for the * fences block for the BO to become idle */ - dma_resv_wait_timeout_rcu(bo->base.resv, true, false, - 30 * HZ); + dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, false, + 30 * HZ); } - if (bo->bdev->driver->release_notify) - bo->bdev->driver->release_notify(bo); + if (bo->bdev->funcs->release_notify) + bo->bdev->funcs->release_notify(bo); drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); - ttm_mem_io_lock(man, false); - ttm_mem_io_free_vm(bo); - ttm_mem_io_unlock(man); - } + ttm_mem_io_free(bdev, bo->resource); - if (!dma_resv_test_signaled_rcu(bo->base.resv, true) || - !dma_resv_trylock(bo->base.resv)) { - /* The BO is not idle, resurrect it for delayed destroy */ - ttm_bo_flush_all_fences(bo); - bo->deleted = true; + if (!dma_resv_test_signaled(&bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP) || + (want_init_on_free() && (bo->ttm != NULL)) || + bo->type == ttm_bo_type_sg || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; - spin_lock(&ttm_bo_glob.lru_lock); + spin_lock(&bo->bdev->lru_lock); - /* - * Make NO_EVICT bos immediately available to - * shrinkers, now that they are queued for - * destruction. - */ - if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) { - bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT; - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, &bo->mem); - } + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } - kref_init(&bo->kref); - list_add_tail(&bo->ddestroy, &bdev->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + kref_init(&bo->kref); + spin_unlock(&bo->bdev->lru_lock); - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); - return; - } + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); - spin_lock(&ttm_bo_glob.lru_lock); - ttm_bo_del_from_lru(bo); - list_del(&bo->ddestroy); - spin_unlock(&ttm_bo_glob.lru_lock); + /* Schedule the worker on the closest NUMA node. This + * improves performance since system memory might be + * cleared on free and that is best done on a CPU core + * close to it. + */ + queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); + return; + } - ttm_bo_cleanup_memtype_use(bo); - dma_resv_unlock(bo->base.resv); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + } - BUG_ON(bo->mem.mm_node != NULL); - atomic_dec(&ttm_bo_glob.bo_count); - dma_fence_put(bo->moving); - if (!ttm_bo_uses_embedded_gem_object(bo)) - dma_resv_fini(&bo->base._resv); + atomic_dec(&ttm_glob.bo_count); bo->destroy(bo); - ttm_mem_global_free(&ttm_mem_glob, acc_size); } +/* TODO: remove! */ void ttm_bo_put(struct ttm_buffer_object *bo) { kref_put(&bo->kref, ttm_bo_release); } -EXPORT_SYMBOL(ttm_bo_put); -int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) +void ttm_bo_fini(struct ttm_buffer_object *bo) { - return cancel_delayed_work_sync(&bdev->wq); + ttm_bo_put(bo); } -EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); +EXPORT_SYMBOL(ttm_bo_fini); -void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) { - if (resched) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); + struct ttm_placement hop_placement; + struct ttm_resource *hop_mem; + int ret; + + hop_placement.num_placement = 1; + hop_placement.placement = hop; + + /* find space in the bounce domain */ + ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); + if (ret) + return ret; + /* move to the bounce domain */ + ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); + if (ret) { + ttm_resource_free(bo, &hop_mem); + return ret; + } + return 0; } -EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); static int ttm_bo_evict(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_reg evict_mem; + struct ttm_device *bdev = bo->bdev; + struct ttm_resource *evict_mem; struct ttm_placement placement; + struct ttm_place hop; int ret = 0; + memset(&hop, 0, sizeof(hop)); + dma_resv_assert_held(bo->base.resv); placement.num_placement = 0; - placement.num_busy_placement = 0; - bdev->driver->evict_flags(bo, &placement); + bdev->funcs->evict_flags(bo, &placement); - if (!placement.num_placement && !placement.num_busy_placement) { - ret = ttm_bo_pipeline_gutting(bo); + if (!placement.num_placement) { + ret = ttm_bo_wait_ctx(bo, ctx); if (ret) return ret; - return ttm_tt_create(bo, false); + /* + * Since we've already synced, this frees backing store + * immediately. + */ + return ttm_bo_pipeline_gutting(bo); } - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.bus.io_reserved_vm = false; - evict_mem.bus.io_reserved_count = 0; - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { @@ -690,544 +394,499 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, ctx); - if (unlikely(ret)) { - if (ret != -ERESTARTSYS) + do { + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret != -EMULTIHOP) + break; + + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + } while (!ret); + + if (ret) { + ttm_resource_free(bo, &evict_mem); + if (ret != -ERESTARTSYS && ret != -EINTR) pr_err("Buffer eviction failed\n"); - ttm_bo_mem_put(bo, &evict_mem); - goto out; } - bo->evicted = true; out: return ret; } +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { + struct ttm_resource *res = bo->resource; + struct ttm_device *bdev = bo->bdev; + + dma_resv_assert_held(bo->base.resv); + if (bo->resource->mem_type == TTM_PL_SYSTEM) + return true; + /* Don't evict this BO if it's outside of the * requested placement range */ - if (place->fpfn >= (bo->mem.start + bo->mem.size) || - (place->lpfn && place->lpfn <= bo->mem.start)) - return false; - - return true; + return ttm_resource_intersects(bdev, res, place, bo->base.size); } EXPORT_SYMBOL(ttm_bo_eviction_valuable); /** - * Check the target bo is allowable to be evicted or swapout, including cases: - * - * a. if share same reservation object with ctx->resv, have assumption - * reservation objects should already be locked, so not lock again and - * return true directly when either the opreation allow_reserved_eviction - * or the target bo already is in delayed free list; + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. * - * b. Otherwise, trylock it. + * Return: 0 if successful or the resource disappeared. Negative error code on error. */ -static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, - struct ttm_operation_ctx *ctx, bool *locked, bool *busy) +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) { - bool ret = false; - - if (bo->base.resv == ctx->resv) { - dma_resv_assert_held(bo->base.resv); - if (ctx->flags & TTM_OPT_FLAG_ALLOW_RES_EVICT) - ret = true; - *locked = false; - if (busy) - *busy = false; + struct ttm_resource_cursor cursor; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; + + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); + ttm_resource_cursor_fini(&cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; + } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (!bo->resource || bo->resource->mem_type != mem_type) + goto out_bo_moved; + + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); } else { - ret = dma_resv_trylock(bo->base.resv); - *locked = ret; - if (busy) - *busy = !ret; + ret = ttm_bo_evict(bo, ctx); } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + return ret; +out_no_ref: + spin_unlock(&bdev->lru_lock); return ret; } /** - * ttm_mem_evict_wait_busy - wait for a busy BO to become available - * - * @busy_bo: BO which couldn't be locked with trylock - * @ctx: operation context - * @ticket: acquire ticket - * - * Try to lock a busy buffer object to avoid failing eviction. + * struct ttm_bo_evict_walk - Parameters for the evict walk. */ -static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) -{ - int r; - - if (!busy_bo || !ticket) - return -EBUSY; - - if (ctx->interruptible) - r = dma_resv_lock_interruptible(busy_bo->base.resv, - ticket); - else - r = dma_resv_lock(busy_bo->base.resv, ticket); - - /* - * TODO: It would be better to keep the BO locked until allocation is at - * least tried one more time, but that would mean a much larger rework - * of TTM. - */ - if (!r) - dma_resv_unlock(busy_bo->base.resv); - - return r == -EDEADLK ? -EBUSY : r; -} +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; + + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ + bool try_low; + /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ + bool hit_low; +}; -static int ttm_mem_evict_first(struct ttm_bo_device *bdev, - uint32_t mem_type, - const struct ttm_place *place, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) +static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - bool locked = false; - unsigned i; - int ret; - - spin_lock(&ttm_bo_glob.lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &man->lru[i], lru) { - bool busy; - - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, - &busy)) { - if (busy && !busy_bo && ticket != - dma_resv_locking_ctx(bo->base.resv)) - busy_bo = bo; - continue; - } + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + s64 lret; - if (place && !bdev->driver->eviction_valuable(bo, - place)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; - } - if (!ttm_bo_get_unless_zero(bo)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; - } - break; - } - - /* If the inner loop terminated early, we have our candidate */ - if (&bo->lru != &man->lru[i]) - break; - - bo = NULL; - } + if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css, + evict_walk->try_low, &evict_walk->hit_low)) + return 0; - if (!bo) { - if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) - busy_bo = NULL; - spin_unlock(&ttm_bo_glob.lru_lock); - ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); - if (busy_bo) - ttm_bo_put(busy_bo); - return ret; - } + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, - ctx->no_wait_gpu, locked); - ttm_bo_put(bo); - return ret; + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->arg.ctx); } - spin_unlock(&ttm_bo_glob.lru_lock); + if (lret) + goto out; - ret = ttm_bo_evict(bo, ctx); - if (locked) - ttm_bo_unreserve(bo); + evict_walk->evicted++; + if (evict_walk->res) + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, + evict_walk->res, NULL); + if (lret == 0) + return 1; +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; - ttm_bo_put(bo); - return ret; + return lret; } -void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) -{ - struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; - - if (mem->mm_node) - (*man->func->put_node)(man, mem); -} -EXPORT_SYMBOL(ttm_bo_mem_put); +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; -/** - * Add the last move fence to the BO and reserve a new shared slot. - */ -static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, - struct ttm_mem_type_manager *man, - struct ttm_mem_reg *mem, - bool no_wait_gpu) +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res, + struct dmem_cgroup_pool_state *limit_pool) { - struct dma_fence *fence; - int ret; + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { + .ctx = ctx, + .ticket = ticket, + } + }, + .place = place, + .evictor = evictor, + .res = res, + .limit_pool = limit_pool, + }; + s64 lret; - spin_lock(&man->move_lock); - fence = dma_fence_get(man->move); - spin_unlock(&man->move_lock); + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); - if (!fence) - return 0; - - if (no_wait_gpu) - return -EBUSY; + /* One more attempt if we hit low limit? */ + if (!lret && evict_walk.hit_low) { + evict_walk.try_low = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } + if (lret || !ticket) + goto out; - dma_resv_add_shared_fence(bo->base.resv, fence); + /* Reset low limit */ + evict_walk.try_low = evict_walk.hit_low = false; + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.arg.trylock_only = false; - ret = dma_resv_reserve_shared(bo->base.resv, 1); - if (unlikely(ret)) { - dma_fence_put(fence); - return ret; +retry: + do { + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.arg.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); + + /* We hit the low limit? Try once more */ + if (!lret && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; } - - dma_fence_put(bo->moving); - bo->moving = fence; +out: + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; return 0; } /** - * Repeatedly evict memory from the LRU for @mem_type until we create enough - * space, or we've evicted everything and there isn't enough space. + * ttm_bo_pin - Pin the buffer object. + * @bo: The buffer object to pin + * + * Make sure the buffer is not evicted any more during memory pressure. + * @bo must be unpinned again by calling ttm_bo_unpin(). */ -static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, - const struct ttm_place *place, - struct ttm_mem_reg *mem, - struct ttm_operation_ctx *ctx) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - struct ww_acquire_ctx *ticket; - int ret; - - ticket = dma_resv_locking_ctx(bo->base.resv); - do { - ret = (*man->func->get_node)(man, bo, place, mem); - if (unlikely(ret != 0)) - return ret; - if (mem->mm_node) - break; - ret = ttm_mem_evict_first(bdev, mem->mem_type, place, ctx, - ticket); - if (unlikely(ret != 0)) - return ret; - } while (1); - - return ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); -} - -static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, - uint32_t cur_placement, - uint32_t proposed_placement) -{ - uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; - uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; - - /** - * Keep current caching if possible. - */ - - if ((cur_placement & caching) != 0) - result |= (cur_placement & caching); - else if ((man->default_caching & caching) != 0) - result |= man->default_caching; - else if ((TTM_PL_FLAG_CACHED & caching) != 0) - result |= TTM_PL_FLAG_CACHED; - else if ((TTM_PL_FLAG_WC & caching) != 0) - result |= TTM_PL_FLAG_WC; - else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) - result |= TTM_PL_FLAG_UNCACHED; - - return result; -} - -static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, - uint32_t mem_type, - const struct ttm_place *place, - uint32_t *masked_placement) +void ttm_bo_pin(struct ttm_buffer_object *bo) { - uint32_t cur_flags = ttm_bo_type_flags(mem_type); - - if ((cur_flags & place->flags & TTM_PL_MASK_MEM) == 0) - return false; - - if ((place->flags & man->available_caching) == 0) - return false; - - cur_flags |= (place->flags & man->available_caching); - - *masked_placement = cur_flags; - return true; + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + if (!bo->pin_count++ && bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); } +EXPORT_SYMBOL(ttm_bo_pin); /** - * ttm_bo_mem_placement - check if placement is compatible - * @bo: BO to find memory for - * @place: where to search - * @mem: the memory object to fill in - * @ctx: operation context + * ttm_bo_unpin - Unpin the buffer object. + * @bo: The buffer object to unpin * - * Check if placement is compatible and fill in mem structure. - * Returns -EBUSY if placement won't work or negative error code. - * 0 when placement can be used. + * Allows the buffer object to be evicted again during memory pressure. */ -static int ttm_bo_mem_placement(struct ttm_buffer_object *bo, - const struct ttm_place *place, - struct ttm_mem_reg *mem, - struct ttm_operation_ctx *ctx) +void ttm_bo_unpin(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; - uint32_t mem_type = TTM_PL_SYSTEM; - struct ttm_mem_type_manager *man; - uint32_t cur_flags = 0; - int ret; - - ret = ttm_mem_type_from_place(place, &mem_type); - if (ret) - return ret; - - man = &bdev->man[mem_type]; - if (!man->has_type || !man->use_type) - return -EBUSY; + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + if (WARN_ON_ONCE(!bo->pin_count)) + return; - if (!ttm_bo_mt_compatible(man, mem_type, place, &cur_flags)) - return -EBUSY; + spin_lock(&bo->bdev->lru_lock); + if (!--bo->pin_count && bo->resource) { + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + } + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_unpin); - cur_flags = ttm_bo_select_caching(man, bo->mem.placement, cur_flags); - /* - * Use the access and other non-mapping-related flag bits from - * the memory placement flags to the current flags - */ - ttm_flag_masked(&cur_flags, place->flags, ~TTM_PL_MASK_MEMTYPE); +/* + * Add the pipelined eviction fencesto the BO as kernel dependency and reserve new + * fence slots. + */ +static int ttm_bo_add_pipelined_eviction_fences(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + bool no_wait_gpu) +{ + struct dma_fence *fence; + int i; - mem->mem_type = mem_type; - mem->placement = cur_flags; + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (!fence) + continue; - spin_lock(&ttm_bo_glob.lru_lock); - ttm_bo_del_from_lru(bo); - ttm_bo_add_mem_to_lru(bo, mem); - spin_unlock(&ttm_bo_glob.lru_lock); + if (no_wait_gpu) { + if (!dma_fence_is_signaled(fence)) { + spin_unlock(&man->eviction_lock); + return -EBUSY; + } + } else { + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + } + } + spin_unlock(&man->eviction_lock); - return 0; + /* TODO: this call should be removed. */ + return dma_resv_reserve_fences(bo->base.resv, 1); } /** - * Creates space for memory region @mem according to its type. + * ttm_bo_alloc_resource - Allocate backing store for a BO * - * This function first searches for free space in compatible memory types in - * the priority order defined by the driver. If free space isn't found, then - * ttm_bo_mem_force_space is attempted in priority order to evict and find - * space. + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @ctx: if and how to sleep, lock buffers and alloc memory + * @force_space: If we should evict buffers to force space + * @res: The resulting struct ttm_resource. + * + * Allocates a resource for the buffer object pointed to by @bo, using the + * placement flags in @placement, potentially evicting other buffer objects when + * @force_space is true. + * This function may sleep while waiting for resources to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == true). + * -ENOSPC: Could not allocate space for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. */ -int ttm_bo_mem_space(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_mem_reg *mem, - struct ttm_operation_ctx *ctx) +static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx, + bool force_space, + struct ttm_resource **res) { - struct ttm_bo_device *bdev = bo->bdev; - bool type_found = false; + struct ttm_device *bdev = bo->bdev; + struct ww_acquire_ctx *ticket; int i, ret; - ret = dma_resv_reserve_shared(bo->base.resv, 1); + ticket = dma_resv_locking_ctx(bo->base.resv); + ret = dma_resv_reserve_fences(bo->base.resv, TTM_NUM_MOVE_FENCES); if (unlikely(ret)) return ret; - mem->mm_node = NULL; for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; - struct ttm_mem_type_manager *man; + struct dmem_cgroup_pool_state *limit_pool = NULL; + struct ttm_resource_manager *man; + bool may_evict; - ret = ttm_bo_mem_placement(bo, place, mem, ctx); - if (ret == -EBUSY) + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) continue; - if (ret) - goto error; - type_found = true; - mem->mm_node = NULL; - if (mem->mem_type == TTM_PL_SYSTEM) - return 0; + if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; - man = &bdev->man[mem->mem_type]; - ret = (*man->func->get_node)(man, bo, place, mem); - if (unlikely(ret)) - goto error; + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); + ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); + if (ret) { + if (ret != -ENOSPC && ret != -EAGAIN) { + dmem_cgroup_pool_state_put(limit_pool); + return ret; + } + if (!may_evict) { + dmem_cgroup_pool_state_put(limit_pool); + continue; + } - if (!mem->mm_node) - continue; + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res, limit_pool); + dmem_cgroup_pool_state_put(limit_pool); + if (ret == -EBUSY) + continue; + if (ret) + return ret; + } - ret = ttm_bo_add_move_fence(bo, man, mem, ctx->no_wait_gpu); + ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); if (unlikely(ret)) { - (*man->func->put_node)(man, mem); + ttm_resource_free(bo, res); if (ret == -EBUSY) continue; - goto error; + return ret; } return 0; } - for (i = 0; i < placement->num_busy_placement; ++i) { - const struct ttm_place *place = &placement->busy_placement[i]; - - ret = ttm_bo_mem_placement(bo, place, mem, ctx); - if (ret == -EBUSY) - continue; - if (ret) - goto error; - - type_found = true; - mem->mm_node = NULL; - if (mem->mem_type == TTM_PL_SYSTEM) - return 0; - - ret = ttm_bo_mem_force_space(bo, place, mem, ctx); - if (ret == 0 && mem->mm_node) - return 0; - - if (ret && ret != -EBUSY) - goto error; - } - - ret = -ENOMEM; - if (!type_found) { - pr_err(TTM_PFX "No compatible memory type found\n"); - ret = -EINVAL; - } - -error: - if (bo->mem.mem_type == TTM_PL_SYSTEM && !list_empty(&bo->lru)) { - spin_lock(&ttm_bo_glob.lru_lock); - ttm_bo_move_to_lru_tail(bo, NULL); - spin_unlock(&ttm_bo_glob.lru_lock); - } - - return ret; + return -ENOSPC; } -EXPORT_SYMBOL(ttm_bo_mem_space); -static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_operation_ctx *ctx) +/* + * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @res: The resulting struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Tries both idle allocation and forcefully eviction of buffers. See + * ttm_bo_alloc_resource for details. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource **res, + struct ttm_operation_ctx *ctx) { - int ret = 0; - struct ttm_mem_reg mem; + bool force_space = false; + int ret; - dma_resv_assert_held(bo->base.resv); + do { + ret = ttm_bo_alloc_resource(bo, placement, ctx, + force_space, res); + force_space = !force_space; + } while (ret == -ENOSPC && force_space); - mem.num_pages = bo->num_pages; - mem.size = mem.num_pages << PAGE_SHIFT; - mem.page_alignment = bo->mem.page_alignment; - mem.bus.io_reserved_vm = false; - mem.bus.io_reserved_count = 0; - /* - * Determine where to move the buffer. - */ - ret = ttm_bo_mem_space(bo, placement, &mem, ctx); - if (ret) - goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, ctx); -out_unlock: - if (ret && mem.mm_node) - ttm_bo_mem_put(bo, &mem); return ret; } +EXPORT_SYMBOL(ttm_bo_mem_space); -static bool ttm_bo_places_compat(const struct ttm_place *places, - unsigned num_placement, - struct ttm_mem_reg *mem, - uint32_t *new_flags) -{ - unsigned i; - - for (i = 0; i < num_placement; i++) { - const struct ttm_place *heap = &places[i]; - - if (mem->mm_node && (mem->start < heap->fpfn || - (heap->lpfn != 0 && (mem->start + mem->num_pages) > heap->lpfn))) - continue; - - *new_flags = heap->flags; - if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) && - (*new_flags & mem->placement & TTM_PL_MASK_MEM) && - (!(*new_flags & TTM_PL_FLAG_CONTIGUOUS) || - (mem->placement & TTM_PL_FLAG_CONTIGUOUS))) - return true; - } - return false; -} - -bool ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem, - uint32_t *new_flags) -{ - if (ttm_bo_places_compat(placement->placement, placement->num_placement, - mem, new_flags)) - return true; - - if ((placement->busy_placement != placement->placement || - placement->num_busy_placement > placement->num_placement) && - ttm_bo_places_compat(placement->busy_placement, - placement->num_busy_placement, - mem, new_flags)) - return true; - - return false; -} -EXPORT_SYMBOL(ttm_bo_mem_compat); - +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) { + struct ttm_resource *res; + struct ttm_place hop; + bool force_space; int ret; - uint32_t new_flags; dma_resv_assert_held(bo->base.resv); /* * Remove the backing store if no placement is given. */ - if (!placement->num_placement && !placement->num_busy_placement) { - ret = ttm_bo_pipeline_gutting(bo); - if (ret) - return ret; + if (!placement->num_placement) + return ttm_bo_pipeline_gutting(bo); - return ttm_tt_create(bo, false); - } + force_space = false; + do { + /* Check whether we need to move buffer. */ + if (bo->resource && + ttm_resource_compatible(bo->resource, placement, + force_space)) + return 0; + + /* Moving of pinned BOs is forbidden */ + if (bo->pin_count) + return -EINVAL; - /* - * Check whether we need to move buffer. - */ - if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) { - ret = ttm_bo_move_buffer(bo, placement, ctx); - if (ret) - return ret; - } else { /* - * Use the access and other non-mapping-related flag bits from - * the compatible memory placement flags to the active flags + * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. */ - ttm_flag_masked(&bo->mem.placement, new_flags, - ~TTM_PL_MASK_MEMTYPE); - } + ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, + &res); + force_space = !force_space; + if (ret == -ENOSPC) + continue; + if (ret) + return ret; + +bounce: + ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + /* try and move to final place now. */ + if (!ret) + goto bounce; + } + if (ret) { + ttm_resource_free(bo, &res); + return ret; + } + + } while (ret && force_space); + + /* For backward compatibility with userspace */ + if (ret == -ENOSPC) + return bo->bdev->alloc_flags & TTM_ALLOCATION_PROPAGATE_ENOSPC ? + ret : -ENOMEM; + /* * We might need to add a TTM. */ - if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { + if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { ret = ttm_tt_create(bo, true); if (ret) return ret; @@ -1236,133 +895,142 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_validate); -int ttm_bo_init_reserved(struct ttm_bo_device *bdev, - struct ttm_buffer_object *bo, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - struct ttm_operation_ctx *ctx, - size_t acc_size, - struct sg_table *sg, - struct dma_resv *resv, +/** + * ttm_bo_init_reserved + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @ctx: TTM operation context for memory allocation. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, enables driver-specific objects + * derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * Furthermore, if resv == NULL, the buffer's reservation lock will be held, + * and it is the caller's responsibility to call ttm_bo_unreserve. + * + * If a failure occurs, the function will call the @destroy function. Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, struct ttm_operation_ctx *ctx, + struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { - struct ttm_mem_global *mem_glob = &ttm_mem_glob; - int ret = 0; - unsigned long num_pages; - bool locked; - - ret = ttm_mem_global_alloc(mem_glob, acc_size, ctx); - if (ret) { - pr_err("Out of kernel memory\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - return -ENOMEM; - } - - num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (num_pages == 0) { - pr_err("Illegal buffer object size\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - ttm_mem_global_free(mem_glob, acc_size); - return -EINVAL; - } - bo->destroy = destroy ? destroy : ttm_bo_default_destroy; + int ret; kref_init(&bo->kref); - INIT_LIST_HEAD(&bo->lru); - INIT_LIST_HEAD(&bo->ddestroy); - INIT_LIST_HEAD(&bo->swap); - INIT_LIST_HEAD(&bo->io_reserve_lru); bo->bdev = bdev; bo->type = type; - bo->num_pages = num_pages; - bo->mem.size = num_pages << PAGE_SHIFT; - bo->mem.mem_type = TTM_PL_SYSTEM; - bo->mem.num_pages = bo->num_pages; - bo->mem.mm_node = NULL; - bo->mem.page_alignment = page_alignment; - bo->mem.bus.io_reserved_vm = false; - bo->mem.bus.io_reserved_count = 0; - bo->moving = NULL; - bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); - bo->acc_size = acc_size; + bo->page_alignment = alignment; + bo->destroy = destroy; + bo->pin_count = 0; bo->sg = sg; - if (resv) { + bo->bulk_move = NULL; + if (resv) bo->base.resv = resv; - dma_resv_assert_held(bo->base.resv); - } else { + else bo->base.resv = &bo->base._resv; - } - if (!ttm_bo_uses_embedded_gem_object(bo)) { - /* - * bo.gem is not initialized, so we have to setup the - * struct elements we want use regardless. - */ - dma_resv_init(&bo->base._resv); - drm_vma_node_reset(&bo->base.vma_node); - } - atomic_inc(&ttm_bo_glob.bo_count); + atomic_inc(&ttm_glob.bo_count); /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ - if (bo->type == ttm_bo_type_device || - bo->type == ttm_bo_type_sg) + if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, - bo->mem.num_pages); + PFN_UP(bo->base.size)); + if (ret) + goto err_put; + } /* passed reservation objects should already be locked, * since otherwise lockdep will be angered in radeon. */ - if (!resv) { - locked = dma_resv_trylock(bo->base.resv); - WARN_ON(!locked); - } - - if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, ctx); + if (!resv) + WARN_ON(!dma_resv_trylock(bo->base.resv)); + else + dma_resv_assert_held(resv); - if (unlikely(ret)) { - if (!resv) - ttm_bo_unreserve(bo); + ret = ttm_bo_validate(bo, placement, ctx); + if (unlikely(ret)) + goto err_unlock; - ttm_bo_put(bo); - return ret; - } + return 0; - spin_lock(&ttm_bo_glob.lru_lock); - ttm_bo_move_to_lru_tail(bo, NULL); - spin_unlock(&ttm_bo_glob.lru_lock); +err_unlock: + if (!resv) + dma_resv_unlock(bo->base.resv); +err_put: + ttm_bo_put(bo); return ret; } EXPORT_SYMBOL(ttm_bo_init_reserved); -int ttm_bo_init(struct ttm_bo_device *bdev, - struct ttm_buffer_object *bo, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - size_t acc_size, - struct sg_table *sg, - struct dma_resv *resv, - void (*destroy) (struct ttm_buffer_object *)) +/** + * ttm_bo_init_validate + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * + * If a failure occurs, the function will call the @destroy function, Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, bool interruptible, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) { struct ttm_operation_ctx ctx = { interruptible, false }; int ret; - ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, &ctx, acc_size, + ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, sg, resv, destroy); if (ret) return ret; @@ -1372,519 +1040,264 @@ int ttm_bo_init(struct ttm_bo_device *bdev, return 0; } -EXPORT_SYMBOL(ttm_bo_init); +EXPORT_SYMBOL(ttm_bo_init_validate); -size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += ttm_round_pot(npages * sizeof(void *)); - size += ttm_round_pot(sizeof(struct ttm_tt)); - return size; -} -EXPORT_SYMBOL(ttm_bo_acc_size); +/* + * buffer object vm functions. + */ -size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; + struct ttm_device *bdev = bo->bdev; - size += ttm_round_pot(struct_size); - size += ttm_round_pot(npages * (2*sizeof(void *) + sizeof(dma_addr_t))); - size += ttm_round_pot(sizeof(struct ttm_dma_tt)); - return size; + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + ttm_mem_io_free(bdev, bo->resource); } -EXPORT_SYMBOL(ttm_bo_dma_acc_size); - -int ttm_bo_create(struct ttm_bo_device *bdev, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - struct ttm_buffer_object **p_bo) -{ - struct ttm_buffer_object *bo; - size_t acc_size; - int ret; +EXPORT_SYMBOL(ttm_bo_unmap_virtual); - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - if (unlikely(bo == NULL)) - return -ENOMEM; +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) +{ + long ret; - acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); - ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - interruptible, acc_size, - NULL, NULL, NULL); - if (likely(ret == 0)) - *p_bo = bo; + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return 0; + else + return -EBUSY; + } - return ret; + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) + return -EBUSY; + return 0; } -EXPORT_SYMBOL(ttm_bo_create); +EXPORT_SYMBOL(ttm_bo_wait_ctx); -static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, - unsigned mem_type) +/** + * struct ttm_bo_swapout_walk - Parameters for the swapout walk + */ +struct ttm_bo_swapout_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ + gfp_t gfp_flags; + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ + bool hit_low, evict_low; +}; + +static s64 +ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false, - .flags = TTM_OPT_FLAG_FORCE_ALLOC - }; - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct ttm_bo_global *glob = &ttm_bo_glob; - struct dma_fence *fence; - int ret; - unsigned i; + struct ttm_place place = {.mem_type = bo->resource->mem_type}; + struct ttm_bo_swapout_walk *swapout_walk = + container_of(walk, typeof(*swapout_walk), walk); + struct ttm_operation_ctx *ctx = walk->arg.ctx; + s64 ret; /* - * Can't use standard list traversal since we're unlocking. + * While the bo may already reside in SYSTEM placement, set + * SYSTEM as new placement to cover also the move further below. + * The driver may use the fact that we're moving from SYSTEM + * as an indication that we're about to swap out. */ - - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - while (!list_empty(&man->lru[i])) { - spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, &ctx, - NULL); - if (ret) - return ret; - spin_lock(&glob->lru_lock); - } + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, &place)) { + ret = -EBUSY; + goto out; } - spin_unlock(&glob->lru_lock); - spin_lock(&man->move_lock); - fence = dma_fence_get(man->move); - spin_unlock(&man->move_lock); - - if (fence) { - ret = dma_fence_wait(fence, false); - dma_fence_put(fence); - if (ret) - return ret; + if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || + bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || + bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) { + ret = -EBUSY; + goto out; } - return 0; -} + if (bo->deleted) { + pgoff_t num_pages = bo->ttm->num_pages; -int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ - struct ttm_mem_type_manager *man; - int ret = -EINVAL; + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; - if (mem_type >= TTM_NUM_MEM_TYPES) { - pr_err("Illegal memory type %d\n", mem_type); - return ret; + ttm_bo_cleanup_memtype_use(bo); + ret = num_pages; + goto out; } - man = &bdev->man[mem_type]; - if (!man->has_type) { - pr_err("Trying to take down uninitialized memory manager type %u\n", - mem_type); - return ret; - } + /* + * Move to system cached + */ + if (bo->resource->mem_type != TTM_PL_SYSTEM) { + struct ttm_resource *evict_mem; + struct ttm_place hop; - man->use_type = false; - man->has_type = false; + memset(&hop, 0, sizeof(hop)); + place.mem_type = TTM_PL_SYSTEM; + ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL); + if (ret) + goto out; - ret = 0; - if (mem_type > 0) { - ret = ttm_bo_force_list_clean(bdev, mem_type); + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); if (ret) { - pr_err("Cleanup eviction failed\n"); - return ret; + WARN(ret == -EMULTIHOP, + "Unexpected multihop in swapout - likely driver bug.\n"); + ttm_resource_free(bo, &evict_mem); + goto out; } - - ret = (*man->func->takedown)(man); - } - - dma_fence_put(man->move); - man->move = NULL; - - return ret; -} -EXPORT_SYMBOL(ttm_bo_clean_mm); - -int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - - if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { - pr_err("Illegal memory manager memory type %u\n", mem_type); - return -EINVAL; - } - - if (!man->has_type) { - pr_err("Memory type %u has not been initialized\n", mem_type); - return 0; } - return ttm_bo_force_list_clean(bdev, mem_type); -} -EXPORT_SYMBOL(ttm_bo_evict_mm); - -int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, - unsigned long p_size) -{ - int ret; - struct ttm_mem_type_manager *man; - unsigned i; - - BUG_ON(type >= TTM_NUM_MEM_TYPES); - man = &bdev->man[type]; - BUG_ON(man->has_type); - man->io_reserve_fastpath = true; - man->use_io_reserve_lru = false; - mutex_init(&man->io_reserve_mutex); - spin_lock_init(&man->move_lock); - INIT_LIST_HEAD(&man->io_reserve_lru); - - ret = bdev->driver->init_mem_type(bdev, type, man); + /* + * Make sure BO is idle. + */ + ret = ttm_bo_wait_ctx(bo, ctx); if (ret) - return ret; - man->bdev = bdev; - - if (type != TTM_PL_SYSTEM) { - ret = (*man->func->init)(man, p_size); - if (ret) - return ret; - } - man->has_type = true; - man->use_type = true; - man->size = p_size; - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - INIT_LIST_HEAD(&man->lru[i]); - man->move = NULL; - - return 0; -} -EXPORT_SYMBOL(ttm_bo_init_mm); - -static void ttm_bo_global_kobj_release(struct kobject *kobj) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - __free_page(glob->dummy_read_page); -} - -static void ttm_bo_global_release(void) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - - mutex_lock(&ttm_global_mutex); - if (--ttm_bo_glob_use_count > 0) goto out; - kobject_del(&glob->kobj); - kobject_put(&glob->kobj); - ttm_mem_global_release(&ttm_mem_glob); - memset(glob, 0, sizeof(*glob)); -out: - mutex_unlock(&ttm_global_mutex); -} - -static int ttm_bo_global_init(void) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret = 0; - unsigned i; - - mutex_lock(&ttm_global_mutex); - if (++ttm_bo_glob_use_count > 1) - goto out; + ttm_bo_unmap_virtual(bo); + if (bo->bdev->funcs->swap_notify) + bo->bdev->funcs->swap_notify(bo); - ret = ttm_mem_global_init(&ttm_mem_glob); - if (ret) - goto out; + if (ttm_tt_is_populated(bo->ttm)) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); - spin_lock_init(&glob->lru_lock); - glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); + ret = ttm_tt_swapout(bo->bdev, bo->ttm, swapout_walk->gfp_flags); - if (unlikely(glob->dummy_read_page == NULL)) { - ret = -ENOMEM; - goto out; + spin_lock(&bo->bdev->lru_lock); + if (ret) + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); } - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - INIT_LIST_HEAD(&glob->swap_lru[i]); - INIT_LIST_HEAD(&glob->device_list); - atomic_set(&glob->bo_count, 0); - - ret = kobject_init_and_add( - &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); - if (unlikely(ret != 0)) - kobject_put(&glob->kobj); out: - mutex_unlock(&ttm_global_mutex); - return ret; -} - -int ttm_bo_device_release(struct ttm_bo_device *bdev) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret = 0; - unsigned i = TTM_NUM_MEM_TYPES; - struct ttm_mem_type_manager *man; - - while (i--) { - man = &bdev->man[i]; - if (man->has_type) { - man->use_type = false; - if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { - ret = -EBUSY; - pr_err("DRM memory manager type %d is not clean\n", - i); - } - man->has_type = false; - } - } - - mutex_lock(&ttm_global_mutex); - list_del(&bdev->device_list); - mutex_unlock(&ttm_global_mutex); - - cancel_delayed_work_sync(&bdev->wq); - - if (ttm_bo_delayed_delete(bdev, true)) - pr_debug("Delayed destroy list was clean\n"); - - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) - if (list_empty(&bdev->man[0].lru[0])) - pr_debug("Swap list %d was clean\n", i); - spin_unlock(&glob->lru_lock); - - if (!ret) - ttm_bo_global_release(); + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; return ret; } -EXPORT_SYMBOL(ttm_bo_device_release); - -int ttm_bo_device_init(struct ttm_bo_device *bdev, - struct ttm_bo_driver *driver, - struct address_space *mapping, - struct drm_vma_offset_manager *vma_manager, - bool need_dma32) -{ - struct ttm_bo_global *glob = &ttm_bo_glob; - int ret; - if (WARN_ON(vma_manager == NULL)) - return -EINVAL; - - ret = ttm_bo_global_init(); - if (ret) - return ret; - - bdev->driver = driver; - - memset(bdev->man, 0, sizeof(bdev->man)); - - /* - * Initialize the system memory buffer type. - * Other types need to be driver / IOCTL initialized. - */ - ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); - if (unlikely(ret != 0)) - goto out_no_sys; - - bdev->vma_manager = vma_manager; - INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = mapping; - bdev->need_dma32 = need_dma32; - mutex_lock(&ttm_global_mutex); - list_add_tail(&bdev->device_list, &glob->device_list); - mutex_unlock(&ttm_global_mutex); - - return 0; -out_no_sys: - ttm_bo_global_release(); - return ret; -} -EXPORT_SYMBOL(ttm_bo_device_init); +const struct ttm_lru_walk_ops ttm_swap_ops = { + .process_bo = ttm_bo_swapout_cb, +}; -/* - * buffer object vm functions. +/** + * ttm_bo_swapout() - Swap out buffer objects on the LRU list to shmem. + * @bdev: The ttm device. + * @ctx: The ttm_operation_ctx governing the swapout operation. + * @man: The resource manager whose resources / buffer objects are + * goint to be swapped out. + * @gfp_flags: The gfp flags used for shmem page allocations. + * @target: The desired number of bytes to swap out. + * + * Return: The number of bytes actually swapped out, or negative error code + * on error. */ - -bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) +s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + struct ttm_resource_manager *man, gfp_t gfp_flags, + s64 target) { - struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - - if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (mem->mem_type == TTM_PL_SYSTEM) - return false; - - if (man->flags & TTM_MEMTYPE_FLAG_CMA) - return false; + struct ttm_bo_swapout_walk swapout_walk = { + .walk = { + .ops = &ttm_swap_ops, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, + }, + .gfp_flags = gfp_flags, + }; - if (mem->placement & TTM_PL_FLAG_CACHED) - return false; - } - return true; + return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target); } -void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) +void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; + if (bo->ttm == NULL) + return; - drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); - ttm_mem_io_free_vm(bo); + ttm_tt_unpopulate(bo->bdev, bo->ttm); + ttm_tt_destroy(bo->bdev, bo->ttm); + bo->ttm = NULL; } -void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +/** + * ttm_bo_populate() - Ensure that a buffer object has backing pages + * @bo: The buffer object + * @ctx: The ttm_operation_ctx governing the operation. + * + * For buffer objects in a memory type whose manager uses + * struct ttm_tt for backing pages, ensure those backing pages + * are present and with valid content. The bo's resource is also + * placed on the correct LRU list if it was previously swapped + * out. + * + * Return: 0 if successful, negative error code on failure. + * Note: May return -EINTR or -ERESTARTSYS if @ctx::interruptible + * is set to true. + */ +int ttm_bo_populate(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - - ttm_mem_io_lock(man, false); - ttm_bo_unmap_virtual_locked(bo); - ttm_mem_io_unlock(man); -} + struct ttm_tt *tt = bo->ttm; + bool swapped; + int ret; + dma_resv_assert_held(bo->base.resv); -EXPORT_SYMBOL(ttm_bo_unmap_virtual); + if (!tt) + return 0; -int ttm_bo_wait(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait) -{ - long timeout = 15 * HZ; + swapped = ttm_tt_is_swapped(tt); + ret = ttm_tt_populate(bo->bdev, tt, ctx); + if (ret) + return ret; - if (no_wait) { - if (dma_resv_test_signaled_rcu(bo->base.resv, true)) - return 0; - else - return -EBUSY; + if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && + bo->resource) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); } - timeout = dma_resv_wait_timeout_rcu(bo->base.resv, true, - interruptible, timeout); - if (timeout < 0) - return timeout; - - if (timeout == 0) - return -EBUSY; - - dma_resv_add_excl_fence(bo->base.resv, NULL); return 0; } -EXPORT_SYMBOL(ttm_bo_wait); +EXPORT_SYMBOL(ttm_bo_populate); -/** - * A buffer object shrink method that tries to swap out the first - * buffer object on the bo_global::swap_lru list. - */ -int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx) +int ttm_bo_setup_export(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { - struct ttm_buffer_object *bo; - int ret = -EBUSY; - bool locked; - unsigned i; - - spin_lock(&glob->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &glob->swap_lru[i], swap) { - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &locked, - NULL)) - continue; - - if (!ttm_bo_get_unless_zero(bo)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; - } - - ret = 0; - break; - } - if (!ret) - break; - } - - if (ret) { - spin_unlock(&glob->lru_lock); - return ret; - } + int ret; - if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, false, false, locked); - ttm_bo_put(bo); + ret = ttm_bo_reserve(bo, false, false, NULL); + if (ret != 0) return ret; - } - ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); - - /** - * Move to system cached - */ - - if (bo->mem.mem_type != TTM_PL_SYSTEM || - bo->ttm->caching_state != tt_cached) { - struct ttm_operation_ctx ctx = { false, false }; - struct ttm_mem_reg evict_mem; - - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; - evict_mem.mem_type = TTM_PL_SYSTEM; - - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, &ctx); - if (unlikely(ret != 0)) - goto out; - } - - /** - * Make sure BO is idle. - */ - - ret = ttm_bo_wait(bo, false, false); - if (unlikely(ret != 0)) - goto out; - - ttm_bo_unmap_virtual(bo); - - /** - * Swap out. Buffer will be swapped in again as soon as - * anyone tries to access a ttm page. - */ - - if (bo->bdev->driver->swap_notify) - bo->bdev->driver->swap_notify(bo); - - ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); -out: - - /** - * - * Unreserve without putting on LRU to avoid swapping out an - * already swapped buffer. - */ - if (locked) - dma_resv_unlock(bo->base.resv); - ttm_bo_put(bo); + ret = ttm_bo_populate(bo, ctx); + ttm_bo_unreserve(bo); return ret; } -EXPORT_SYMBOL(ttm_bo_swapout); - -void ttm_bo_swapout_all(struct ttm_bo_device *bdev) -{ - struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - - while (ttm_bo_swapout(&ttm_bo_glob, &ctx) == 0); -} -EXPORT_SYMBOL(ttm_bo_swapout_all); +EXPORT_SYMBOL(ttm_bo_setup_export); |