diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 1450 |
1 files changed, 782 insertions, 668 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index db3dc7ef5382..bd27607f8076 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -31,8 +31,13 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/drm_print.h> +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -40,26 +45,19 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> +#include <linux/cgroup_dmem.h> #include <linux/dma-resv.h> #include "ttm_module.h" - -/* default destructor */ -static void ttm_bo_default_destroy(struct ttm_buffer_object *bo) -{ - kfree(bo); -} +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) { - struct drm_printer p = drm_debug_printer(TTM_PFX); + struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); struct ttm_resource_manager *man; int i, mem_type; - drm_printf(&p, "No space for %p (%lu pages, %zuK, %zuM)\n", - bo, bo->resource->num_pages, bo->base.size >> 10, - bo->base.size >> 20); for (i = 0; i < placement->num_placement; i++) { mem_type = placement->placement[i].mem_type; drm_printf(&p, " placement[%d]=0x%08X (%d)\n", @@ -69,119 +67,67 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, } } -static inline void ttm_bo_move_to_pinned(struct ttm_buffer_object *bo) -{ - struct ttm_device *bdev = bo->bdev; - - list_move_tail(&bo->lru, &bdev->pinned); - - if (bdev->funcs->del_from_lru_notify) - bdev->funcs->del_from_lru_notify(bo); -} - -static inline void ttm_bo_del_from_lru(struct ttm_buffer_object *bo) -{ - struct ttm_device *bdev = bo->bdev; - - list_del_init(&bo->lru); - - if (bdev->funcs->del_from_lru_notify) - bdev->funcs->del_from_lru_notify(bo); -} - -static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos, - struct ttm_buffer_object *bo) -{ - if (!pos->first) - pos->first = bo; - pos->last = bo; -} - -void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo, - struct ttm_resource *mem, - struct ttm_lru_bulk_move *bulk) +/** + * ttm_bo_move_to_lru_tail + * + * @bo: The buffer object. + * + * Move this BO to the tail of all lru lists used to lookup and reserve an + * object. This function must be called with struct ttm_global::lru_lock + * held, and is used to make a BO less likely to be considered for eviction. + */ +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man; - - if (!bo->deleted) - dma_resv_assert_held(bo->base.resv); - - if (bo->pin_count) { - ttm_bo_move_to_pinned(bo); - return; - } - - if (!mem) - return; - - man = ttm_manager_type(bdev, mem->mem_type); - list_move_tail(&bo->lru, &man->lru[bo->priority]); - - if (bdev->funcs->del_from_lru_notify) - bdev->funcs->del_from_lru_notify(bo); - - if (bulk && !bo->pin_count) { - switch (bo->resource->mem_type) { - case TTM_PL_TT: - ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo); - break; + dma_resv_assert_held(bo->base.resv); - case TTM_PL_VRAM: - ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo); - break; - } - } + if (bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); -void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk) +/** + * ttm_bo_set_bulk_move - update BOs bulk move object + * + * @bo: The buffer object. + * @bulk: bulk move structure + * + * Update the BOs bulk move object, making sure that resources are added/removed + * as well. A bulk move allows to move many resource on the LRU at once, + * resulting in much less overhead of maintaining the LRU. + * The only requirement is that the resources stay together on the LRU and are + * never separated. This is enforces by setting the bulk_move structure on a BO. + * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of + * their LRU list. + */ +void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) { - unsigned i; - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i]; - struct ttm_resource_manager *man; - - if (!pos->first) - continue; - - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); - - man = ttm_manager_type(pos->first->bdev, TTM_PL_TT); - list_bulk_move_tail(&man->lru[i], &pos->first->lru, - &pos->last->lru); - } - - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i]; - struct ttm_resource_manager *man; - - if (!pos->first) - continue; + dma_resv_assert_held(bo->base.resv); - dma_resv_assert_held(pos->first->base.resv); - dma_resv_assert_held(pos->last->base.resv); + if (bo->bulk_move == bulk) + return; - man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM); - list_bulk_move_tail(&man->lru[i], &pos->first->lru, - &pos->last->lru); - } + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + bo->bulk_move = bulk; + if (bo->resource) + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); } -EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail); +EXPORT_SYMBOL(ttm_bo_set_bulk_move); static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_resource *mem, bool evict, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { - struct ttm_resource_manager *old_man, *new_man; struct ttm_device *bdev = bo->bdev; + bool old_use_tt, new_use_tt; int ret; - old_man = ttm_manager_type(bdev, bo->resource->mem_type); - new_man = ttm_manager_type(bdev, mem->mem_type); + old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; + new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; ttm_bo_unmap_virtual(bo); @@ -189,21 +135,25 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, * Create and bind a ttm if required. */ - if (new_man->use_tt) { + if (new_use_tt) { /* Zero init the new TTM structure if the old location should * have used one as well. */ - ret = ttm_tt_create(bo, old_man->use_tt); + ret = ttm_tt_create(bo, old_use_tt); if (ret) goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_populate(bo->bdev, bo->ttm, ctx); + ret = ttm_bo_populate(bo, ctx); if (ret) goto out_err; } } + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto out_err; + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); if (ret) { if (ret == -EMULTIHOP) @@ -215,8 +165,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, return 0; out_err: - new_man = ttm_manager_type(bdev, bo->resource->mem_type); - if (!new_man->use_tt) + if (!old_use_tt) ttm_bo_tt_destroy(bo); return ret; @@ -272,7 +221,7 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) struct dma_resv_iter cursor; struct dma_fence *fence; - dma_resv_iter_begin(&cursor, resv, true); + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); dma_resv_for_each_fence_unlocked(&cursor, fence) { if (!fence->ops->signaled) dma_fence_enable_sw_signaling(fence); @@ -280,125 +229,22 @@ static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) dma_resv_iter_end(&cursor); } -/** - * ttm_bo_cleanup_refs - * If bo idle, remove from lru lists, and unref. - * If not idle, block if possible. - * - * Must be called with lru_lock and reservation held, this function - * will drop the lru lock and optionally the reservation lock before returning. - * - * @bo: The buffer object to clean-up - * @interruptible: Any sleeps should occur interruptibly. - * @no_wait_gpu: Never wait for gpu. Return -EBUSY instead. - * @unlock_resv: Unlock the reservation lock as well. +/* + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. */ - -static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait_gpu, - bool unlock_resv) +static void ttm_bo_delayed_delete(struct work_struct *work) { - struct dma_resv *resv = &bo->base._resv; - int ret; - - if (dma_resv_test_signaled(resv, true)) - ret = 0; - else - ret = -EBUSY; - - if (ret && !no_wait_gpu) { - long lret; - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&bo->bdev->lru_lock); - - lret = dma_resv_wait_timeout(resv, true, interruptible, - 30 * HZ); - - if (lret < 0) - return lret; - else if (lret == 0) - return -EBUSY; - - spin_lock(&bo->bdev->lru_lock); - if (unlock_resv && !dma_resv_trylock(bo->base.resv)) { - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - spin_unlock(&bo->bdev->lru_lock); - return 0; - } - ret = 0; - } + struct ttm_buffer_object *bo; - if (ret || unlikely(list_empty(&bo->ddestroy))) { - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - spin_unlock(&bo->bdev->lru_lock); - return ret; - } + bo = container_of(work, typeof(*bo), delayed_delete); - ttm_bo_move_to_pinned(bo); - list_del_init(&bo->ddestroy); - spin_unlock(&bo->bdev->lru_lock); + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); ttm_bo_cleanup_memtype_use(bo); - - if (unlock_resv) - dma_resv_unlock(bo->base.resv); - + dma_resv_unlock(bo->base.resv); ttm_bo_put(bo); - - return 0; -} - -/* - * Traverse the delayed list, and call ttm_bo_cleanup_refs on all - * encountered buffers. - */ -bool ttm_bo_delayed_delete(struct ttm_device *bdev, bool remove_all) -{ - struct list_head removed; - bool empty; - - INIT_LIST_HEAD(&removed); - - spin_lock(&bdev->lru_lock); - while (!list_empty(&bdev->ddestroy)) { - struct ttm_buffer_object *bo; - - bo = list_first_entry(&bdev->ddestroy, struct ttm_buffer_object, - ddestroy); - list_move_tail(&bo->ddestroy, &removed); - if (!ttm_bo_get_unless_zero(bo)) - continue; - - if (remove_all || bo->base.resv != &bo->base._resv) { - spin_unlock(&bdev->lru_lock); - dma_resv_lock(bo->base.resv, NULL); - - spin_lock(&bdev->lru_lock); - ttm_bo_cleanup_refs(bo, false, !remove_all, true); - - } else if (dma_resv_trylock(bo->base.resv)) { - ttm_bo_cleanup_refs(bo, false, !remove_all, true); - } else { - spin_unlock(&bdev->lru_lock); - } - - ttm_bo_put(bo); - spin_lock(&bdev->lru_lock); - } - list_splice_tail(&removed, &bdev->ddestroy); - empty = list_empty(&bdev->ddestroy); - spin_unlock(&bdev->lru_lock); - - return empty; } static void ttm_bo_release(struct kref *kref) @@ -409,6 +255,7 @@ static void ttm_bo_release(struct kref *kref) int ret; WARN_ON_ONCE(bo->pin_count); + WARN_ON_ONCE(bo->bulk_move); if (!bo->deleted) { ret = ttm_bo_individualize_resv(bo); @@ -416,7 +263,8 @@ static void ttm_bo_release(struct kref *kref) /* Last resort, if we fail to allocate memory for the * fences block for the BO to become idle */ - dma_resv_wait_timeout(bo->base.resv, true, false, + dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, false, 30 * HZ); } @@ -425,73 +273,66 @@ static void ttm_bo_release(struct kref *kref) drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); ttm_mem_io_free(bdev, bo->resource); - } - if (!dma_resv_test_signaled(bo->base.resv, true) || - !dma_resv_trylock(bo->base.resv)) { - /* The BO is not idle, resurrect it for delayed destroy */ - ttm_bo_flush_all_fences(bo); - bo->deleted = true; + if (!dma_resv_test_signaled(&bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP) || + (want_init_on_free() && (bo->ttm != NULL)) || + bo->type == ttm_bo_type_sg || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; - spin_lock(&bo->bdev->lru_lock); + spin_lock(&bo->bdev->lru_lock); - /* - * Make pinned bos immediately available to - * shrinkers, now that they are queued for - * destruction. - * - * FIXME: QXL is triggering this. Can be removed when the - * driver is fixed. - */ - if (bo->pin_count) { - bo->pin_count = 0; - ttm_bo_move_to_lru_tail(bo, bo->resource, NULL); - } + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } - kref_init(&bo->kref); - list_add_tail(&bo->ddestroy, &bdev->ddestroy); - spin_unlock(&bo->bdev->lru_lock); + kref_init(&bo->kref); + spin_unlock(&bo->bdev->lru_lock); - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); - return; - } + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); - spin_lock(&bo->bdev->lru_lock); - ttm_bo_del_from_lru(bo); - list_del(&bo->ddestroy); - spin_unlock(&bo->bdev->lru_lock); + /* Schedule the worker on the closest NUMA node. This + * improves performance since system memory might be + * cleared on free and that is best done on a CPU core + * close to it. + */ + queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); + return; + } - ttm_bo_cleanup_memtype_use(bo); - dma_resv_unlock(bo->base.resv); + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); + } atomic_dec(&ttm_glob.bo_count); - dma_fence_put(bo->moving); bo->destroy(bo); } +/* TODO: remove! */ void ttm_bo_put(struct ttm_buffer_object *bo) { kref_put(&bo->kref, ttm_bo_release); } -EXPORT_SYMBOL(ttm_bo_put); -int ttm_bo_lock_delayed_workqueue(struct ttm_device *bdev) +void ttm_bo_fini(struct ttm_buffer_object *bo) { - return cancel_delayed_work_sync(&bdev->wq); -} -EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); - -void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched) -{ - if (resched) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); + ttm_bo_put(bo); } -EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); +EXPORT_SYMBOL(ttm_bo_fini); static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, - struct ttm_resource **mem, struct ttm_operation_ctx *ctx, struct ttm_place *hop) { @@ -499,8 +340,8 @@ static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, struct ttm_resource *hop_mem; int ret; - hop_placement.num_placement = hop_placement.num_busy_placement = 1; - hop_placement.placement = hop_placement.busy_placement = hop; + hop_placement.num_placement = 1; + hop_placement.placement = hop; /* find space in the bounce domain */ ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); @@ -529,11 +370,10 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, dma_resv_assert_held(bo->base.resv); placement.num_placement = 0; - placement.num_busy_placement = 0; bdev->funcs->evict_flags(bo, &placement); - if (!placement.num_placement && !placement.num_busy_placement) { - ret = ttm_bo_wait(bo, true, false); + if (!placement.num_placement) { + ret = ttm_bo_wait_ctx(bo, ctx); if (ret) return ret; @@ -554,25 +394,37 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, goto out; } -bounce: - ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); - if (ret == -EMULTIHOP) { - ret = ttm_bo_bounce_temp_buffer(bo, &evict_mem, ctx, &hop); - if (ret) { + do { + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret != -EMULTIHOP) + break; + + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + } while (!ret); + + if (ret) { + ttm_resource_free(bo, &evict_mem); + if (ret != -ERESTARTSYS && ret != -EINTR) pr_err("Buffer eviction failed\n"); - ttm_resource_free(bo, &evict_mem); - goto out; - } - /* try and move to final place now. */ - goto bounce; } out: return ret; } +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, const struct ttm_place *place) { + struct ttm_resource *res = bo->resource; + struct ttm_device *bdev = bo->bdev; + dma_resv_assert_held(bo->base.resv); if (bo->resource->mem_type == TTM_PL_SYSTEM) return true; @@ -580,349 +432,398 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, /* Don't evict this BO if it's outside of the * requested placement range */ - if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) || - (place->lpfn && place->lpfn <= bo->resource->start)) - return false; - - return true; + return ttm_resource_intersects(bdev, res, place, bo->base.size); } EXPORT_SYMBOL(ttm_bo_eviction_valuable); -/* - * Check the target bo is allowable to be evicted or swapout, including cases: - * - * a. if share same reservation object with ctx->resv, have assumption - * reservation objects should already be locked, so not lock again and - * return true directly when either the opreation allow_reserved_eviction - * or the target bo already is in delayed free list; +/** + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. * - * b. Otherwise, trylock it. + * Return: 0 if successful or the resource disappeared. Negative error code on error. */ -static bool ttm_bo_evict_swapout_allowable(struct ttm_buffer_object *bo, - struct ttm_operation_ctx *ctx, - const struct ttm_place *place, - bool *locked, bool *busy) +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) { - bool ret = false; - - if (bo->base.resv == ctx->resv) { - dma_resv_assert_held(bo->base.resv); - if (ctx->allow_res_evict) - ret = true; - *locked = false; - if (busy) - *busy = false; - } else { - ret = dma_resv_trylock(bo->base.resv); - *locked = ret; - if (busy) - *busy = !ret; + struct ttm_resource_cursor cursor; + struct ttm_buffer_object *bo; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; + + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); + ttm_resource_cursor_fini(&cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (!bo->resource || bo->resource->mem_type != mem_type) + goto out_bo_moved; - if (ret && place && (bo->resource->mem_type != place->mem_type || - !bo->bdev->funcs->eviction_valuable(bo, place))) { - ret = false; - if (*locked) { - dma_resv_unlock(bo->base.resv); - *locked = false; - } + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); + } else { + ret = ttm_bo_evict(bo, ctx); } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + return ret; +out_no_ref: + spin_unlock(&bdev->lru_lock); return ret; } /** - * ttm_mem_evict_wait_busy - wait for a busy BO to become available - * - * @busy_bo: BO which couldn't be locked with trylock - * @ctx: operation context - * @ticket: acquire ticket - * - * Try to lock a busy buffer object to avoid failing eviction. + * struct ttm_bo_evict_walk - Parameters for the evict walk. */ -static int ttm_mem_evict_wait_busy(struct ttm_buffer_object *busy_bo, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; + + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ + bool try_low; + /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ + bool hit_low; +}; + +static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - int r; + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + s64 lret; - if (!busy_bo || !ticket) - return -EBUSY; + if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css, + evict_walk->try_low, &evict_walk->hit_low)) + return 0; - if (ctx->interruptible) - r = dma_resv_lock_interruptible(busy_bo->base.resv, - ticket); - else - r = dma_resv_lock(busy_bo->base.resv, ticket); + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; - /* - * TODO: It would be better to keep the BO locked until allocation is at - * least tried one more time, but that would mean a much larger rework - * of TTM. - */ - if (!r) - dma_resv_unlock(busy_bo->base.resv); + if (bo->deleted) { + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->arg.ctx); + } - return r == -EDEADLK ? -EBUSY : r; -} + if (lret) + goto out; -int ttm_mem_evict_first(struct ttm_device *bdev, - struct ttm_resource_manager *man, - const struct ttm_place *place, - struct ttm_operation_ctx *ctx, - struct ww_acquire_ctx *ticket) -{ - struct ttm_buffer_object *bo = NULL, *busy_bo = NULL; - bool locked = false; - unsigned i; - int ret; + evict_walk->evicted++; + if (evict_walk->res) + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, + evict_walk->res, NULL); + if (lret == 0) + return 1; +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; - spin_lock(&bdev->lru_lock); - for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { - list_for_each_entry(bo, &man->lru[i], lru) { - bool busy; - - if (!ttm_bo_evict_swapout_allowable(bo, ctx, place, - &locked, &busy)) { - if (busy && !busy_bo && ticket != - dma_resv_locking_ctx(bo->base.resv)) - busy_bo = bo; - continue; - } + return lret; +} - if (!ttm_bo_get_unless_zero(bo)) { - if (locked) - dma_resv_unlock(bo->base.resv); - continue; +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; + +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res, + struct dmem_cgroup_pool_state *limit_pool) +{ + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { + .ctx = ctx, + .ticket = ticket, } - break; - } - - /* If the inner loop terminated early, we have our candidate */ - if (&bo->lru != &man->lru[i]) - break; - - bo = NULL; + }, + .place = place, + .evictor = evictor, + .res = res, + .limit_pool = limit_pool, + }; + s64 lret; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + + /* One more attempt if we hit low limit? */ + if (!lret && evict_walk.hit_low) { + evict_walk.try_low = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); } + if (lret || !ticket) + goto out; - if (!bo) { - if (busy_bo && !ttm_bo_get_unless_zero(busy_bo)) - busy_bo = NULL; - spin_unlock(&bdev->lru_lock); - ret = ttm_mem_evict_wait_busy(busy_bo, ctx, ticket); - if (busy_bo) - ttm_bo_put(busy_bo); - return ret; - } + /* Reset low limit */ + evict_walk.try_low = evict_walk.hit_low = false; + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.arg.trylock_only = false; - if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, ctx->interruptible, - ctx->no_wait_gpu, locked); - ttm_bo_put(bo); - return ret; +retry: + do { + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.arg.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); + + /* We hit the low limit? Try once more */ + if (!lret && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; } - - spin_unlock(&bdev->lru_lock); - - ret = ttm_bo_evict(bo, ctx); - if (locked) - ttm_bo_unreserve(bo); - else - ttm_bo_move_to_lru_tail_unlocked(bo); - - ttm_bo_put(bo); - return ret; +out: + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; + return 0; } -/* - * Add the last move fence to the BO and reserve a new shared slot. We only use - * a shared slot to avoid unecessary sync and rely on the subsequent bo move to - * either stall or use an exclusive fence respectively set bo->moving. +/** + * ttm_bo_pin - Pin the buffer object. + * @bo: The buffer object to pin + * + * Make sure the buffer is not evicted any more during memory pressure. + * @bo must be unpinned again by calling ttm_bo_unpin(). */ -static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo, - struct ttm_resource_manager *man, - struct ttm_resource *mem, - bool no_wait_gpu) +void ttm_bo_pin(struct ttm_buffer_object *bo) { - struct dma_fence *fence; - int ret; - - spin_lock(&man->move_lock); - fence = dma_fence_get(man->move); - spin_unlock(&man->move_lock); - - if (!fence) - return 0; - - if (no_wait_gpu) { - ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; - dma_fence_put(fence); - return ret; - } + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + if (!bo->pin_count++ && bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_pin); - dma_resv_add_shared_fence(bo->base.resv, fence); +/** + * ttm_bo_unpin - Unpin the buffer object. + * @bo: The buffer object to unpin + * + * Allows the buffer object to be evicted again during memory pressure. + */ +void ttm_bo_unpin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + if (WARN_ON_ONCE(!bo->pin_count)) + return; - ret = dma_resv_reserve_shared(bo->base.resv, 1); - if (unlikely(ret)) { - dma_fence_put(fence); - return ret; + spin_lock(&bo->bdev->lru_lock); + if (!--bo->pin_count && bo->resource) { + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); } - - dma_fence_put(bo->moving); - bo->moving = fence; - return 0; + spin_unlock(&bo->bdev->lru_lock); } +EXPORT_SYMBOL(ttm_bo_unpin); /* - * Repeatedly evict memory from the LRU for @mem_type until we create enough - * space, or we've evicted everything and there isn't enough space. + * Add the pipelined eviction fencesto the BO as kernel dependency and reserve new + * fence slots. */ -static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, - const struct ttm_place *place, - struct ttm_resource **mem, - struct ttm_operation_ctx *ctx) +static int ttm_bo_add_pipelined_eviction_fences(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + bool no_wait_gpu) { - struct ttm_device *bdev = bo->bdev; - struct ttm_resource_manager *man; - struct ww_acquire_ctx *ticket; - int ret; + struct dma_fence *fence; + int i; - man = ttm_manager_type(bdev, place->mem_type); - ticket = dma_resv_locking_ctx(bo->base.resv); - do { - ret = ttm_resource_alloc(bo, place, mem); - if (likely(!ret)) - break; - if (unlikely(ret != -ENOSPC)) - return ret; - ret = ttm_mem_evict_first(bdev, man, place, ctx, - ticket); - if (unlikely(ret != 0)) - return ret; - } while (1); + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (!fence) + continue; + + if (no_wait_gpu) { + if (!dma_fence_is_signaled(fence)) { + spin_unlock(&man->eviction_lock); + return -EBUSY; + } + } else { + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + } + } + spin_unlock(&man->eviction_lock); - return ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); + /* TODO: this call should be removed. */ + return dma_resv_reserve_fences(bo->base.resv, 1); } -/* - * Creates space for memory region @mem according to its type. +/** + * ttm_bo_alloc_resource - Allocate backing store for a BO * - * This function first searches for free space in compatible memory types in - * the priority order defined by the driver. If free space isn't found, then - * ttm_bo_mem_force_space is attempted in priority order to evict and find - * space. + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @ctx: if and how to sleep, lock buffers and alloc memory + * @force_space: If we should evict buffers to force space + * @res: The resulting struct ttm_resource. + * + * Allocates a resource for the buffer object pointed to by @bo, using the + * placement flags in @placement, potentially evicting other buffer objects when + * @force_space is true. + * This function may sleep while waiting for resources to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == true). + * -ENOSPC: Could not allocate space for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. */ -int ttm_bo_mem_space(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_resource **mem, - struct ttm_operation_ctx *ctx) +static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx, + bool force_space, + struct ttm_resource **res) { struct ttm_device *bdev = bo->bdev; - bool type_found = false; + struct ww_acquire_ctx *ticket; int i, ret; - ret = dma_resv_reserve_shared(bo->base.resv, 1); + ticket = dma_resv_locking_ctx(bo->base.resv); + ret = dma_resv_reserve_fences(bo->base.resv, TTM_NUM_MOVE_FENCES); if (unlikely(ret)) return ret; for (i = 0; i < placement->num_placement; ++i) { const struct ttm_place *place = &placement->placement[i]; + struct dmem_cgroup_pool_state *limit_pool = NULL; struct ttm_resource_manager *man; + bool may_evict; man = ttm_manager_type(bdev, place->mem_type); if (!man || !ttm_resource_manager_used(man)) continue; - type_found = true; - ret = ttm_resource_alloc(bo, place, mem); - if (ret == -ENOSPC) + if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) continue; - if (unlikely(ret)) - goto error; - ret = ttm_bo_add_move_fence(bo, man, *mem, ctx->no_wait_gpu); + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); + ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); + if (ret) { + if (ret != -ENOSPC && ret != -EAGAIN) { + dmem_cgroup_pool_state_put(limit_pool); + return ret; + } + if (!may_evict) { + dmem_cgroup_pool_state_put(limit_pool); + continue; + } + + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res, limit_pool); + dmem_cgroup_pool_state_put(limit_pool); + if (ret == -EBUSY) + continue; + if (ret) + return ret; + } + + ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); if (unlikely(ret)) { - ttm_resource_free(bo, mem); + ttm_resource_free(bo, res); if (ret == -EBUSY) continue; - goto error; + return ret; } return 0; } - for (i = 0; i < placement->num_busy_placement; ++i) { - const struct ttm_place *place = &placement->busy_placement[i]; - struct ttm_resource_manager *man; - - man = ttm_manager_type(bdev, place->mem_type); - if (!man || !ttm_resource_manager_used(man)) - continue; - - type_found = true; - ret = ttm_bo_mem_force_space(bo, place, mem, ctx); - if (likely(!ret)) - return 0; - - if (ret && ret != -EBUSY) - goto error; - } - - ret = -ENOMEM; - if (!type_found) { - pr_err(TTM_PFX "No compatible memory type found\n"); - ret = -EINVAL; - } - -error: - if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count) - ttm_bo_move_to_lru_tail_unlocked(bo); - - return ret; + return -ENOSPC; } -EXPORT_SYMBOL(ttm_bo_mem_space); -static int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_operation_ctx *ctx) +/* + * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @res: The resulting struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Tries both idle allocation and forcefully eviction of buffers. See + * ttm_bo_alloc_resource for details. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource **res, + struct ttm_operation_ctx *ctx) { - struct ttm_resource *mem; - struct ttm_place hop; + bool force_space = false; int ret; - dma_resv_assert_held(bo->base.resv); + do { + ret = ttm_bo_alloc_resource(bo, placement, ctx, + force_space, res); + force_space = !force_space; + } while (ret == -ENOSPC && force_space); - /* - * Determine where to move the buffer. - * - * If driver determines move is going to need - * an extra step then it will return -EMULTIHOP - * and the buffer will be moved to the temporary - * stop and the driver will be called to make - * the second hop. - */ - ret = ttm_bo_mem_space(bo, placement, &mem, ctx); - if (ret) - return ret; -bounce: - ret = ttm_bo_handle_move_mem(bo, mem, false, ctx, &hop); - if (ret == -EMULTIHOP) { - ret = ttm_bo_bounce_temp_buffer(bo, &mem, ctx, &hop); - if (ret) - goto out; - /* try and move to final place now. */ - goto bounce; - } -out: - if (ret) - ttm_resource_free(bo, &mem); return ret; } +EXPORT_SYMBOL(ttm_bo_mem_space); +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ int ttm_bo_validate(struct ttm_buffer_object *bo, struct ttm_placement *placement, struct ttm_operation_ctx *ctx) { + struct ttm_resource *res; + struct ttm_place hop; + bool force_space; int ret; dma_resv_assert_held(bo->base.resv); @@ -930,21 +831,62 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, /* * Remove the backing store if no placement is given. */ - if (!placement->num_placement && !placement->num_busy_placement) + if (!placement->num_placement) return ttm_bo_pipeline_gutting(bo); - /* - * Check whether we need to move buffer. - */ - if (!ttm_resource_compat(bo->resource, placement)) { - ret = ttm_bo_move_buffer(bo, placement, ctx); + force_space = false; + do { + /* Check whether we need to move buffer. */ + if (bo->resource && + ttm_resource_compatible(bo->resource, placement, + force_space)) + return 0; + + /* Moving of pinned BOs is forbidden */ + if (bo->pin_count) + return -EINVAL; + + /* + * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. + */ + ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, + &res); + force_space = !force_space; + if (ret == -ENOSPC) + continue; if (ret) return ret; - } + +bounce: + ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + /* try and move to final place now. */ + if (!ret) + goto bounce; + } + if (ret) { + ttm_resource_free(bo, &res); + return ret; + } + + } while (ret && force_space); + + /* For backward compatibility with userspace */ + if (ret == -ENOSPC) + return bo->bdev->alloc_flags & TTM_ALLOCATION_PROPAGATE_ENOSPC ? + ret : -ENOMEM; + /* * We might need to add a TTM. */ - if (bo->resource->mem_type == TTM_PL_SYSTEM) { + if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { ret = ttm_tt_create(bo, true); if (ret) return ret; @@ -953,96 +895,143 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_validate); -int ttm_bo_init_reserved(struct ttm_device *bdev, - struct ttm_buffer_object *bo, - size_t size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - struct ttm_operation_ctx *ctx, - struct sg_table *sg, - struct dma_resv *resv, +/** + * ttm_bo_init_reserved + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @ctx: TTM operation context for memory allocation. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, enables driver-specific objects + * derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * Furthermore, if resv == NULL, the buffer's reservation lock will be held, + * and it is the caller's responsibility to call ttm_bo_unreserve. + * + * If a failure occurs, the function will call the @destroy function. Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, struct ttm_operation_ctx *ctx, + struct sg_table *sg, struct dma_resv *resv, void (*destroy) (struct ttm_buffer_object *)) { - static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM }; - bool locked; int ret; - bo->destroy = destroy ? destroy : ttm_bo_default_destroy; - kref_init(&bo->kref); - INIT_LIST_HEAD(&bo->lru); - INIT_LIST_HEAD(&bo->ddestroy); bo->bdev = bdev; bo->type = type; - bo->page_alignment = page_alignment; - bo->moving = NULL; + bo->page_alignment = alignment; + bo->destroy = destroy; bo->pin_count = 0; bo->sg = sg; - if (resv) { + bo->bulk_move = NULL; + if (resv) bo->base.resv = resv; - dma_resv_assert_held(bo->base.resv); - } else { + else bo->base.resv = &bo->base._resv; - } atomic_inc(&ttm_glob.bo_count); - ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource); - if (unlikely(ret)) { - ttm_bo_put(bo); - return ret; - } - /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ - if (bo->type == ttm_bo_type_device || - bo->type == ttm_bo_type_sg) + if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, - bo->resource->num_pages); + PFN_UP(bo->base.size)); + if (ret) + goto err_put; + } /* passed reservation objects should already be locked, * since otherwise lockdep will be angered in radeon. */ - if (!resv) { - locked = dma_resv_trylock(bo->base.resv); - WARN_ON(!locked); - } - - if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, ctx); + if (!resv) + WARN_ON(!dma_resv_trylock(bo->base.resv)); + else + dma_resv_assert_held(resv); - if (unlikely(ret)) { - if (!resv) - ttm_bo_unreserve(bo); + ret = ttm_bo_validate(bo, placement, ctx); + if (unlikely(ret)) + goto err_unlock; - ttm_bo_put(bo); - return ret; - } + return 0; - ttm_bo_move_to_lru_tail_unlocked(bo); +err_unlock: + if (!resv) + dma_resv_unlock(bo->base.resv); +err_put: + ttm_bo_put(bo); return ret; } EXPORT_SYMBOL(ttm_bo_init_reserved); -int ttm_bo_init(struct ttm_device *bdev, - struct ttm_buffer_object *bo, - size_t size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - struct sg_table *sg, - struct dma_resv *resv, - void (*destroy) (struct ttm_buffer_object *)) +/** + * ttm_bo_init_validate + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * + * If a failure occurs, the function will call the @destroy function, Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, bool interruptible, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) { struct ttm_operation_ctx ctx = { interruptible, false }; int ret; - ret = ttm_bo_init_reserved(bdev, bo, size, type, placement, - page_alignment, &ctx, sg, resv, destroy); + ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, + sg, resv, destroy); if (ret) return ret; @@ -1051,12 +1040,17 @@ int ttm_bo_init(struct ttm_device *bdev, return 0; } -EXPORT_SYMBOL(ttm_bo_init); +EXPORT_SYMBOL(ttm_bo_init_validate); /* * buffer object vm functions. */ +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) { struct ttm_device *bdev = bo->bdev; @@ -1066,36 +1060,59 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) } EXPORT_SYMBOL(ttm_bo_unmap_virtual); -int ttm_bo_wait(struct ttm_buffer_object *bo, - bool interruptible, bool no_wait) +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - long timeout = 15 * HZ; + long ret; - if (no_wait) { - if (dma_resv_test_signaled(bo->base.resv, true)) + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) return 0; else return -EBUSY; } - timeout = dma_resv_wait_timeout(bo->base.resv, true, interruptible, - timeout); - if (timeout < 0) - return timeout; - - if (timeout == 0) + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) return -EBUSY; - return 0; } -EXPORT_SYMBOL(ttm_bo_wait); +EXPORT_SYMBOL(ttm_bo_wait_ctx); -int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, - gfp_t gfp_flags) +/** + * struct ttm_bo_swapout_walk - Parameters for the swapout walk + */ +struct ttm_bo_swapout_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ + gfp_t gfp_flags; + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ + bool hit_low, evict_low; +}; + +static s64 +ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_place place; - bool locked; - int ret; + struct ttm_place place = {.mem_type = bo->resource->mem_type}; + struct ttm_bo_swapout_walk *swapout_walk = + container_of(walk, typeof(*swapout_walk), walk); + struct ttm_operation_ctx *ctx = walk->arg.ctx; + s64 ret; /* * While the bo may already reside in SYSTEM placement, set @@ -1103,47 +1120,48 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, * The driver may use the fact that we're moving from SYSTEM * as an indication that we're about to swap out. */ - memset(&place, 0, sizeof(place)); - place.mem_type = bo->resource->mem_type; - if (!ttm_bo_evict_swapout_allowable(bo, ctx, &place, &locked, NULL)) - return -EBUSY; + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, &place)) { + ret = -EBUSY; + goto out; + } if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || - bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED || - !ttm_bo_get_unless_zero(bo)) { - if (locked) - dma_resv_unlock(bo->base.resv); - return -EBUSY; + bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) { + ret = -EBUSY; + goto out; } if (bo->deleted) { - ret = ttm_bo_cleanup_refs(bo, false, false, locked); - ttm_bo_put(bo); - return ret == -EBUSY ? -ENOSPC : ret; - } + pgoff_t num_pages = bo->ttm->num_pages; - ttm_bo_move_to_pinned(bo); - /* TODO: Cleanup the locking */ - spin_unlock(&bo->bdev->lru_lock); + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; + + ttm_bo_cleanup_memtype_use(bo); + ret = num_pages; + goto out; + } /* * Move to system cached */ if (bo->resource->mem_type != TTM_PL_SYSTEM) { - struct ttm_operation_ctx ctx = { false, false }; struct ttm_resource *evict_mem; struct ttm_place hop; memset(&hop, 0, sizeof(hop)); place.mem_type = TTM_PL_SYSTEM; - ret = ttm_resource_alloc(bo, &place, &evict_mem); - if (unlikely(ret)) + ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL); + if (ret) goto out; - ret = ttm_bo_handle_move_mem(bo, evict_mem, true, &ctx, &hop); - if (unlikely(ret != 0)) { - WARN(ret == -EMULTIHOP, "Unexpected multihop in swaput - likely driver bug.\n"); + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret) { + WARN(ret == -EMULTIHOP, + "Unexpected multihop in swapout - likely driver bug.\n"); + ttm_resource_free(bo, &evict_mem); goto out; } } @@ -1151,31 +1169,68 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx, /* * Make sure BO is idle. */ - ret = ttm_bo_wait(bo, false, false); - if (unlikely(ret != 0)) + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) goto out; ttm_bo_unmap_virtual(bo); - - /* - * Swap out. Buffer will be swapped in again as soon as - * anyone tries to access a ttm page. - */ if (bo->bdev->funcs->swap_notify) bo->bdev->funcs->swap_notify(bo); - if (ttm_tt_is_populated(bo->ttm)) - ret = ttm_tt_swapout(bo->bdev, bo->ttm, gfp_flags); + if (ttm_tt_is_populated(bo->ttm)) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); + + ret = ttm_tt_swapout(bo->bdev, bo->ttm, swapout_walk->gfp_flags); + + spin_lock(&bo->bdev->lru_lock); + if (ret) + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); + } + out: + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; - /* - * Unreserve without putting on LRU to avoid swapping out an - * already swapped buffer. - */ - if (locked) - dma_resv_unlock(bo->base.resv); - ttm_bo_put(bo); - return ret == -EBUSY ? -ENOSPC : ret; + return ret; +} + +const struct ttm_lru_walk_ops ttm_swap_ops = { + .process_bo = ttm_bo_swapout_cb, +}; + +/** + * ttm_bo_swapout() - Swap out buffer objects on the LRU list to shmem. + * @bdev: The ttm device. + * @ctx: The ttm_operation_ctx governing the swapout operation. + * @man: The resource manager whose resources / buffer objects are + * goint to be swapped out. + * @gfp_flags: The gfp flags used for shmem page allocations. + * @target: The desired number of bytes to swap out. + * + * Return: The number of bytes actually swapped out, or negative error code + * on error. + */ +s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + struct ttm_resource_manager *man, gfp_t gfp_flags, + s64 target) +{ + struct ttm_bo_swapout_walk swapout_walk = { + .walk = { + .ops = &ttm_swap_ops, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, + }, + .gfp_flags = gfp_flags, + }; + + return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target); } void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) @@ -1187,3 +1242,62 @@ void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) ttm_tt_destroy(bo->bdev, bo->ttm); bo->ttm = NULL; } + +/** + * ttm_bo_populate() - Ensure that a buffer object has backing pages + * @bo: The buffer object + * @ctx: The ttm_operation_ctx governing the operation. + * + * For buffer objects in a memory type whose manager uses + * struct ttm_tt for backing pages, ensure those backing pages + * are present and with valid content. The bo's resource is also + * placed on the correct LRU list if it was previously swapped + * out. + * + * Return: 0 if successful, negative error code on failure. + * Note: May return -EINTR or -ERESTARTSYS if @ctx::interruptible + * is set to true. + */ +int ttm_bo_populate(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + struct ttm_tt *tt = bo->ttm; + bool swapped; + int ret; + + dma_resv_assert_held(bo->base.resv); + + if (!tt) + return 0; + + swapped = ttm_tt_is_swapped(tt); + ret = ttm_tt_populate(bo->bdev, tt, ctx); + if (ret) + return ret; + + if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && + bo->resource) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); + } + + return 0; +} +EXPORT_SYMBOL(ttm_bo_populate); + +int ttm_bo_setup_export(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) +{ + int ret; + + ret = ttm_bo_reserve(bo, false, false, NULL); + if (ret != 0) + return ret; + + ret = ttm_bo_populate(bo, ctx); + ttm_bo_unreserve(bo); + return ret; +} +EXPORT_SYMBOL(ttm_bo_setup_export); |