diff options
Diffstat (limited to 'drivers/gpu/drm/ttm/ttm_bo.c')
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 2397 |
1 files changed, 964 insertions, 1433 deletions
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index cb9dd674670c..bd27607f8076 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -30,9 +31,13 @@ #define pr_fmt(fmt) "[TTM] " fmt -#include <drm/ttm/ttm_module.h> -#include <drm/ttm/ttm_bo_driver.h> +#include <drm/drm_print.h> +#include <drm/ttm/ttm_allocation.h> +#include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> +#include <drm/ttm/ttm_tt.h> + +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -40,345 +45,133 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/atomic.h> +#include <linux/cgroup_dmem.h> +#include <linux/dma-resv.h> -#define TTM_ASSERT_LOCKED(param) -#define TTM_DEBUG(fmt, arg...) -#define TTM_BO_HASH_ORDER 13 - -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); -static void ttm_bo_global_kobj_release(struct kobject *kobj); - -static struct attribute ttm_bo_count = { - .name = "bo_count", - .mode = S_IRUGO -}; - -static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) -{ - int i; - - for (i = 0; i <= TTM_PL_PRIV5; i++) - if (flags & (1 << i)) { - *mem_type = i; - return 0; - } - return -EINVAL; -} - -static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) -{ - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - - pr_err(" has_type: %d\n", man->has_type); - pr_err(" use_type: %d\n", man->use_type); - pr_err(" flags: 0x%08X\n", man->flags); - pr_err(" gpu_offset: 0x%08lX\n", man->gpu_offset); - pr_err(" size: %llu\n", man->size); - pr_err(" available_caching: 0x%08X\n", man->available_caching); - pr_err(" default_caching: 0x%08X\n", man->default_caching); - if (mem_type != TTM_PL_SYSTEM) - (*man->func->debug)(man, TTM_PFX); -} +#include "ttm_module.h" +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) { - int i, ret, mem_type; + struct drm_printer p = drm_dbg_printer(NULL, DRM_UT_CORE, TTM_PFX); + struct ttm_resource_manager *man; + int i, mem_type; - pr_err("No space for %p (%lu pages, %luK, %luM)\n", - bo, bo->mem.num_pages, bo->mem.size >> 10, - bo->mem.size >> 20); for (i = 0; i < placement->num_placement; i++) { - ret = ttm_mem_type_from_flags(placement->placement[i], - &mem_type); - if (ret) - return; - pr_err(" placement[%d]=0x%08X (%d)\n", - i, placement->placement[i], mem_type); - ttm_mem_type_debug(bo->bdev, mem_type); - } -} - -static ssize_t ttm_bo_global_show(struct kobject *kobj, - struct attribute *attr, - char *buffer) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - return snprintf(buffer, PAGE_SIZE, "%lu\n", - (unsigned long) atomic_read(&glob->bo_count)); -} - -static struct attribute *ttm_bo_global_attrs[] = { - &ttm_bo_count, - NULL -}; - -static const struct sysfs_ops ttm_bo_global_ops = { - .show = &ttm_bo_global_show -}; - -static struct kobj_type ttm_bo_glob_kobj_type = { - .release = &ttm_bo_global_kobj_release, - .sysfs_ops = &ttm_bo_global_ops, - .default_attrs = ttm_bo_global_attrs -}; - - -static inline uint32_t ttm_bo_type_flags(unsigned type) -{ - return 1 << (type); -} - -static void ttm_bo_release_list(struct kref *list_kref) -{ - struct ttm_buffer_object *bo = - container_of(list_kref, struct ttm_buffer_object, list_kref); - struct ttm_bo_device *bdev = bo->bdev; - size_t acc_size = bo->acc_size; - - BUG_ON(atomic_read(&bo->list_kref.refcount)); - BUG_ON(atomic_read(&bo->kref.refcount)); - BUG_ON(atomic_read(&bo->cpu_writers)); - BUG_ON(bo->sync_obj != NULL); - BUG_ON(bo->mem.mm_node != NULL); - BUG_ON(!list_empty(&bo->lru)); - BUG_ON(!list_empty(&bo->ddestroy)); - - if (bo->ttm) - ttm_tt_destroy(bo->ttm); - atomic_dec(&bo->glob->bo_count); - if (bo->resv == &bo->ttm_resv) - reservation_object_fini(&bo->ttm_resv); - - if (bo->destroy) - bo->destroy(bo); - else { - kfree(bo); - } - ttm_mem_global_free(bdev->glob->mem_glob, acc_size); -} - -void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man; - - lockdep_assert_held(&bo->resv->lock.base); - - if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { - - BUG_ON(!list_empty(&bo->lru)); - - man = &bdev->man[bo->mem.mem_type]; - list_add_tail(&bo->lru, &man->lru); - kref_get(&bo->list_kref); - - if (bo->ttm != NULL) { - list_add_tail(&bo->swap, &bo->glob->swap_lru); - kref_get(&bo->list_kref); - } - } -} -EXPORT_SYMBOL(ttm_bo_add_to_lru); - -int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) -{ - int put_count = 0; - - if (!list_empty(&bo->swap)) { - list_del_init(&bo->swap); - ++put_count; + mem_type = placement->placement[i].mem_type; + drm_printf(&p, " placement[%d]=0x%08X (%d)\n", + i, placement->placement[i].flags, mem_type); + man = ttm_manager_type(bo->bdev, mem_type); + ttm_resource_manager_debug(man, &p); } - if (!list_empty(&bo->lru)) { - list_del_init(&bo->lru); - ++put_count; - } - - /* - * TODO: Add a driver hook to delete from - * driver-specific LRU's here. - */ - - return put_count; -} - -static void ttm_bo_ref_bug(struct kref *list_kref) -{ - BUG(); -} - -void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, - bool never_free) -{ - kref_sub(&bo->list_kref, count, - (never_free) ? ttm_bo_ref_bug : ttm_bo_release_list); } -void ttm_bo_del_sub_from_lru(struct ttm_buffer_object *bo) +/** + * ttm_bo_move_to_lru_tail + * + * @bo: The buffer object. + * + * Move this BO to the tail of all lru lists used to lookup and reserve an + * object. This function must be called with struct ttm_global::lru_lock + * held, and is used to make a BO less likely to be considered for eviction. + */ +void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - int put_count; + dma_resv_assert_held(bo->base.resv); - spin_lock(&bo->glob->lru_lock); - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&bo->glob->lru_lock); - ttm_bo_list_ref_sub(bo, put_count, true); + if (bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); } -EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); +EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); -/* - * Call bo->mutex locked. +/** + * ttm_bo_set_bulk_move - update BOs bulk move object + * + * @bo: The buffer object. + * @bulk: bulk move structure + * + * Update the BOs bulk move object, making sure that resources are added/removed + * as well. A bulk move allows to move many resource on the LRU at once, + * resulting in much less overhead of maintaining the LRU. + * The only requirement is that the resources stay together on the LRU and are + * never separated. This is enforces by setting the bulk_move structure on a BO. + * ttm_lru_bulk_move_tail() should be used to move all resources to the tail of + * their LRU list. */ -static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) +void ttm_bo_set_bulk_move(struct ttm_buffer_object *bo, + struct ttm_lru_bulk_move *bulk) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_global *glob = bo->glob; - int ret = 0; - uint32_t page_flags = 0; + dma_resv_assert_held(bo->base.resv); - TTM_ASSERT_LOCKED(&bo->mutex); - bo->ttm = NULL; - - if (bdev->need_dma32) - page_flags |= TTM_PAGE_FLAG_DMA32; - - switch (bo->type) { - case ttm_bo_type_device: - if (zero_alloc) - page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; - case ttm_bo_type_kernel: - bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, - page_flags, glob->dummy_read_page); - if (unlikely(bo->ttm == NULL)) - ret = -ENOMEM; - break; - case ttm_bo_type_sg: - bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, - page_flags | TTM_PAGE_FLAG_SG, - glob->dummy_read_page); - if (unlikely(bo->ttm == NULL)) { - ret = -ENOMEM; - break; - } - bo->ttm->sg = bo->sg; - break; - default: - pr_err("Illegal buffer object type\n"); - ret = -EINVAL; - break; - } + if (bo->bulk_move == bulk) + return; - return ret; + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + bo->bulk_move = bulk; + if (bo->resource) + ttm_resource_add_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); } +EXPORT_SYMBOL(ttm_bo_set_bulk_move); static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, - struct ttm_mem_reg *mem, - bool evict, bool interruptible, - bool no_wait_gpu) + struct ttm_resource *mem, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) { - struct ttm_bo_device *bdev = bo->bdev; - bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); - bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); - struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; - struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; - int ret = 0; + struct ttm_device *bdev = bo->bdev; + bool old_use_tt, new_use_tt; + int ret; - if (old_is_pci || new_is_pci || - ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { - ret = ttm_mem_io_lock(old_man, true); - if (unlikely(ret != 0)) - goto out_err; - ttm_bo_unmap_virtual_locked(bo); - ttm_mem_io_unlock(old_man); - } + old_use_tt = !bo->resource || ttm_manager_type(bdev, bo->resource->mem_type)->use_tt; + new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt; + + ttm_bo_unmap_virtual(bo); /* * Create and bind a ttm if required. */ - if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (bo->ttm == NULL) { - bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); - ret = ttm_bo_add_ttm(bo, zero); - if (ret) - goto out_err; - } - - ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); + if (new_use_tt) { + /* Zero init the new TTM structure if the old location should + * have used one as well. + */ + ret = ttm_tt_create(bo, old_use_tt); if (ret) goto out_err; if (mem->mem_type != TTM_PL_SYSTEM) { - ret = ttm_tt_bind(bo->ttm, mem); + ret = ttm_bo_populate(bo, ctx); if (ret) goto out_err; } - - if (bo->mem.mem_type == TTM_PL_SYSTEM) { - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, mem); - bo->mem = *mem; - mem->mm_node = NULL; - goto moved; - } } - if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, mem); - - if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && - !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) - ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); - else if (bdev->driver->move) - ret = bdev->driver->move(bo, evict, interruptible, - no_wait_gpu, mem); - else - ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); + ret = dma_resv_reserve_fences(bo->base.resv, 1); + if (ret) + goto out_err; + ret = bdev->funcs->move(bo, evict, ctx, mem, hop); if (ret) { - if (bdev->driver->move_notify) { - struct ttm_mem_reg tmp_mem = *mem; - *mem = bo->mem; - bo->mem = tmp_mem; - bdev->driver->move_notify(bo, mem); - bo->mem = *mem; - *mem = tmp_mem; - } - + if (ret == -EMULTIHOP) + return ret; goto out_err; } -moved: - if (bo->evicted) { - ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); - if (ret) - pr_err("Can not flush read caches\n"); - bo->evicted = false; - } - - if (bo->mem.mm_node) { - bo->offset = (bo->mem.start << PAGE_SHIFT) + - bdev->man[bo->mem.mem_type].gpu_offset; - bo->cur_placement = bo->mem.placement; - } else - bo->offset = 0; - + ctx->bytes_moved += bo->base.size; return 0; out_err: - new_man = &bdev->man[bo->mem.mem_type]; - if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { - ttm_tt_unbind(bo->ttm); - ttm_tt_destroy(bo->ttm); - bo->ttm = NULL; - } + if (!old_use_tt) + ttm_bo_tt_destroy(bo); return ret; } -/** +/* * Call bo::reserved. * Will release GPU memory type usage on destruction. * This is the place to put in driver specific hooks to release @@ -388,303 +181,210 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { - if (bo->bdev->driver->move_notify) - bo->bdev->driver->move_notify(bo, NULL); - - if (bo->ttm) { - ttm_tt_unbind(bo->ttm); - ttm_tt_destroy(bo->ttm); - bo->ttm = NULL; - } - ttm_bo_mem_put(bo, &bo->mem); + if (bo->bdev->funcs->delete_mem_notify) + bo->bdev->funcs->delete_mem_notify(bo); - ww_mutex_unlock (&bo->resv->lock); + ttm_bo_tt_destroy(bo); + ttm_resource_free(bo, &bo->resource); } -static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) +static int ttm_bo_individualize_resv(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_global *glob = bo->glob; - struct ttm_bo_driver *driver = bdev->driver; - void *sync_obj = NULL; - int put_count; - int ret; - - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); + int r; - spin_lock(&bdev->fence_lock); - (void) ttm_bo_wait(bo, false, false, true); - if (!ret && !bo->sync_obj) { - spin_unlock(&bdev->fence_lock); - put_count = ttm_bo_del_from_lru(bo); + if (bo->base.resv == &bo->base._resv) + return 0; - spin_unlock(&glob->lru_lock); - ttm_bo_cleanup_memtype_use(bo); + BUG_ON(!dma_resv_trylock(&bo->base._resv)); - ttm_bo_list_ref_sub(bo, put_count, true); + r = dma_resv_copy_fences(&bo->base._resv, bo->base.resv); + dma_resv_unlock(&bo->base._resv); + if (r) + return r; - return; + if (bo->type != ttm_bo_type_sg) { + /* This works because the BO is about to be destroyed and nobody + * reference it any more. The only tricky case is the trylock on + * the resv object while holding the lru_lock. + */ + spin_lock(&bo->bdev->lru_lock); + bo->base.resv = &bo->base._resv; + spin_unlock(&bo->bdev->lru_lock); } - if (bo->sync_obj) - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); - - if (!ret) - ww_mutex_unlock(&bo->resv->lock); - kref_get(&bo->list_kref); - list_add_tail(&bo->ddestroy, &bdev->ddestroy); - spin_unlock(&glob->lru_lock); + return r; +} - if (sync_obj) { - driver->sync_obj_flush(sync_obj); - driver->sync_obj_unref(&sync_obj); +static void ttm_bo_flush_all_fences(struct ttm_buffer_object *bo) +{ + struct dma_resv *resv = &bo->base._resv; + struct dma_resv_iter cursor; + struct dma_fence *fence; + + dma_resv_iter_begin(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) { + if (!fence->ops->signaled) + dma_fence_enable_sw_signaling(fence); } - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); + dma_resv_iter_end(&cursor); } -/** - * function ttm_bo_cleanup_refs_and_unlock - * If bo idle, remove from delayed- and lru lists, and unref. - * If not idle, do nothing. - * - * Must be called with lru_lock and reservation held, this function - * will drop both before returning. - * - * @interruptible Any sleeps should occur interruptibly. - * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. +/* + * Block for the dma_resv object to become idle, lock the buffer and clean up + * the resource and tt object. */ - -static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, - bool interruptible, - bool no_wait_gpu) +static void ttm_bo_delayed_delete(struct work_struct *work) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_bo_driver *driver = bdev->driver; - struct ttm_bo_global *glob = bo->glob; - int put_count; - int ret; - - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, false, true); - - if (ret && !no_wait_gpu) { - void *sync_obj; - - /* - * Take a reference to the fence and unreserve, - * at this point the buffer should be dead, so - * no new sync objects can be attached. - */ - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); - - ww_mutex_unlock(&bo->resv->lock); - spin_unlock(&glob->lru_lock); - - ret = driver->sync_obj_wait(sync_obj, false, interruptible); - driver->sync_obj_unref(&sync_obj); - if (ret) - return ret; - - /* - * remove sync_obj with ttm_bo_wait, the wait should be - * finished, and no new wait object should have been added. - */ - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, false, true); - WARN_ON(ret); - spin_unlock(&bdev->fence_lock); - if (ret) - return ret; - - spin_lock(&glob->lru_lock); - ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); - - /* - * We raced, and lost, someone else holds the reservation now, - * and is probably busy in ttm_bo_cleanup_memtype_use. - * - * Even if it's not the case, because we finished waiting any - * delayed destruction would succeed, so just return success - * here. - */ - if (ret) { - spin_unlock(&glob->lru_lock); - return 0; - } - } else - spin_unlock(&bdev->fence_lock); - - if (ret || unlikely(list_empty(&bo->ddestroy))) { - ww_mutex_unlock(&bo->resv->lock); - spin_unlock(&glob->lru_lock); - return ret; - } + struct ttm_buffer_object *bo; - put_count = ttm_bo_del_from_lru(bo); - list_del_init(&bo->ddestroy); - ++put_count; + bo = container_of(work, typeof(*bo), delayed_delete); - spin_unlock(&glob->lru_lock); + dma_resv_wait_timeout(&bo->base._resv, DMA_RESV_USAGE_BOOKKEEP, false, + MAX_SCHEDULE_TIMEOUT); + dma_resv_lock(bo->base.resv, NULL); ttm_bo_cleanup_memtype_use(bo); - - ttm_bo_list_ref_sub(bo, put_count, true); - - return 0; + dma_resv_unlock(bo->base.resv); + ttm_bo_put(bo); } -/** - * Traverse the delayed list, and call ttm_bo_cleanup_refs on all - * encountered buffers. - */ - -static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) +static void ttm_bo_release(struct kref *kref) { - struct ttm_bo_global *glob = bdev->glob; - struct ttm_buffer_object *entry = NULL; - int ret = 0; - - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - goto out_unlock; - - entry = list_first_entry(&bdev->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&entry->list_kref); + struct ttm_buffer_object *bo = + container_of(kref, struct ttm_buffer_object, kref); + struct ttm_device *bdev = bo->bdev; + int ret; - for (;;) { - struct ttm_buffer_object *nentry = NULL; + WARN_ON_ONCE(bo->pin_count); + WARN_ON_ONCE(bo->bulk_move); - if (entry->ddestroy.next != &bdev->ddestroy) { - nentry = list_first_entry(&entry->ddestroy, - struct ttm_buffer_object, ddestroy); - kref_get(&nentry->list_kref); + if (!bo->deleted) { + ret = ttm_bo_individualize_resv(bo); + if (ret) { + /* Last resort, if we fail to allocate memory for the + * fences block for the BO to become idle + */ + dma_resv_wait_timeout(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, false, + 30 * HZ); } - ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); - if (remove_all && ret) { - spin_unlock(&glob->lru_lock); - ret = ttm_bo_reserve_nolru(entry, false, false, - false, 0); - spin_lock(&glob->lru_lock); - } + if (bo->bdev->funcs->release_notify) + bo->bdev->funcs->release_notify(bo); + + drm_vma_offset_remove(bdev->vma_manager, &bo->base.vma_node); + ttm_mem_io_free(bdev, bo->resource); + + if (!dma_resv_test_signaled(&bo->base._resv, + DMA_RESV_USAGE_BOOKKEEP) || + (want_init_on_free() && (bo->ttm != NULL)) || + bo->type == ttm_bo_type_sg || + !dma_resv_trylock(bo->base.resv)) { + /* The BO is not idle, resurrect it for delayed destroy */ + ttm_bo_flush_all_fences(bo); + bo->deleted = true; + + spin_lock(&bo->bdev->lru_lock); + + /* + * Make pinned bos immediately available to + * shrinkers, now that they are queued for + * destruction. + * + * FIXME: QXL is triggering this. Can be removed when the + * driver is fixed. + */ + if (bo->pin_count) { + bo->pin_count = 0; + ttm_resource_move_to_lru_tail(bo->resource); + } - if (!ret) - ret = ttm_bo_cleanup_refs_and_unlock(entry, false, - !remove_all); - else - spin_unlock(&glob->lru_lock); + kref_init(&bo->kref); + spin_unlock(&bo->bdev->lru_lock); - kref_put(&entry->list_kref, ttm_bo_release_list); - entry = nentry; + INIT_WORK(&bo->delayed_delete, ttm_bo_delayed_delete); - if (ret || !entry) - goto out; + /* Schedule the worker on the closest NUMA node. This + * improves performance since system memory might be + * cleared on free and that is best done on a CPU core + * close to it. + */ + queue_work_node(bdev->pool.nid, bdev->wq, &bo->delayed_delete); + return; + } - spin_lock(&glob->lru_lock); - if (list_empty(&entry->ddestroy)) - break; + ttm_bo_cleanup_memtype_use(bo); + dma_resv_unlock(bo->base.resv); } -out_unlock: - spin_unlock(&glob->lru_lock); -out: - if (entry) - kref_put(&entry->list_kref, ttm_bo_release_list); - return ret; + atomic_dec(&ttm_glob.bo_count); + bo->destroy(bo); } -static void ttm_bo_delayed_workqueue(struct work_struct *work) +/* TODO: remove! */ +void ttm_bo_put(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = - container_of(work, struct ttm_bo_device, wq.work); - - if (ttm_bo_delayed_delete(bdev, false)) { - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); - } + kref_put(&bo->kref, ttm_bo_release); } -static void ttm_bo_release(struct kref *kref) +void ttm_bo_fini(struct ttm_buffer_object *bo) { - struct ttm_buffer_object *bo = - container_of(kref, struct ttm_buffer_object, kref); - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - - write_lock(&bdev->vm_lock); - if (likely(bo->vm_node != NULL)) { - rb_erase(&bo->vm_rb, &bdev->addr_space_rb); - drm_mm_put_block(bo->vm_node); - bo->vm_node = NULL; - } - write_unlock(&bdev->vm_lock); - ttm_mem_io_lock(man, false); - ttm_mem_io_free_vm(bo); - ttm_mem_io_unlock(man); - ttm_bo_cleanup_refs_or_queue(bo); - kref_put(&bo->list_kref, ttm_bo_release_list); + ttm_bo_put(bo); } +EXPORT_SYMBOL(ttm_bo_fini); -void ttm_bo_unref(struct ttm_buffer_object **p_bo) +static int ttm_bo_bounce_temp_buffer(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx, + struct ttm_place *hop) { - struct ttm_buffer_object *bo = *p_bo; - - *p_bo = NULL; - kref_put(&bo->kref, ttm_bo_release); -} -EXPORT_SYMBOL(ttm_bo_unref); + struct ttm_placement hop_placement; + struct ttm_resource *hop_mem; + int ret; -int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) -{ - return cancel_delayed_work_sync(&bdev->wq); -} -EXPORT_SYMBOL(ttm_bo_lock_delayed_workqueue); + hop_placement.num_placement = 1; + hop_placement.placement = hop; -void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) -{ - if (resched) - schedule_delayed_work(&bdev->wq, - ((HZ / 100) < 1) ? 1 : HZ / 100); + /* find space in the bounce domain */ + ret = ttm_bo_mem_space(bo, &hop_placement, &hop_mem, ctx); + if (ret) + return ret; + /* move to the bounce domain */ + ret = ttm_bo_handle_move_mem(bo, hop_mem, false, ctx, NULL); + if (ret) { + ttm_resource_free(bo, &hop_mem); + return ret; + } + return 0; } -EXPORT_SYMBOL(ttm_bo_unlock_delayed_workqueue); -static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, - bool no_wait_gpu) +static int ttm_bo_evict(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_reg evict_mem; + struct ttm_device *bdev = bo->bdev; + struct ttm_resource *evict_mem; struct ttm_placement placement; + struct ttm_place hop; int ret = 0; - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); + memset(&hop, 0, sizeof(hop)); - if (unlikely(ret != 0)) { - if (ret != -ERESTARTSYS) { - pr_err("Failed to expire sync object before buffer eviction\n"); - } - goto out; - } + dma_resv_assert_held(bo->base.resv); - lockdep_assert_held(&bo->resv->lock.base); + placement.num_placement = 0; + bdev->funcs->evict_flags(bo, &placement); + + if (!placement.num_placement) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + return ret; - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.bus.io_reserved_vm = false; - evict_mem.bus.io_reserved_count = 0; + /* + * Since we've already synced, this frees backing store + * immediately. + */ + return ttm_bo_pipeline_gutting(bo); + } - placement.fpfn = 0; - placement.lpfn = 0; - placement.num_placement = 0; - placement.num_busy_placement = 0; - bdev->driver->evict_flags(bo, &placement); - ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, - no_wait_gpu); + ret = ttm_bo_mem_space(bo, &placement, &evict_mem, ctx); if (ret) { if (ret != -ERESTARTSYS) { pr_err("Failed to find memory space for buffer 0x%p eviction\n", @@ -694,360 +394,500 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, goto out; } - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, - no_wait_gpu); + do { + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret != -EMULTIHOP) + break; + + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + } while (!ret); + if (ret) { - if (ret != -ERESTARTSYS) + ttm_resource_free(bo, &evict_mem); + if (ret != -ERESTARTSYS && ret != -EINTR) pr_err("Buffer eviction failed\n"); - ttm_bo_mem_put(bo, &evict_mem); - goto out; } - bo->evicted = true; out: return ret; } -static int ttm_mem_evict_first(struct ttm_bo_device *bdev, - uint32_t mem_type, - bool interruptible, - bool no_wait_gpu) +/** + * ttm_bo_eviction_valuable + * + * @bo: The buffer object to evict + * @place: the placement we need to make room for + * + * Check if it is valuable to evict the BO to make room for the given placement. + */ +bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo, + const struct ttm_place *place) +{ + struct ttm_resource *res = bo->resource; + struct ttm_device *bdev = bo->bdev; + + dma_resv_assert_held(bo->base.resv); + if (bo->resource->mem_type == TTM_PL_SYSTEM) + return true; + + /* Don't evict this BO if it's outside of the + * requested placement range + */ + return ttm_resource_intersects(bdev, res, place, bo->base.size); +} +EXPORT_SYMBOL(ttm_bo_eviction_valuable); + +/** + * ttm_bo_evict_first() - Evict the first bo on the manager's LRU list. + * @bdev: The ttm device. + * @man: The manager whose bo to evict. + * @ctx: The TTM operation ctx governing the eviction. + * + * Return: 0 if successful or the resource disappeared. Negative error code on error. + */ +int ttm_bo_evict_first(struct ttm_device *bdev, struct ttm_resource_manager *man, + struct ttm_operation_ctx *ctx) { - struct ttm_bo_global *glob = bdev->glob; - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; + struct ttm_resource_cursor cursor; struct ttm_buffer_object *bo; - int ret = -EBUSY, put_count; + struct ttm_resource *res; + unsigned int mem_type; + int ret = 0; - spin_lock(&glob->lru_lock); - list_for_each_entry(bo, &man->lru, lru) { - ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); - if (!ret) - break; + spin_lock(&bdev->lru_lock); + ttm_resource_cursor_init(&cursor, man); + res = ttm_resource_manager_first(&cursor); + ttm_resource_cursor_fini(&cursor); + if (!res) { + ret = -ENOENT; + goto out_no_ref; } + bo = res->bo; + if (!ttm_bo_get_unless_zero(bo)) + goto out_no_ref; + mem_type = res->mem_type; + spin_unlock(&bdev->lru_lock); + ret = ttm_bo_reserve(bo, ctx->interruptible, ctx->no_wait_gpu, NULL); + if (ret) + goto out_no_lock; + if (!bo->resource || bo->resource->mem_type != mem_type) + goto out_bo_moved; - if (ret) { - spin_unlock(&glob->lru_lock); - return ret; + if (bo->deleted) { + ret = ttm_bo_wait_ctx(bo, ctx); + if (!ret) + ttm_bo_cleanup_memtype_use(bo); + } else { + ret = ttm_bo_evict(bo, ctx); } +out_bo_moved: + dma_resv_unlock(bo->base.resv); +out_no_lock: + ttm_bo_put(bo); + return ret; - kref_get(&bo->list_kref); +out_no_ref: + spin_unlock(&bdev->lru_lock); + return ret; +} - if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, - no_wait_gpu); - kref_put(&bo->list_kref, ttm_bo_release_list); - return ret; - } +/** + * struct ttm_bo_evict_walk - Parameters for the evict walk. + */ +struct ttm_bo_evict_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @place: The place passed to the resource allocation. */ + const struct ttm_place *place; + /** @evictor: The buffer object we're trying to make room for. */ + struct ttm_buffer_object *evictor; + /** @res: The allocated resource if any. */ + struct ttm_resource **res; + /** @evicted: Number of successful evictions. */ + unsigned long evicted; + + /** @limit_pool: Which pool limit we should test against */ + struct dmem_cgroup_pool_state *limit_pool; + /** @try_low: Whether we should attempt to evict BO's with low watermark threshold */ + bool try_low; + /** @hit_low: If we cannot evict a bo when @try_low is false (first pass) */ + bool hit_low; +}; - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); +static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) +{ + struct ttm_bo_evict_walk *evict_walk = + container_of(walk, typeof(*evict_walk), walk); + s64 lret; - BUG_ON(ret != 0); + if (!dmem_cgroup_state_evict_valuable(evict_walk->limit_pool, bo->resource->css, + evict_walk->try_low, &evict_walk->hit_low)) + return 0; - ttm_bo_list_ref_sub(bo, put_count, true); + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, evict_walk->place)) + return 0; - ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); - ttm_bo_unreserve(bo); + if (bo->deleted) { + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); + if (!lret) + ttm_bo_cleanup_memtype_use(bo); + } else { + lret = ttm_bo_evict(bo, walk->arg.ctx); + } - kref_put(&bo->list_kref, ttm_bo_release_list); - return ret; -} + if (lret) + goto out; -void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) -{ - struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; + evict_walk->evicted++; + if (evict_walk->res) + lret = ttm_resource_alloc(evict_walk->evictor, evict_walk->place, + evict_walk->res, NULL); + if (lret == 0) + return 1; +out: + /* Errors that should terminate the walk. */ + if (lret == -ENOSPC) + return -EBUSY; - if (mem->mm_node) - (*man->func->put_node)(man, mem); + return lret; } -EXPORT_SYMBOL(ttm_bo_mem_put); -/** - * Repeatedly evict memory from the LRU for @mem_type until we create enough - * space, or we've evicted everything and there isn't enough space. - */ -static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, - uint32_t mem_type, - struct ttm_placement *placement, - struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) +static const struct ttm_lru_walk_ops ttm_evict_walk_ops = { + .process_bo = ttm_bo_evict_cb, +}; + +static int ttm_bo_evict_alloc(struct ttm_device *bdev, + struct ttm_resource_manager *man, + const struct ttm_place *place, + struct ttm_buffer_object *evictor, + struct ttm_operation_ctx *ctx, + struct ww_acquire_ctx *ticket, + struct ttm_resource **res, + struct dmem_cgroup_pool_state *limit_pool) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - int ret; + struct ttm_bo_evict_walk evict_walk = { + .walk = { + .ops = &ttm_evict_walk_ops, + .arg = { + .ctx = ctx, + .ticket = ticket, + } + }, + .place = place, + .evictor = evictor, + .res = res, + .limit_pool = limit_pool, + }; + s64 lret; + + evict_walk.walk.arg.trylock_only = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + + /* One more attempt if we hit low limit? */ + if (!lret && evict_walk.hit_low) { + evict_walk.try_low = true; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } + if (lret || !ticket) + goto out; + /* Reset low limit */ + evict_walk.try_low = evict_walk.hit_low = false; + /* If ticket-locking, repeat while making progress. */ + evict_walk.walk.arg.trylock_only = false; + +retry: do { - ret = (*man->func->get_node)(man, bo, placement, mem); - if (unlikely(ret != 0)) - return ret; - if (mem->mm_node) - break; - ret = ttm_mem_evict_first(bdev, mem_type, - interruptible, no_wait_gpu); - if (unlikely(ret != 0)) - return ret; - } while (1); - if (mem->mm_node == NULL) - return -ENOMEM; - mem->mem_type = mem_type; + /* The walk may clear the evict_walk.walk.ticket field */ + evict_walk.walk.arg.ticket = ticket; + evict_walk.evicted = 0; + lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); + } while (!lret && evict_walk.evicted); + + /* We hit the low limit? Try once more */ + if (!lret && evict_walk.hit_low && !evict_walk.try_low) { + evict_walk.try_low = true; + goto retry; + } +out: + if (lret < 0) + return lret; + if (lret == 0) + return -EBUSY; return 0; } -static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, - uint32_t cur_placement, - uint32_t proposed_placement) +/** + * ttm_bo_pin - Pin the buffer object. + * @bo: The buffer object to pin + * + * Make sure the buffer is not evicted any more during memory pressure. + * @bo must be unpinned again by calling ttm_bo_unpin(). + */ +void ttm_bo_pin(struct ttm_buffer_object *bo) { - uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; - uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + spin_lock(&bo->bdev->lru_lock); + if (bo->resource) + ttm_resource_del_bulk_move(bo->resource, bo); + if (!bo->pin_count++ && bo->resource) + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); +} +EXPORT_SYMBOL(ttm_bo_pin); - /** - * Keep current caching if possible. - */ +/** + * ttm_bo_unpin - Unpin the buffer object. + * @bo: The buffer object to unpin + * + * Allows the buffer object to be evicted again during memory pressure. + */ +void ttm_bo_unpin(struct ttm_buffer_object *bo) +{ + dma_resv_assert_held(bo->base.resv); + WARN_ON_ONCE(!kref_read(&bo->kref)); + if (WARN_ON_ONCE(!bo->pin_count)) + return; - if ((cur_placement & caching) != 0) - result |= (cur_placement & caching); - else if ((man->default_caching & caching) != 0) - result |= man->default_caching; - else if ((TTM_PL_FLAG_CACHED & caching) != 0) - result |= TTM_PL_FLAG_CACHED; - else if ((TTM_PL_FLAG_WC & caching) != 0) - result |= TTM_PL_FLAG_WC; - else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) - result |= TTM_PL_FLAG_UNCACHED; - - return result; + spin_lock(&bo->bdev->lru_lock); + if (!--bo->pin_count && bo->resource) { + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + } + spin_unlock(&bo->bdev->lru_lock); } +EXPORT_SYMBOL(ttm_bo_unpin); -static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, - uint32_t mem_type, - uint32_t proposed_placement, - uint32_t *masked_placement) +/* + * Add the pipelined eviction fencesto the BO as kernel dependency and reserve new + * fence slots. + */ +static int ttm_bo_add_pipelined_eviction_fences(struct ttm_buffer_object *bo, + struct ttm_resource_manager *man, + bool no_wait_gpu) { - uint32_t cur_flags = ttm_bo_type_flags(mem_type); - - if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) - return false; + struct dma_fence *fence; + int i; - if ((proposed_placement & man->available_caching) == 0) - return false; + spin_lock(&man->eviction_lock); + for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) { + fence = man->eviction_fences[i]; + if (!fence) + continue; - cur_flags |= (proposed_placement & man->available_caching); + if (no_wait_gpu) { + if (!dma_fence_is_signaled(fence)) { + spin_unlock(&man->eviction_lock); + return -EBUSY; + } + } else { + dma_resv_add_fence(bo->base.resv, fence, DMA_RESV_USAGE_KERNEL); + } + } + spin_unlock(&man->eviction_lock); - *masked_placement = cur_flags; - return true; + /* TODO: this call should be removed. */ + return dma_resv_reserve_fences(bo->base.resv, 1); } /** - * Creates space for memory region @mem according to its type. + * ttm_bo_alloc_resource - Allocate backing store for a BO + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @ctx: if and how to sleep, lock buffers and alloc memory + * @force_space: If we should evict buffers to force space + * @res: The resulting struct ttm_resource. * - * This function first searches for free space in compatible memory types in - * the priority order defined by the driver. If free space isn't found, then - * ttm_bo_mem_force_space is attempted in priority order to evict and find - * space. + * Allocates a resource for the buffer object pointed to by @bo, using the + * placement flags in @placement, potentially evicting other buffer objects when + * @force_space is true. + * This function may sleep while waiting for resources to become available. + * Returns: + * -EBUSY: No space available (only if no_wait == true). + * -ENOSPC: Could not allocate space for the buffer object, either due to + * fragmentation or concurrent allocators. + * -ERESTARTSYS: An interruptible sleep was interrupted by a signal. */ -int ttm_bo_mem_space(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - struct ttm_mem_reg *mem, - bool interruptible, - bool no_wait_gpu) +static int ttm_bo_alloc_resource(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx, + bool force_space, + struct ttm_resource **res) { - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man; - uint32_t mem_type = TTM_PL_SYSTEM; - uint32_t cur_flags = 0; - bool type_found = false; - bool type_ok = false; - bool has_erestartsys = false; + struct ttm_device *bdev = bo->bdev; + struct ww_acquire_ctx *ticket; int i, ret; - mem->mm_node = NULL; - for (i = 0; i < placement->num_placement; ++i) { - ret = ttm_mem_type_from_flags(placement->placement[i], - &mem_type); - if (ret) - return ret; - man = &bdev->man[mem_type]; + ticket = dma_resv_locking_ctx(bo->base.resv); + ret = dma_resv_reserve_fences(bo->base.resv, TTM_NUM_MOVE_FENCES); + if (unlikely(ret)) + return ret; - type_ok = ttm_bo_mt_compatible(man, - mem_type, - placement->placement[i], - &cur_flags); + for (i = 0; i < placement->num_placement; ++i) { + const struct ttm_place *place = &placement->placement[i]; + struct dmem_cgroup_pool_state *limit_pool = NULL; + struct ttm_resource_manager *man; + bool may_evict; - if (!type_ok) + man = ttm_manager_type(bdev, place->mem_type); + if (!man || !ttm_resource_manager_used(man)) continue; - cur_flags = ttm_bo_select_caching(man, bo->mem.placement, - cur_flags); - /* - * Use the access and other non-mapping-related flag bits from - * the memory placement flags to the current flags - */ - ttm_flag_masked(&cur_flags, placement->placement[i], - ~TTM_PL_MASK_MEMTYPE); + if (place->flags & (force_space ? TTM_PL_FLAG_DESIRED : + TTM_PL_FLAG_FALLBACK)) + continue; - if (mem_type == TTM_PL_SYSTEM) - break; + may_evict = (force_space && place->mem_type != TTM_PL_SYSTEM); + ret = ttm_resource_alloc(bo, place, res, force_space ? &limit_pool : NULL); + if (ret) { + if (ret != -ENOSPC && ret != -EAGAIN) { + dmem_cgroup_pool_state_put(limit_pool); + return ret; + } + if (!may_evict) { + dmem_cgroup_pool_state_put(limit_pool); + continue; + } - if (man->has_type && man->use_type) { - type_found = true; - ret = (*man->func->get_node)(man, bo, placement, mem); - if (unlikely(ret)) + ret = ttm_bo_evict_alloc(bdev, man, place, bo, ctx, + ticket, res, limit_pool); + dmem_cgroup_pool_state_put(limit_pool); + if (ret == -EBUSY) + continue; + if (ret) return ret; } - if (mem->mm_node) - break; - } - - if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { - mem->mem_type = mem_type; - mem->placement = cur_flags; - return 0; - } - if (!type_found) - return -EINVAL; + ret = ttm_bo_add_pipelined_eviction_fences(bo, man, ctx->no_wait_gpu); + if (unlikely(ret)) { + ttm_resource_free(bo, res); + if (ret == -EBUSY) + continue; - for (i = 0; i < placement->num_busy_placement; ++i) { - ret = ttm_mem_type_from_flags(placement->busy_placement[i], - &mem_type); - if (ret) return ret; - man = &bdev->man[mem_type]; - if (!man->has_type) - continue; - if (!ttm_bo_mt_compatible(man, - mem_type, - placement->busy_placement[i], - &cur_flags)) - continue; - - cur_flags = ttm_bo_select_caching(man, bo->mem.placement, - cur_flags); - /* - * Use the access and other non-mapping-related flag bits from - * the memory placement flags to the current flags - */ - ttm_flag_masked(&cur_flags, placement->busy_placement[i], - ~TTM_PL_MASK_MEMTYPE); - - - if (mem_type == TTM_PL_SYSTEM) { - mem->mem_type = mem_type; - mem->placement = cur_flags; - mem->mm_node = NULL; - return 0; } - - ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, - interruptible, no_wait_gpu); - if (ret == 0 && mem->mm_node) { - mem->placement = cur_flags; - return 0; - } - if (ret == -ERESTARTSYS) - has_erestartsys = true; + return 0; } - ret = (has_erestartsys) ? -ERESTARTSYS : -ENOMEM; - return ret; -} -EXPORT_SYMBOL(ttm_bo_mem_space); - -int ttm_bo_move_buffer(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) -{ - int ret = 0; - struct ttm_mem_reg mem; - struct ttm_bo_device *bdev = bo->bdev; - - lockdep_assert_held(&bo->resv->lock.base); - /* - * FIXME: It's possible to pipeline buffer moves. - * Have the driver move function wait for idle when necessary, - * instead of doing it here. - */ - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); - spin_unlock(&bdev->fence_lock); - if (ret) - return ret; - mem.num_pages = bo->num_pages; - mem.size = mem.num_pages << PAGE_SHIFT; - mem.page_alignment = bo->mem.page_alignment; - mem.bus.io_reserved_vm = false; - mem.bus.io_reserved_count = 0; - /* - * Determine where to move the buffer. - */ - ret = ttm_bo_mem_space(bo, placement, &mem, - interruptible, no_wait_gpu); - if (ret) - goto out_unlock; - ret = ttm_bo_handle_move_mem(bo, &mem, false, - interruptible, no_wait_gpu); -out_unlock: - if (ret && mem.mm_node) - ttm_bo_mem_put(bo, &mem); - return ret; + return -ENOSPC; } -static int ttm_bo_mem_compat(struct ttm_placement *placement, - struct ttm_mem_reg *mem) +/* + * ttm_bo_mem_space - Wrapper around ttm_bo_alloc_resource + * + * @bo: Pointer to a struct ttm_buffer_object of which we want a resource for + * @placement: Proposed new placement for the buffer object + * @res: The resulting struct ttm_resource. + * @ctx: if and how to sleep, lock buffers and alloc memory + * + * Tries both idle allocation and forcefully eviction of buffers. See + * ttm_bo_alloc_resource for details. + */ +int ttm_bo_mem_space(struct ttm_buffer_object *bo, + struct ttm_placement *placement, + struct ttm_resource **res, + struct ttm_operation_ctx *ctx) { - int i; + bool force_space = false; + int ret; - if (mem->mm_node && placement->lpfn != 0 && - (mem->start < placement->fpfn || - mem->start + mem->num_pages > placement->lpfn)) - return -1; + do { + ret = ttm_bo_alloc_resource(bo, placement, ctx, + force_space, res); + force_space = !force_space; + } while (ret == -ENOSPC && force_space); - for (i = 0; i < placement->num_placement; i++) { - if ((placement->placement[i] & mem->placement & - TTM_PL_MASK_CACHING) && - (placement->placement[i] & mem->placement & - TTM_PL_MASK_MEM)) - return i; - } - return -1; + return ret; } +EXPORT_SYMBOL(ttm_bo_mem_space); +/** + * ttm_bo_validate + * + * @bo: The buffer object. + * @placement: Proposed placement for the buffer object. + * @ctx: validation parameters. + * + * Changes placement and caching policy of the buffer object + * according proposed placement. + * Returns + * -EINVAL on invalid proposed placement. + * -ENOMEM on out-of-memory condition. + * -EBUSY if no_wait is true and buffer busy. + * -ERESTARTSYS if interrupted by a signal. + */ int ttm_bo_validate(struct ttm_buffer_object *bo, - struct ttm_placement *placement, - bool interruptible, - bool no_wait_gpu) + struct ttm_placement *placement, + struct ttm_operation_ctx *ctx) { + struct ttm_resource *res; + struct ttm_place hop; + bool force_space; int ret; - lockdep_assert_held(&bo->resv->lock.base); - /* Check that range is valid */ - if (placement->lpfn || placement->fpfn) - if (placement->fpfn > placement->lpfn || - (placement->lpfn - placement->fpfn) < bo->num_pages) - return -EINVAL; + dma_resv_assert_held(bo->base.resv); + /* - * Check whether we need to move buffer. + * Remove the backing store if no placement is given. */ - ret = ttm_bo_mem_compat(placement, &bo->mem); - if (ret < 0) { - ret = ttm_bo_move_buffer(bo, placement, interruptible, - no_wait_gpu); - if (ret) - return ret; - } else { + if (!placement->num_placement) + return ttm_bo_pipeline_gutting(bo); + + force_space = false; + do { + /* Check whether we need to move buffer. */ + if (bo->resource && + ttm_resource_compatible(bo->resource, placement, + force_space)) + return 0; + + /* Moving of pinned BOs is forbidden */ + if (bo->pin_count) + return -EINVAL; + /* - * Use the access and other non-mapping-related flag bits from - * the compatible memory placement flags to the active flags + * Determine where to move the buffer. + * + * If driver determines move is going to need + * an extra step then it will return -EMULTIHOP + * and the buffer will be moved to the temporary + * stop and the driver will be called to make + * the second hop. */ - ttm_flag_masked(&bo->mem.placement, placement->placement[ret], - ~TTM_PL_MASK_MEMTYPE); - } + ret = ttm_bo_alloc_resource(bo, placement, ctx, force_space, + &res); + force_space = !force_space; + if (ret == -ENOSPC) + continue; + if (ret) + return ret; + +bounce: + ret = ttm_bo_handle_move_mem(bo, res, false, ctx, &hop); + if (ret == -EMULTIHOP) { + ret = ttm_bo_bounce_temp_buffer(bo, ctx, &hop); + /* try and move to final place now. */ + if (!ret) + goto bounce; + } + if (ret) { + ttm_resource_free(bo, &res); + return ret; + } + + } while (ret && force_space); + + /* For backward compatibility with userspace */ + if (ret == -ENOSPC) + return bo->bdev->alloc_flags & TTM_ALLOCATION_PROPAGATE_ENOSPC ? + ret : -ENOMEM; + /* * We might need to add a TTM. */ - if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { - ret = ttm_bo_add_ttm(bo, true); + if (!bo->resource || bo->resource->mem_type == TTM_PL_SYSTEM) { + ret = ttm_tt_create(bo, true); if (ret) return ret; } @@ -1055,718 +895,409 @@ int ttm_bo_validate(struct ttm_buffer_object *bo, } EXPORT_SYMBOL(ttm_bo_validate); -int ttm_bo_check_placement(struct ttm_buffer_object *bo, - struct ttm_placement *placement) -{ - BUG_ON((placement->fpfn || placement->lpfn) && - (bo->mem.num_pages > (placement->lpfn - placement->fpfn))); - - return 0; -} - -int ttm_bo_init(struct ttm_bo_device *bdev, - struct ttm_buffer_object *bo, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - struct file *persistent_swap_storage, - size_t acc_size, - struct sg_table *sg, - void (*destroy) (struct ttm_buffer_object *)) +/** + * ttm_bo_init_reserved + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @ctx: TTM operation context for memory allocation. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, enables driver-specific objects + * derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * Furthermore, if resv == NULL, the buffer's reservation lock will be held, + * and it is the caller's responsibility to call ttm_bo_unreserve. + * + * If a failure occurs, the function will call the @destroy function. Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, struct ttm_operation_ctx *ctx, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) { - int ret = 0; - unsigned long num_pages; - struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; - bool locked; - - ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); - if (ret) { - pr_err("Out of kernel memory\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - return -ENOMEM; - } - - num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; - if (num_pages == 0) { - pr_err("Illegal buffer object size\n"); - if (destroy) - (*destroy)(bo); - else - kfree(bo); - ttm_mem_global_free(mem_glob, acc_size); - return -EINVAL; - } - bo->destroy = destroy; + int ret; kref_init(&bo->kref); - kref_init(&bo->list_kref); - atomic_set(&bo->cpu_writers, 0); - INIT_LIST_HEAD(&bo->lru); - INIT_LIST_HEAD(&bo->ddestroy); - INIT_LIST_HEAD(&bo->swap); - INIT_LIST_HEAD(&bo->io_reserve_lru); bo->bdev = bdev; - bo->glob = bdev->glob; bo->type = type; - bo->num_pages = num_pages; - bo->mem.size = num_pages << PAGE_SHIFT; - bo->mem.mem_type = TTM_PL_SYSTEM; - bo->mem.num_pages = bo->num_pages; - bo->mem.mm_node = NULL; - bo->mem.page_alignment = page_alignment; - bo->mem.bus.io_reserved_vm = false; - bo->mem.bus.io_reserved_count = 0; - bo->priv_flags = 0; - bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); - bo->persistent_swap_storage = persistent_swap_storage; - bo->acc_size = acc_size; + bo->page_alignment = alignment; + bo->destroy = destroy; + bo->pin_count = 0; bo->sg = sg; - bo->resv = &bo->ttm_resv; - reservation_object_init(bo->resv); - atomic_inc(&bo->glob->bo_count); - - ret = ttm_bo_check_placement(bo, placement); + bo->bulk_move = NULL; + if (resv) + bo->base.resv = resv; + else + bo->base.resv = &bo->base._resv; + atomic_inc(&ttm_glob.bo_count); /* * For ttm_bo_type_device buffers, allocate * address space from the device. */ - if (likely(!ret) && - (bo->type == ttm_bo_type_device || - bo->type == ttm_bo_type_sg)) - ret = ttm_bo_setup_vm(bo); - - locked = ww_mutex_trylock(&bo->resv->lock); - WARN_ON(!locked); - - if (likely(!ret)) - ret = ttm_bo_validate(bo, placement, interruptible, false); + if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) { + ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node, + PFN_UP(bo->base.size)); + if (ret) + goto err_put; + } - ttm_bo_unreserve(bo); + /* passed reservation objects should already be locked, + * since otherwise lockdep will be angered in radeon. + */ + if (!resv) + WARN_ON(!dma_resv_trylock(bo->base.resv)); + else + dma_resv_assert_held(resv); + ret = ttm_bo_validate(bo, placement, ctx); if (unlikely(ret)) - ttm_bo_unref(&bo); - - return ret; -} -EXPORT_SYMBOL(ttm_bo_init); - -size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += PAGE_ALIGN(npages * sizeof(void *)); - size += ttm_round_pot(sizeof(struct ttm_tt)); - return size; -} -EXPORT_SYMBOL(ttm_bo_acc_size); - -size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, - unsigned long bo_size, - unsigned struct_size) -{ - unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; - size_t size = 0; - - size += ttm_round_pot(struct_size); - size += PAGE_ALIGN(npages * sizeof(void *)); - size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); - size += ttm_round_pot(sizeof(struct ttm_dma_tt)); - return size; -} -EXPORT_SYMBOL(ttm_bo_dma_acc_size); - -int ttm_bo_create(struct ttm_bo_device *bdev, - unsigned long size, - enum ttm_bo_type type, - struct ttm_placement *placement, - uint32_t page_alignment, - bool interruptible, - struct file *persistent_swap_storage, - struct ttm_buffer_object **p_bo) -{ - struct ttm_buffer_object *bo; - size_t acc_size; - int ret; + goto err_unlock; - bo = kzalloc(sizeof(*bo), GFP_KERNEL); - if (unlikely(bo == NULL)) - return -ENOMEM; + return 0; - acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); - ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, - interruptible, persistent_swap_storage, acc_size, - NULL, NULL); - if (likely(ret == 0)) - *p_bo = bo; +err_unlock: + if (!resv) + dma_resv_unlock(bo->base.resv); +err_put: + ttm_bo_put(bo); return ret; } -EXPORT_SYMBOL(ttm_bo_create); +EXPORT_SYMBOL(ttm_bo_init_reserved); -static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, - unsigned mem_type, bool allow_errors) +/** + * ttm_bo_init_validate + * + * @bdev: Pointer to a ttm_device struct. + * @bo: Pointer to a ttm_buffer_object to be initialized. + * @type: Requested type of buffer object. + * @placement: Initial placement for buffer object. + * @alignment: Data alignment in pages. + * @interruptible: If needing to sleep to wait for GPU resources, + * sleep interruptible. + * pinned in physical memory. If this behaviour is not desired, this member + * holds a pointer to a persistent shmem object. Typically, this would + * point to the shmem object backing a GEM object if TTM is used to back a + * GEM user interface. + * @sg: Scatter-gather table. + * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one. + * @destroy: Destroy function. Use NULL for kfree(). + * + * This function initializes a pre-allocated struct ttm_buffer_object. + * As this object may be part of a larger structure, this function, + * together with the @destroy function, + * enables driver-specific objects derived from a ttm_buffer_object. + * + * On successful return, the caller owns an object kref to @bo. The kref and + * list_kref are usually set to 1, but note that in some situations, other + * tasks may already be holding references to @bo as well. + * + * If a failure occurs, the function will call the @destroy function, Thus, + * after a failure, dereferencing @bo is illegal and will likely cause memory + * corruption. + * + * Returns + * -ENOMEM: Out of memory. + * -EINVAL: Invalid placement flags. + * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources. + */ +int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo, + enum ttm_bo_type type, struct ttm_placement *placement, + uint32_t alignment, bool interruptible, + struct sg_table *sg, struct dma_resv *resv, + void (*destroy) (struct ttm_buffer_object *)) { - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; - struct ttm_bo_global *glob = bdev->glob; + struct ttm_operation_ctx ctx = { interruptible, false }; int ret; - /* - * Can't use standard list traversal since we're unlocking. - */ - - spin_lock(&glob->lru_lock); - while (!list_empty(&man->lru)) { - spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, false, false); - if (ret) { - if (allow_errors) { - return ret; - } else { - pr_err("Cleanup eviction failed\n"); - } - } - spin_lock(&glob->lru_lock); - } - spin_unlock(&glob->lru_lock); - return 0; -} - -int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ - struct ttm_mem_type_manager *man; - int ret = -EINVAL; - - if (mem_type >= TTM_NUM_MEM_TYPES) { - pr_err("Illegal memory type %d\n", mem_type); - return ret; - } - man = &bdev->man[mem_type]; - - if (!man->has_type) { - pr_err("Trying to take down uninitialized memory manager type %u\n", - mem_type); + ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx, + sg, resv, destroy); + if (ret) return ret; - } - - man->use_type = false; - man->has_type = false; - ret = 0; - if (mem_type > 0) { - ttm_bo_force_list_clean(bdev, mem_type, false); + if (!resv) + ttm_bo_unreserve(bo); - ret = (*man->func->takedown)(man); - } - - return ret; + return 0; } -EXPORT_SYMBOL(ttm_bo_clean_mm); - -int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) -{ - struct ttm_mem_type_manager *man = &bdev->man[mem_type]; +EXPORT_SYMBOL(ttm_bo_init_validate); - if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { - pr_err("Illegal memory manager memory type %u\n", mem_type); - return -EINVAL; - } +/* + * buffer object vm functions. + */ - if (!man->has_type) { - pr_err("Memory type %u has not been initialized\n", mem_type); - return 0; - } +/** + * ttm_bo_unmap_virtual + * + * @bo: tear down the virtual mappings for this BO + */ +void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) +{ + struct ttm_device *bdev = bo->bdev; - return ttm_bo_force_list_clean(bdev, mem_type, true); + drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping); + ttm_mem_io_free(bdev, bo->resource); } -EXPORT_SYMBOL(ttm_bo_evict_mm); +EXPORT_SYMBOL(ttm_bo_unmap_virtual); -int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, - unsigned long p_size) +/** + * ttm_bo_wait_ctx - wait for buffer idle. + * + * @bo: The buffer object. + * @ctx: defines how to wait + * + * Waits for the buffer to be idle. Used timeout depends on the context. + * Returns -EBUSY if wait timed outt, -ERESTARTSYS if interrupted by a signal or + * zero on success. + */ +int ttm_bo_wait_ctx(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx) { - int ret = -EINVAL; - struct ttm_mem_type_manager *man; - - BUG_ON(type >= TTM_NUM_MEM_TYPES); - man = &bdev->man[type]; - BUG_ON(man->has_type); - man->io_reserve_fastpath = true; - man->use_io_reserve_lru = false; - mutex_init(&man->io_reserve_mutex); - INIT_LIST_HEAD(&man->io_reserve_lru); - - ret = bdev->driver->init_mem_type(bdev, type, man); - if (ret) - return ret; - man->bdev = bdev; + long ret; - ret = 0; - if (type != TTM_PL_SYSTEM) { - ret = (*man->func->init)(man, p_size); - if (ret) - return ret; + if (ctx->no_wait_gpu) { + if (dma_resv_test_signaled(bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return 0; + else + return -EBUSY; } - man->has_type = true; - man->use_type = true; - man->size = p_size; - - INIT_LIST_HEAD(&man->lru); + ret = dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, 15 * HZ); + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret == 0)) + return -EBUSY; return 0; } -EXPORT_SYMBOL(ttm_bo_init_mm); +EXPORT_SYMBOL(ttm_bo_wait_ctx); -static void ttm_bo_global_kobj_release(struct kobject *kobj) -{ - struct ttm_bo_global *glob = - container_of(kobj, struct ttm_bo_global, kobj); - - ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); - __free_page(glob->dummy_read_page); - kfree(glob); -} - -void ttm_bo_global_release(struct drm_global_reference *ref) -{ - struct ttm_bo_global *glob = ref->object; - - kobject_del(&glob->kobj); - kobject_put(&glob->kobj); -} -EXPORT_SYMBOL(ttm_bo_global_release); +/** + * struct ttm_bo_swapout_walk - Parameters for the swapout walk + */ +struct ttm_bo_swapout_walk { + /** @walk: The walk base parameters. */ + struct ttm_lru_walk walk; + /** @gfp_flags: The gfp flags to use for ttm_tt_swapout() */ + gfp_t gfp_flags; + /** @hit_low: Whether we should attempt to swap BO's with low watermark threshold */ + /** @evict_low: If we cannot swap a bo when @try_low is false (first pass) */ + bool hit_low, evict_low; +}; -int ttm_bo_global_init(struct drm_global_reference *ref) +static s64 +ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) { - struct ttm_bo_global_ref *bo_ref = - container_of(ref, struct ttm_bo_global_ref, ref); - struct ttm_bo_global *glob = ref->object; - int ret; + struct ttm_place place = {.mem_type = bo->resource->mem_type}; + struct ttm_bo_swapout_walk *swapout_walk = + container_of(walk, typeof(*swapout_walk), walk); + struct ttm_operation_ctx *ctx = walk->arg.ctx; + s64 ret; - mutex_init(&glob->device_list_mutex); - spin_lock_init(&glob->lru_lock); - glob->mem_glob = bo_ref->mem_glob; - glob->dummy_read_page = alloc_page(__GFP_ZERO | GFP_DMA32); + /* + * While the bo may already reside in SYSTEM placement, set + * SYSTEM as new placement to cover also the move further below. + * The driver may use the fact that we're moving from SYSTEM + * as an indication that we're about to swap out. + */ + if (bo->pin_count || !bo->bdev->funcs->eviction_valuable(bo, &place)) { + ret = -EBUSY; + goto out; + } - if (unlikely(glob->dummy_read_page == NULL)) { - ret = -ENOMEM; - goto out_no_drp; + if (!bo->ttm || !ttm_tt_is_populated(bo->ttm) || + bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL || + bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED) { + ret = -EBUSY; + goto out; } - INIT_LIST_HEAD(&glob->swap_lru); - INIT_LIST_HEAD(&glob->device_list); + if (bo->deleted) { + pgoff_t num_pages = bo->ttm->num_pages; - ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); - ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); - if (unlikely(ret != 0)) { - pr_err("Could not register buffer object swapout\n"); - goto out_no_shrink; - } + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; - atomic_set(&glob->bo_count, 0); + ttm_bo_cleanup_memtype_use(bo); + ret = num_pages; + goto out; + } - ret = kobject_init_and_add( - &glob->kobj, &ttm_bo_glob_kobj_type, ttm_get_kobj(), "buffer_objects"); - if (unlikely(ret != 0)) - kobject_put(&glob->kobj); - return ret; -out_no_shrink: - __free_page(glob->dummy_read_page); -out_no_drp: - kfree(glob); - return ret; -} -EXPORT_SYMBOL(ttm_bo_global_init); + /* + * Move to system cached + */ + if (bo->resource->mem_type != TTM_PL_SYSTEM) { + struct ttm_resource *evict_mem; + struct ttm_place hop; + memset(&hop, 0, sizeof(hop)); + place.mem_type = TTM_PL_SYSTEM; + ret = ttm_resource_alloc(bo, &place, &evict_mem, NULL); + if (ret) + goto out; -int ttm_bo_device_release(struct ttm_bo_device *bdev) -{ - int ret = 0; - unsigned i = TTM_NUM_MEM_TYPES; - struct ttm_mem_type_manager *man; - struct ttm_bo_global *glob = bdev->glob; - - while (i--) { - man = &bdev->man[i]; - if (man->has_type) { - man->use_type = false; - if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { - ret = -EBUSY; - pr_err("DRM memory manager type %d is not clean\n", - i); - } - man->has_type = false; + ret = ttm_bo_handle_move_mem(bo, evict_mem, true, ctx, &hop); + if (ret) { + WARN(ret == -EMULTIHOP, + "Unexpected multihop in swapout - likely driver bug.\n"); + ttm_resource_free(bo, &evict_mem); + goto out; } } - mutex_lock(&glob->device_list_mutex); - list_del(&bdev->device_list); - mutex_unlock(&glob->device_list_mutex); + /* + * Make sure BO is idle. + */ + ret = ttm_bo_wait_ctx(bo, ctx); + if (ret) + goto out; - cancel_delayed_work_sync(&bdev->wq); + ttm_bo_unmap_virtual(bo); + if (bo->bdev->funcs->swap_notify) + bo->bdev->funcs->swap_notify(bo); - while (ttm_bo_delayed_delete(bdev, true)) - ; + if (ttm_tt_is_populated(bo->ttm)) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_del_bulk_move(bo->resource, bo); + spin_unlock(&bo->bdev->lru_lock); - spin_lock(&glob->lru_lock); - if (list_empty(&bdev->ddestroy)) - TTM_DEBUG("Delayed destroy list was clean\n"); + ret = ttm_tt_swapout(bo->bdev, bo->ttm, swapout_walk->gfp_flags); - if (list_empty(&bdev->man[0].lru)) - TTM_DEBUG("Swap list was clean\n"); - spin_unlock(&glob->lru_lock); + spin_lock(&bo->bdev->lru_lock); + if (ret) + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); + } - BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); - write_lock(&bdev->vm_lock); - drm_mm_takedown(&bdev->addr_space_mm); - write_unlock(&bdev->vm_lock); +out: + /* Consider -ENOMEM and -ENOSPC non-fatal. */ + if (ret == -ENOMEM || ret == -ENOSPC) + ret = -EBUSY; return ret; } -EXPORT_SYMBOL(ttm_bo_device_release); - -int ttm_bo_device_init(struct ttm_bo_device *bdev, - struct ttm_bo_global *glob, - struct ttm_bo_driver *driver, - uint64_t file_page_offset, - bool need_dma32) -{ - int ret = -EINVAL; - - rwlock_init(&bdev->vm_lock); - bdev->driver = driver; - - memset(bdev->man, 0, sizeof(bdev->man)); - /* - * Initialize the system memory buffer type. - * Other types need to be driver / IOCTL initialized. - */ - ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); - if (unlikely(ret != 0)) - goto out_no_sys; - - bdev->addr_space_rb = RB_ROOT; - drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); - - INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); - INIT_LIST_HEAD(&bdev->ddestroy); - bdev->dev_mapping = NULL; - bdev->glob = glob; - bdev->need_dma32 = need_dma32; - bdev->val_seq = 0; - spin_lock_init(&bdev->fence_lock); - mutex_lock(&glob->device_list_mutex); - list_add_tail(&bdev->device_list, &glob->device_list); - mutex_unlock(&glob->device_list_mutex); - - return 0; -out_no_sys: - return ret; -} -EXPORT_SYMBOL(ttm_bo_device_init); +const struct ttm_lru_walk_ops ttm_swap_ops = { + .process_bo = ttm_bo_swapout_cb, +}; -/* - * buffer object vm functions. +/** + * ttm_bo_swapout() - Swap out buffer objects on the LRU list to shmem. + * @bdev: The ttm device. + * @ctx: The ttm_operation_ctx governing the swapout operation. + * @man: The resource manager whose resources / buffer objects are + * goint to be swapped out. + * @gfp_flags: The gfp flags used for shmem page allocations. + * @target: The desired number of bytes to swap out. + * + * Return: The number of bytes actually swapped out, or negative error code + * on error. */ - -bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) +s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, + struct ttm_resource_manager *man, gfp_t gfp_flags, + s64 target) { - struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; - - if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { - if (mem->mem_type == TTM_PL_SYSTEM) - return false; - - if (man->flags & TTM_MEMTYPE_FLAG_CMA) - return false; - - if (mem->placement & TTM_PL_FLAG_CACHED) - return false; - } - return true; + struct ttm_bo_swapout_walk swapout_walk = { + .walk = { + .ops = &ttm_swap_ops, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, + }, + .gfp_flags = gfp_flags, + }; + + return ttm_lru_walk_for_evict(&swapout_walk.walk, bdev, man, target); } -void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) +void ttm_bo_tt_destroy(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; - loff_t offset = (loff_t) bo->addr_space_offset; - loff_t holelen = ((loff_t) bo->mem.num_pages) << PAGE_SHIFT; - - if (!bdev->dev_mapping) + if (bo->ttm == NULL) return; - unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); - ttm_mem_io_free_vm(bo); -} - -void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - - ttm_mem_io_lock(man, false); - ttm_bo_unmap_virtual_locked(bo); - ttm_mem_io_unlock(man); -} - -EXPORT_SYMBOL(ttm_bo_unmap_virtual); - -static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - struct rb_node **cur = &bdev->addr_space_rb.rb_node; - struct rb_node *parent = NULL; - struct ttm_buffer_object *cur_bo; - unsigned long offset = bo->vm_node->start; - unsigned long cur_offset; - - while (*cur) { - parent = *cur; - cur_bo = rb_entry(parent, struct ttm_buffer_object, vm_rb); - cur_offset = cur_bo->vm_node->start; - if (offset < cur_offset) - cur = &parent->rb_left; - else if (offset > cur_offset) - cur = &parent->rb_right; - else - BUG(); - } - - rb_link_node(&bo->vm_rb, parent, cur); - rb_insert_color(&bo->vm_rb, &bdev->addr_space_rb); + ttm_tt_unpopulate(bo->bdev, bo->ttm); + ttm_tt_destroy(bo->bdev, bo->ttm); + bo->ttm = NULL; } /** - * ttm_bo_setup_vm: + * ttm_bo_populate() - Ensure that a buffer object has backing pages + * @bo: The buffer object + * @ctx: The ttm_operation_ctx governing the operation. * - * @bo: the buffer to allocate address space for + * For buffer objects in a memory type whose manager uses + * struct ttm_tt for backing pages, ensure those backing pages + * are present and with valid content. The bo's resource is also + * placed on the correct LRU list if it was previously swapped + * out. * - * Allocate address space in the drm device so that applications - * can mmap the buffer and access the contents. This only - * applies to ttm_bo_type_device objects as others are not - * placed in the drm device address space. + * Return: 0 if successful, negative error code on failure. + * Note: May return -EINTR or -ERESTARTSYS if @ctx::interruptible + * is set to true. */ - -static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) +int ttm_bo_populate(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { - struct ttm_bo_device *bdev = bo->bdev; + struct ttm_tt *tt = bo->ttm; + bool swapped; int ret; -retry_pre_get: - ret = drm_mm_pre_get(&bdev->addr_space_mm); - if (unlikely(ret != 0)) - return ret; - - write_lock(&bdev->vm_lock); - bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, - bo->mem.num_pages, 0, 0); + dma_resv_assert_held(bo->base.resv); - if (unlikely(bo->vm_node == NULL)) { - ret = -ENOMEM; - goto out_unlock; - } - - bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, - bo->mem.num_pages, 0); - - if (unlikely(bo->vm_node == NULL)) { - write_unlock(&bdev->vm_lock); - goto retry_pre_get; - } - - ttm_bo_vm_insert_rb(bo); - write_unlock(&bdev->vm_lock); - bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; - - return 0; -out_unlock: - write_unlock(&bdev->vm_lock); - return ret; -} - -int ttm_bo_wait(struct ttm_buffer_object *bo, - bool lazy, bool interruptible, bool no_wait) -{ - struct ttm_bo_driver *driver = bo->bdev->driver; - struct ttm_bo_device *bdev = bo->bdev; - void *sync_obj; - int ret = 0; - - if (likely(bo->sync_obj == NULL)) + if (!tt) return 0; - while (bo->sync_obj) { - - if (driver->sync_obj_signaled(bo->sync_obj)) { - void *tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&tmp_obj); - spin_lock(&bdev->fence_lock); - continue; - } - - if (no_wait) - return -EBUSY; + swapped = ttm_tt_is_swapped(tt); + ret = ttm_tt_populate(bo->bdev, tt, ctx); + if (ret) + return ret; - sync_obj = driver->sync_obj_ref(bo->sync_obj); - spin_unlock(&bdev->fence_lock); - ret = driver->sync_obj_wait(sync_obj, - lazy, interruptible); - if (unlikely(ret != 0)) { - driver->sync_obj_unref(&sync_obj); - spin_lock(&bdev->fence_lock); - return ret; - } - spin_lock(&bdev->fence_lock); - if (likely(bo->sync_obj == sync_obj)) { - void *tmp_obj = bo->sync_obj; - bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, - &bo->priv_flags); - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&sync_obj); - driver->sync_obj_unref(&tmp_obj); - spin_lock(&bdev->fence_lock); - } else { - spin_unlock(&bdev->fence_lock); - driver->sync_obj_unref(&sync_obj); - spin_lock(&bdev->fence_lock); - } + if (swapped && !ttm_tt_is_swapped(tt) && !bo->pin_count && + bo->resource) { + spin_lock(&bo->bdev->lru_lock); + ttm_resource_add_bulk_move(bo->resource, bo); + ttm_resource_move_to_lru_tail(bo->resource); + spin_unlock(&bo->bdev->lru_lock); } - return 0; -} -EXPORT_SYMBOL(ttm_bo_wait); - -int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) -{ - struct ttm_bo_device *bdev = bo->bdev; - int ret = 0; - - /* - * Using ttm_bo_reserve makes sure the lru lists are updated. - */ - ret = ttm_bo_reserve(bo, true, no_wait, false, 0); - if (unlikely(ret != 0)) - return ret; - spin_lock(&bdev->fence_lock); - ret = ttm_bo_wait(bo, false, true, no_wait); - spin_unlock(&bdev->fence_lock); - if (likely(ret == 0)) - atomic_inc(&bo->cpu_writers); - ttm_bo_unreserve(bo); - return ret; + return 0; } -EXPORT_SYMBOL(ttm_bo_synccpu_write_grab); +EXPORT_SYMBOL(ttm_bo_populate); -void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) +int ttm_bo_setup_export(struct ttm_buffer_object *bo, + struct ttm_operation_ctx *ctx) { - atomic_dec(&bo->cpu_writers); -} -EXPORT_SYMBOL(ttm_bo_synccpu_write_release); - -/** - * A buffer object shrink method that tries to swap out the first - * buffer object on the bo_global::swap_lru list. - */ - -static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) -{ - struct ttm_bo_global *glob = - container_of(shrink, struct ttm_bo_global, shrink); - struct ttm_buffer_object *bo; - int ret = -EBUSY; - int put_count; - uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); - - spin_lock(&glob->lru_lock); - list_for_each_entry(bo, &glob->swap_lru, swap) { - ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); - if (!ret) - break; - } - - if (ret) { - spin_unlock(&glob->lru_lock); - return ret; - } - - kref_get(&bo->list_kref); + int ret; - if (!list_empty(&bo->ddestroy)) { - ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); - kref_put(&bo->list_kref, ttm_bo_release_list); + ret = ttm_bo_reserve(bo, false, false, NULL); + if (ret != 0) return ret; - } - - put_count = ttm_bo_del_from_lru(bo); - spin_unlock(&glob->lru_lock); - ttm_bo_list_ref_sub(bo, put_count, true); - - /** - * Wait for GPU, then move to system cached. - */ - - spin_lock(&bo->bdev->fence_lock); - ret = ttm_bo_wait(bo, false, false, false); - spin_unlock(&bo->bdev->fence_lock); - - if (unlikely(ret != 0)) - goto out; - - if ((bo->mem.placement & swap_placement) != swap_placement) { - struct ttm_mem_reg evict_mem; - - evict_mem = bo->mem; - evict_mem.mm_node = NULL; - evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; - evict_mem.mem_type = TTM_PL_SYSTEM; - - ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, - false, false); - if (unlikely(ret != 0)) - goto out; - } - - ttm_bo_unmap_virtual(bo); - - /** - * Swap out. Buffer will be swapped in again as soon as - * anyone tries to access a ttm page. - */ - - if (bo->bdev->driver->swap_notify) - bo->bdev->driver->swap_notify(bo); - - ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); -out: - - /** - * - * Unreserve without putting on LRU to avoid swapping out an - * already swapped buffer. - */ - - ww_mutex_unlock(&bo->resv->lock); - kref_put(&bo->list_kref, ttm_bo_release_list); + ret = ttm_bo_populate(bo, ctx); + ttm_bo_unreserve(bo); return ret; } - -void ttm_bo_swapout_all(struct ttm_bo_device *bdev) -{ - while (ttm_bo_swapout(&bdev->glob->shrink) == 0) - ; -} -EXPORT_SYMBOL(ttm_bo_swapout_all); +EXPORT_SYMBOL(ttm_bo_setup_export); |