diff options
Diffstat (limited to 'drivers/gpu/drm/ttm')
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_bo_test.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c | 60 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/tests/ttm_mock_manager.c | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_agp_backend.c | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_backup.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo.c | 26 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_internal.h | 58 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_util.c | 245 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_bo_vm.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_device.c | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_execbuf_util.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_pool.c | 31 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_range_manager.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_resource.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/ttm/ttm_tt.c | 1 |
16 files changed, 268 insertions, 198 deletions
diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c index 560d12e50e9e..6c77550c51af 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_test.c @@ -175,7 +175,7 @@ struct signal_timer { static void signal_for_ttm_bo_reserve(struct timer_list *t) { - struct signal_timer *s_timer = from_timer(s_timer, t, timer); + struct signal_timer *s_timer = timer_container_of(s_timer, t, timer); struct task_struct *task = s_timer->ctx->task; do_send_sig_info(SIGTERM, SEND_SIG_PRIV, task, PIDTYPE_PID); diff --git a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c index 3148f5d3dbd6..1bcc67977f48 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c +++ b/drivers/gpu/drm/ttm/tests/ttm_bo_validate_test.c @@ -542,14 +542,15 @@ static void ttm_bo_validate_no_placement_signaled(struct kunit *test) bo->ttm = old_tt; } - err = ttm_resource_alloc(bo, place, &bo->resource, NULL); - KUNIT_EXPECT_EQ(test, err, 0); - KUNIT_ASSERT_EQ(test, man->usage, size); - placement = kunit_kzalloc(test, sizeof(*placement), GFP_KERNEL); KUNIT_ASSERT_NOT_NULL(test, placement); ttm_bo_reserve(bo, false, false, NULL); + + err = ttm_resource_alloc(bo, place, &bo->resource, NULL); + KUNIT_EXPECT_EQ(test, err, 0); + KUNIT_ASSERT_EQ(test, man->usage, size); + err = ttm_bo_validate(bo, placement, &ctx); ttm_bo_unreserve(bo); @@ -757,56 +758,6 @@ static void ttm_bo_validate_move_fence_not_signaled(struct kunit *test) ttm_mock_manager_fini(priv->ttm_dev, snd_mem); } -static void ttm_bo_validate_swapout(struct kunit *test) -{ - unsigned long size_big, size = ALIGN(BO_SIZE, PAGE_SIZE); - enum ttm_bo_type bo_type = ttm_bo_type_device; - struct ttm_buffer_object *bo_small, *bo_big; - struct ttm_test_devices *priv = test->priv; - struct ttm_operation_ctx ctx = { }; - struct ttm_placement *placement; - u32 mem_type = TTM_PL_TT; - struct ttm_place *place; - struct sysinfo si; - int err; - - si_meminfo(&si); - size_big = ALIGN(((u64)si.totalram * si.mem_unit / 2), PAGE_SIZE); - - ttm_mock_manager_init(priv->ttm_dev, mem_type, size_big + size); - - place = ttm_place_kunit_init(test, mem_type, 0); - placement = ttm_placement_kunit_init(test, place, 1); - - bo_small = kunit_kzalloc(test, sizeof(*bo_small), GFP_KERNEL); - KUNIT_ASSERT_NOT_NULL(test, bo_small); - - drm_gem_private_object_init(priv->drm, &bo_small->base, size); - - err = ttm_bo_init_reserved(priv->ttm_dev, bo_small, bo_type, placement, - PAGE_SIZE, &ctx, NULL, NULL, - &dummy_ttm_bo_destroy); - KUNIT_EXPECT_EQ(test, err, 0); - dma_resv_unlock(bo_small->base.resv); - - bo_big = ttm_bo_kunit_init(test, priv, size_big, NULL); - - dma_resv_lock(bo_big->base.resv, NULL); - err = ttm_bo_validate(bo_big, placement, &ctx); - dma_resv_unlock(bo_big->base.resv); - - KUNIT_EXPECT_EQ(test, err, 0); - KUNIT_EXPECT_NOT_NULL(test, bo_big->resource); - KUNIT_EXPECT_EQ(test, bo_big->resource->mem_type, mem_type); - KUNIT_EXPECT_EQ(test, bo_small->resource->mem_type, TTM_PL_SYSTEM); - KUNIT_EXPECT_TRUE(test, bo_small->ttm->page_flags & TTM_TT_FLAG_SWAPPED); - - ttm_bo_put(bo_big); - ttm_bo_put(bo_small); - - ttm_mock_manager_fini(priv->ttm_dev, mem_type); -} - static void ttm_bo_validate_happy_evict(struct kunit *test) { u32 mem_type = TTM_PL_VRAM, mem_multihop = TTM_PL_TT, @@ -1201,7 +1152,6 @@ static struct kunit_case ttm_bo_validate_test_cases[] = { KUNIT_CASE(ttm_bo_validate_move_fence_signaled), KUNIT_CASE_PARAM(ttm_bo_validate_move_fence_not_signaled, ttm_bo_validate_wait_gen_params), - KUNIT_CASE(ttm_bo_validate_swapout), KUNIT_CASE(ttm_bo_validate_happy_evict), KUNIT_CASE(ttm_bo_validate_all_pinned_evict), KUNIT_CASE(ttm_bo_validate_allowed_only_evict), diff --git a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c index b91c13f46225..7aaf0d1395ff 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c +++ b/drivers/gpu/drm/ttm/tests/ttm_kunit_helpers.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_tt.h> #include "ttm_kunit_helpers.h" diff --git a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c index f6d1c8a2845d..d7eb6471f2ed 100644 --- a/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c +++ b/drivers/gpu/drm/ttm/tests/ttm_mock_manager.c @@ -2,6 +2,9 @@ /* * Copyright © 2023 Intel Corporation */ + +#include <linux/export.h> + #include <drm/ttm/ttm_resource.h> #include <drm/ttm/ttm_device.h> #include <drm/ttm/ttm_placement.h> diff --git a/drivers/gpu/drm/ttm/ttm_agp_backend.c b/drivers/gpu/drm/ttm/ttm_agp_backend.c index d27691f2e451..fca0a1a3c6fd 100644 --- a/drivers/gpu/drm/ttm/ttm_agp_backend.c +++ b/drivers/gpu/drm/ttm/ttm_agp_backend.c @@ -36,6 +36,7 @@ #include <drm/ttm/ttm_tt.h> #include <drm/ttm/ttm_resource.h> #include <linux/agp_backend.h> +#include <linux/export.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/io.h> diff --git a/drivers/gpu/drm/ttm/ttm_backup.c b/drivers/gpu/drm/ttm/ttm_backup.c index ffaab68bd5dd..cb1b8e5dadf5 100644 --- a/drivers/gpu/drm/ttm/ttm_backup.c +++ b/drivers/gpu/drm/ttm/ttm_backup.c @@ -4,6 +4,8 @@ */ #include <drm/ttm/ttm_backup.h> + +#include <linux/export.h> #include <linux/page-flags.h> #include <linux/swap.h> diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 08a23ab037cb..f4d9e68b21e7 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -35,6 +35,7 @@ #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> +#include <linux/export.h> #include <linux/jiffies.h> #include <linux/slab.h> #include <linux/sched.h> @@ -46,6 +47,7 @@ #include <linux/dma-resv.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, struct ttm_placement *placement) @@ -524,11 +526,11 @@ static s64 ttm_bo_evict_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object * return 0; if (bo->deleted) { - lret = ttm_bo_wait_ctx(bo, walk->ctx); + lret = ttm_bo_wait_ctx(bo, walk->arg.ctx); if (!lret) ttm_bo_cleanup_memtype_use(bo); } else { - lret = ttm_bo_evict(bo, walk->ctx); + lret = ttm_bo_evict(bo, walk->arg.ctx); } if (lret) @@ -564,8 +566,10 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, struct ttm_bo_evict_walk evict_walk = { .walk = { .ops = &ttm_evict_walk_ops, - .ctx = ctx, - .ticket = ticket, + .arg = { + .ctx = ctx, + .ticket = ticket, + } }, .place = place, .evictor = evictor, @@ -574,7 +578,7 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, }; s64 lret; - evict_walk.walk.trylock_only = true; + evict_walk.walk.arg.trylock_only = true; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); /* One more attempt if we hit low limit? */ @@ -588,12 +592,12 @@ static int ttm_bo_evict_alloc(struct ttm_device *bdev, /* Reset low limit */ evict_walk.try_low = evict_walk.hit_low = false; /* If ticket-locking, repeat while making progress. */ - evict_walk.walk.trylock_only = false; + evict_walk.walk.arg.trylock_only = false; retry: do { /* The walk may clear the evict_walk.walk.ticket field */ - evict_walk.walk.ticket = ticket; + evict_walk.walk.arg.ticket = ticket; evict_walk.evicted = 0; lret = ttm_lru_walk_for_evict(&evict_walk.walk, bdev, man, 1); } while (!lret && evict_walk.evicted); @@ -1104,7 +1108,7 @@ ttm_bo_swapout_cb(struct ttm_lru_walk *walk, struct ttm_buffer_object *bo) struct ttm_place place = {.mem_type = bo->resource->mem_type}; struct ttm_bo_swapout_walk *swapout_walk = container_of(walk, typeof(*swapout_walk), walk); - struct ttm_operation_ctx *ctx = walk->ctx; + struct ttm_operation_ctx *ctx = walk->arg.ctx; s64 ret; /* @@ -1215,8 +1219,10 @@ s64 ttm_bo_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx, struct ttm_bo_swapout_walk swapout_walk = { .walk = { .ops = &ttm_swap_ops, - .ctx = ctx, - .trylock_only = true, + .arg = { + .ctx = ctx, + .trylock_only = true, + }, }, .gfp_flags = gfp_flags, }; diff --git a/drivers/gpu/drm/ttm/ttm_bo_internal.h b/drivers/gpu/drm/ttm/ttm_bo_internal.h new file mode 100644 index 000000000000..9d8b747a34db --- /dev/null +++ b/drivers/gpu/drm/ttm/ttm_bo_internal.h @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + * + * + */ + +#ifndef _TTM_BO_INTERNAL_H_ +#define _TTM_BO_INTERNAL_H_ + +#include <drm/ttm/ttm_bo.h> + +/** + * ttm_bo_get - reference a struct ttm_buffer_object + * + * @bo: The buffer object. + */ +static inline void ttm_bo_get(struct ttm_buffer_object *bo) +{ + kref_get(&bo->kref); +} + +/** + * ttm_bo_get_unless_zero - reference a struct ttm_buffer_object unless + * its refcount has already reached zero. + * @bo: The buffer object. + * + * Used to reference a TTM buffer object in lookups where the object is removed + * from the lookup structure during the destructor and for RCU lookups. + * + * Returns: @bo if the referencing was successful, NULL otherwise. + */ +static inline __must_check struct ttm_buffer_object * +ttm_bo_get_unless_zero(struct ttm_buffer_object *bo) +{ + if (!kref_get_unless_zero(&bo->kref)) + return NULL; + return bo; +} + +#endif diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c index 15cab9bda17f..acbbca9d5c92 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_util.c +++ b/drivers/gpu/drm/ttm/ttm_bo_util.c @@ -28,6 +28,8 @@ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ + +#include <linux/export.h> #include <linux/swap.h> #include <linux/vmalloc.h> @@ -37,6 +39,8 @@ #include <drm/drm_cache.h> +#include "ttm_bo_internal.h" + struct ttm_transfer_obj { struct ttm_buffer_object base; struct ttm_buffer_object *bo; @@ -254,6 +258,13 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, ret = dma_resv_trylock(&fbo->base.base._resv); WARN_ON(!ret); + ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); + if (ret) { + dma_resv_unlock(&fbo->base.base._resv); + kfree(fbo); + return ret; + } + if (fbo->base.resource) { ttm_resource_set_bo(fbo->base.resource, &fbo->base); bo->resource = NULL; @@ -262,12 +273,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, fbo->base.bulk_move = NULL; } - ret = dma_resv_reserve_fences(&fbo->base.base._resv, 1); - if (ret) { - kfree(fbo); - return ret; - } - ttm_bo_get(bo); fbo->bo = bo; @@ -378,6 +383,32 @@ static int ttm_bo_kmap_ttm(struct ttm_buffer_object *bo, } /** + * ttm_bo_kmap_try_from_panic + * + * @bo: The buffer object + * @page: The page to map + * + * Sets up a kernel virtual mapping using kmap_local_page_try_from_panic(). + * This should only be called from the panic handler, if you make sure the bo + * is the one being displayed, so is properly allocated, and protected. + * + * Returns the vaddr, that you can use to write to the bo, and that you should + * pass to kunmap_local() when you're done with this page, or NULL if the bo + * is in iomem. + */ +void *ttm_bo_kmap_try_from_panic(struct ttm_buffer_object *bo, unsigned long page) +{ + if (page + 1 > PFN_UP(bo->resource->size)) + return NULL; + + if (!bo->resource->bus.is_iomem && bo->ttm->pages && bo->ttm->pages[page]) + return kmap_local_page_try_from_panic(bo->ttm->pages[page]); + + return NULL; +} +EXPORT_SYMBOL(ttm_bo_kmap_try_from_panic); + +/** * ttm_bo_kmap * * @bo: The buffer object. @@ -769,14 +800,15 @@ error_destroy_tt: return ret; } -static bool ttm_lru_walk_trylock(struct ttm_operation_ctx *ctx, - struct ttm_buffer_object *bo, - bool *needs_unlock) +static bool ttm_lru_walk_trylock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) { - *needs_unlock = false; + struct ttm_operation_ctx *ctx = curs->arg->ctx; + + curs->needs_unlock = false; if (dma_resv_trylock(bo->base.resv)) { - *needs_unlock = true; + curs->needs_unlock = true; return true; } @@ -788,27 +820,27 @@ static bool ttm_lru_walk_trylock(struct ttm_operation_ctx *ctx, return false; } -static int ttm_lru_walk_ticketlock(struct ttm_lru_walk *walk, - struct ttm_buffer_object *bo, - bool *needs_unlock) +static int ttm_lru_walk_ticketlock(struct ttm_bo_lru_cursor *curs, + struct ttm_buffer_object *bo) { + struct ttm_lru_walk_arg *arg = curs->arg; struct dma_resv *resv = bo->base.resv; int ret; - if (walk->ctx->interruptible) - ret = dma_resv_lock_interruptible(resv, walk->ticket); + if (arg->ctx->interruptible) + ret = dma_resv_lock_interruptible(resv, arg->ticket); else - ret = dma_resv_lock(resv, walk->ticket); + ret = dma_resv_lock(resv, arg->ticket); if (!ret) { - *needs_unlock = true; + curs->needs_unlock = true; /* * Only a single ticketlock per loop. Ticketlocks are prone * to return -EDEADLK causing the eviction to fail, so * after waiting for the ticketlock, revert back to * trylocking for this walk. */ - walk->ticket = NULL; + arg->ticket = NULL; } else if (ret == -EDEADLK) { /* Caller needs to exit the ww transaction. */ ret = -ENOSPC; @@ -817,12 +849,6 @@ static int ttm_lru_walk_ticketlock(struct ttm_lru_walk *walk, return ret; } -static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked) -{ - if (locked) - dma_resv_unlock(bo->base.resv); -} - /** * ttm_lru_walk_for_evict() - Perform a LRU list walk, with actions taken on * valid items. @@ -857,64 +883,21 @@ static void ttm_lru_walk_unlock(struct ttm_buffer_object *bo, bool locked) s64 ttm_lru_walk_for_evict(struct ttm_lru_walk *walk, struct ttm_device *bdev, struct ttm_resource_manager *man, s64 target) { - struct ttm_resource_cursor cursor; - struct ttm_resource *res; + struct ttm_bo_lru_cursor cursor; + struct ttm_buffer_object *bo; s64 progress = 0; s64 lret; - spin_lock(&bdev->lru_lock); - ttm_resource_cursor_init(&cursor, man); - ttm_resource_manager_for_each_res(&cursor, res) { - struct ttm_buffer_object *bo = res->bo; - bool bo_needs_unlock = false; - bool bo_locked = false; - int mem_type; - - /* - * Attempt a trylock before taking a reference on the bo, - * since if we do it the other way around, and the trylock fails, - * we need to drop the lru lock to put the bo. - */ - if (ttm_lru_walk_trylock(walk->ctx, bo, &bo_needs_unlock)) - bo_locked = true; - else if (!walk->ticket || walk->ctx->no_wait_gpu || - walk->trylock_only) - continue; - - if (!ttm_bo_get_unless_zero(bo)) { - ttm_lru_walk_unlock(bo, bo_needs_unlock); - continue; - } - - mem_type = res->mem_type; - spin_unlock(&bdev->lru_lock); - - lret = 0; - if (!bo_locked) - lret = ttm_lru_walk_ticketlock(walk, bo, &bo_needs_unlock); - - /* - * Note that in between the release of the lru lock and the - * ticketlock, the bo may have switched resource, - * and also memory type, since the resource may have been - * freed and allocated again with a different memory type. - * In that case, just skip it. - */ - if (!lret && bo->resource && bo->resource->mem_type == mem_type) - lret = walk->ops->process_bo(walk, bo); - - ttm_lru_walk_unlock(bo, bo_needs_unlock); - ttm_bo_put(bo); + ttm_bo_lru_for_each_reserved_guarded(&cursor, man, &walk->arg, bo) { + lret = walk->ops->process_bo(walk, bo); if (lret == -EBUSY || lret == -EALREADY) lret = 0; progress = (lret < 0) ? lret : progress + lret; - - spin_lock(&bdev->lru_lock); if (progress < 0 || progress >= target) break; } - ttm_resource_cursor_fini(&cursor); - spin_unlock(&bdev->lru_lock); + if (IS_ERR(bo)) + return PTR_ERR(bo); return progress; } @@ -952,44 +935,87 @@ EXPORT_SYMBOL(ttm_bo_lru_cursor_fini); * ttm_bo_lru_cursor_init() - Initialize a struct ttm_bo_lru_cursor * @curs: The ttm_bo_lru_cursor to initialize. * @man: The ttm resource_manager whose LRU lists to iterate over. - * @ctx: The ttm_operation_ctx to govern the locking. + * @arg: The ttm_lru_walk_arg to govern the walk. * - * Initialize a struct ttm_bo_lru_cursor. Currently only trylocking - * or prelocked buffer objects are available as detailed by - * @ctx::resv and @ctx::allow_res_evict. Ticketlocking is not - * supported. + * Initialize a struct ttm_bo_lru_cursor. * * Return: Pointer to @curs. The function does not fail. */ struct ttm_bo_lru_cursor * ttm_bo_lru_cursor_init(struct ttm_bo_lru_cursor *curs, struct ttm_resource_manager *man, - struct ttm_operation_ctx *ctx) + struct ttm_lru_walk_arg *arg) { memset(curs, 0, sizeof(*curs)); ttm_resource_cursor_init(&curs->res_curs, man); - curs->ctx = ctx; + curs->arg = arg; return curs; } EXPORT_SYMBOL(ttm_bo_lru_cursor_init); static struct ttm_buffer_object * -ttm_bo_from_res_reserved(struct ttm_resource *res, struct ttm_bo_lru_cursor *curs) +__ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) { - struct ttm_buffer_object *bo = res->bo; + spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; + struct ttm_resource *res = NULL; + struct ttm_buffer_object *bo; + struct ttm_lru_walk_arg *arg = curs->arg; + bool first = !curs->bo; - if (!ttm_lru_walk_trylock(curs->ctx, bo, &curs->needs_unlock)) - return NULL; + ttm_bo_lru_cursor_cleanup_bo(curs); - if (!ttm_bo_get_unless_zero(bo)) { - if (curs->needs_unlock) - dma_resv_unlock(bo->base.resv); - return NULL; + spin_lock(lru_lock); + for (;;) { + int mem_type, ret = 0; + bool bo_locked = false; + + if (first) { + res = ttm_resource_manager_first(&curs->res_curs); + first = false; + } else { + res = ttm_resource_manager_next(&curs->res_curs); + } + if (!res) + break; + + bo = res->bo; + if (ttm_lru_walk_trylock(curs, bo)) + bo_locked = true; + else if (!arg->ticket || arg->ctx->no_wait_gpu || arg->trylock_only) + continue; + + if (!ttm_bo_get_unless_zero(bo)) { + if (curs->needs_unlock) + dma_resv_unlock(bo->base.resv); + continue; + } + + mem_type = res->mem_type; + spin_unlock(lru_lock); + if (!bo_locked) + ret = ttm_lru_walk_ticketlock(curs, bo); + + /* + * Note that in between the release of the lru lock and the + * ticketlock, the bo may have switched resource, + * and also memory type, since the resource may have been + * freed and allocated again with a different memory type. + * In that case, just skip it. + */ + curs->bo = bo; + if (!ret && bo->resource && bo->resource->mem_type == mem_type) + return bo; + + ttm_bo_lru_cursor_cleanup_bo(curs); + if (ret && ret != -EALREADY) + return ERR_PTR(ret); + + spin_lock(lru_lock); } - curs->bo = bo; - return bo; + spin_unlock(lru_lock); + return res ? bo : NULL; } /** @@ -1003,25 +1029,7 @@ ttm_bo_from_res_reserved(struct ttm_resource *res, struct ttm_bo_lru_cursor *cur */ struct ttm_buffer_object *ttm_bo_lru_cursor_next(struct ttm_bo_lru_cursor *curs) { - spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; - struct ttm_resource *res = NULL; - struct ttm_buffer_object *bo; - - ttm_bo_lru_cursor_cleanup_bo(curs); - - spin_lock(lru_lock); - for (;;) { - res = ttm_resource_manager_next(&curs->res_curs); - if (!res) - break; - - bo = ttm_bo_from_res_reserved(res, curs); - if (bo) - break; - } - - spin_unlock(lru_lock); - return res ? bo : NULL; + return __ttm_bo_lru_cursor_next(curs); } EXPORT_SYMBOL(ttm_bo_lru_cursor_next); @@ -1035,21 +1043,8 @@ EXPORT_SYMBOL(ttm_bo_lru_cursor_next); */ struct ttm_buffer_object *ttm_bo_lru_cursor_first(struct ttm_bo_lru_cursor *curs) { - spinlock_t *lru_lock = &curs->res_curs.man->bdev->lru_lock; - struct ttm_buffer_object *bo; - struct ttm_resource *res; - - spin_lock(lru_lock); - res = ttm_resource_manager_first(&curs->res_curs); - if (!res) { - spin_unlock(lru_lock); - return NULL; - } - - bo = ttm_bo_from_res_reserved(res, curs); - spin_unlock(lru_lock); - - return bo ? bo : ttm_bo_lru_cursor_next(curs); + ttm_bo_lru_cursor_cleanup_bo(curs); + return __ttm_bo_lru_cursor_next(curs); } EXPORT_SYMBOL(ttm_bo_lru_cursor_first); diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index bdfa6ecfef05..b47020fca199 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -31,6 +31,8 @@ #define pr_fmt(fmt) "[TTM] " fmt +#include <linux/export.h> + #include <drm/ttm/ttm_bo.h> #include <drm/ttm/ttm_placement.h> #include <drm/ttm/ttm_tt.h> diff --git a/drivers/gpu/drm/ttm/ttm_device.c b/drivers/gpu/drm/ttm/ttm_device.c index 02e797fd1891..c3e2fcbdd2cc 100644 --- a/drivers/gpu/drm/ttm/ttm_device.c +++ b/drivers/gpu/drm/ttm/ttm_device.c @@ -28,6 +28,7 @@ #define pr_fmt(fmt) "[TTM DEVICE] " fmt #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/mm.h> #include <drm/ttm/ttm_bo.h> @@ -36,6 +37,7 @@ #include <drm/ttm/ttm_placement.h> #include "ttm_module.h" +#include "ttm_bo_internal.h" /* * ttm_global_mutex - protecting the global state @@ -123,6 +125,28 @@ out: return ret; } +/** + * ttm_device_prepare_hibernation - move GTT BOs to shmem for hibernation. + * + * @bdev: A pointer to a struct ttm_device to prepare hibernation for. + * + * Return: 0 on success, negative number on failure. + */ +int ttm_device_prepare_hibernation(struct ttm_device *bdev) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + }; + int ret; + + do { + ret = ttm_device_swapout(bdev, &ctx, GFP_KERNEL); + } while (ret > 0); + return ret; +} +EXPORT_SYMBOL(ttm_device_prepare_hibernation); + /* * A buffer object shrink method that tries to swap out the first * buffer object on the global::swap_lru list. diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c index f1c60fa80c2d..bc7a83a9fe44 100644 --- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c +++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c @@ -26,6 +26,8 @@ * **************************************************************************/ +#include <linux/export.h> + #include <drm/ttm/ttm_execbuf_util.h> #include <drm/ttm/ttm_bo.h> diff --git a/drivers/gpu/drm/ttm/ttm_pool.c b/drivers/gpu/drm/ttm/ttm_pool.c index c2ea865be657..baf27c70a419 100644 --- a/drivers/gpu/drm/ttm/ttm_pool.c +++ b/drivers/gpu/drm/ttm/ttm_pool.c @@ -31,6 +31,7 @@ * cause they are rather slow compared to alloc_pages+map. */ +#include <linux/export.h> #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/debugfs.h> @@ -1132,7 +1133,9 @@ void ttm_pool_fini(struct ttm_pool *pool) } EXPORT_SYMBOL(ttm_pool_fini); -/* As long as pages are available make sure to release at least one */ +/* Free average pool number of pages. */ +#define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) + static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { @@ -1140,9 +1143,12 @@ static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, do num_freed += ttm_pool_shrink(); - while (!num_freed && atomic_long_read(&allocated_pages)); + while (num_freed < sc->nr_to_scan && + atomic_long_read(&allocated_pages)); + + sc->nr_scanned = num_freed; - return num_freed; + return num_freed ?: SHRINK_STOP; } /* Return the number of pages available or SHRINK_EMPTY if we have none */ @@ -1233,7 +1239,7 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) { unsigned int i; - if (!pool->use_dma_alloc) { + if (!pool->use_dma_alloc && pool->nid == NUMA_NO_NODE) { seq_puts(m, "unused\n"); return 0; } @@ -1242,7 +1248,12 @@ int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) spin_lock(&shrinker_lock); for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { - seq_puts(m, "DMA "); + if (!ttm_pool_select_type(pool, i, 0)) + continue; + if (pool->use_dma_alloc) + seq_puts(m, "DMA "); + else + seq_printf(m, "N%d ", pool->nid); switch (i) { case ttm_cached: seq_puts(m, "\t:"); @@ -1266,10 +1277,15 @@ EXPORT_SYMBOL(ttm_pool_debugfs); /* Test the shrinker functions and dump the result */ static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) { - struct shrink_control sc = { .gfp_mask = GFP_NOFS }; + struct shrink_control sc = { + .gfp_mask = GFP_NOFS, + .nr_to_scan = TTM_SHRINKER_BATCH, + }; + unsigned long count; fs_reclaim_acquire(GFP_KERNEL); - seq_printf(m, "%lu/%lu\n", ttm_pool_shrinker_count(mm_shrinker, &sc), + count = ttm_pool_shrinker_count(mm_shrinker, &sc); + seq_printf(m, "%lu/%lu\n", count, ttm_pool_shrinker_scan(mm_shrinker, &sc)); fs_reclaim_release(GFP_KERNEL); @@ -1324,6 +1340,7 @@ int ttm_pool_mgr_init(unsigned long num_pages) mm_shrinker->count_objects = ttm_pool_shrinker_count; mm_shrinker->scan_objects = ttm_pool_shrinker_scan; + mm_shrinker->batch = TTM_SHRINKER_BATCH; mm_shrinker->seeks = 1; shrinker_register(mm_shrinker); diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c index ae11d07eb63a..db854b581d83 100644 --- a/drivers/gpu/drm/ttm/ttm_range_manager.c +++ b/drivers/gpu/drm/ttm/ttm_range_manager.c @@ -34,6 +34,8 @@ #include <drm/ttm/ttm_range_manager.h> #include <drm/ttm/ttm_bo.h> #include <drm/drm_mm.h> + +#include <linux/export.h> #include <linux/slab.h> #include <linux/spinlock.h> diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c index 769b0ca9be47..e2c82ad07eb4 100644 --- a/drivers/gpu/drm/ttm/ttm_resource.c +++ b/drivers/gpu/drm/ttm/ttm_resource.c @@ -23,6 +23,7 @@ */ #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/io-mapping.h> #include <linux/iosys-map.h> #include <linux/scatterlist.h> @@ -557,6 +558,9 @@ int ttm_resource_manager_evict_all(struct ttm_device *bdev, cond_resched(); } while (!ret); + if (ret && ret != -ENOENT) + return ret; + spin_lock(&man->move_lock); fence = dma_fence_get(man->move); spin_unlock(&man->move_lock); diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c index 698cd4bf5e46..506e257dfba8 100644 --- a/drivers/gpu/drm/ttm/ttm_tt.c +++ b/drivers/gpu/drm/ttm/ttm_tt.c @@ -33,6 +33,7 @@ #include <linux/cc_platform.h> #include <linux/debugfs.h> +#include <linux/export.h> #include <linux/file.h> #include <linux/module.h> #include <linux/sched.h> |