From 66257db7a5276d7f1aff8850388a0eb675ffa2e1 Mon Sep 17 00:00:00 2001 From: Nicolai Hähnle Date: Thu, 15 Dec 2016 17:23:49 +0100 Subject: drm/ttm: add evict parameter to ttm_bo_driver::move_notify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ensure that the driver can listen to evictions even when they don't take the path through ttm_bo_driver::move. This is crucial for amdgpu, which relies on an eviction counter to skip re-binding page tables when possible. Signed-off-by: Nicolai Hähnle Reviewed-by: Chunming Zhou Reviewed-by: Christian König Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index d5063618efa7..d4973e9c126e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -342,7 +342,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, if (bo->mem.mem_type == TTM_PL_SYSTEM) { if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, mem); + bdev->driver->move_notify(bo, evict, mem); bo->mem = *mem; mem->mm_node = NULL; goto moved; @@ -350,7 +350,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, } if (bdev->driver->move_notify) - bdev->driver->move_notify(bo, mem); + bdev->driver->move_notify(bo, evict, mem); if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) @@ -366,7 +366,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, struct ttm_mem_reg tmp_mem = *mem; *mem = bo->mem; bo->mem = tmp_mem; - bdev->driver->move_notify(bo, mem); + bdev->driver->move_notify(bo, false, mem); bo->mem = *mem; *mem = tmp_mem; } @@ -414,7 +414,7 @@ out_err: static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) { if (bo->bdev->driver->move_notify) - bo->bdev->driver->move_notify(bo, NULL); + bo->bdev->driver->move_notify(bo, false, NULL); ttm_tt_destroy(bo->ttm); bo->ttm = NULL; -- cgit From 2ee7fc92cfd327fe41377f64a7f04ddc30c851e7 Mon Sep 17 00:00:00 2001 From: Christian König Date: Fri, 6 Jan 2017 19:16:07 +0100 Subject: drm/ttm: remove allow_errors parameter from ttm_bo_force_list_clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not allowing errors here is completely pointless and actually dangerous cause trying to continue on an error can cause an endless loop. Reviewed-by: Sinclair Yeh Signed-off-by: Christian König Reviewed-by: Roger.He Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index d4973e9c126e..89bbcf0300f4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1291,7 +1291,7 @@ int ttm_bo_create(struct ttm_bo_device *bdev, EXPORT_SYMBOL(ttm_bo_create); static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, - unsigned mem_type, bool allow_errors) + unsigned mem_type) { struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_bo_global *glob = bdev->glob; @@ -1306,13 +1306,8 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, while (!list_empty(&man->lru)) { spin_unlock(&glob->lru_lock); ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); - if (ret) { - if (allow_errors) { - return ret; - } else { - pr_err("Cleanup eviction failed\n"); - } - } + if (ret) + return ret; spin_lock(&glob->lru_lock); } spin_unlock(&glob->lru_lock); @@ -1324,13 +1319,8 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, if (fence) { ret = dma_fence_wait(fence, false); dma_fence_put(fence); - if (ret) { - if (allow_errors) { - return ret; - } else { - pr_err("Cleanup eviction failed\n"); - } - } + if (ret) + return ret; } return 0; @@ -1359,7 +1349,11 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) ret = 0; if (mem_type > 0) { - ttm_bo_force_list_clean(bdev, mem_type, false); + ret = ttm_bo_force_list_clean(bdev, mem_type); + if (ret) { + pr_err("Cleanup eviction failed\n"); + return ret; + } ret = (*man->func->takedown)(man); } @@ -1382,7 +1376,7 @@ int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) return 0; } - return ttm_bo_force_list_clean(bdev, mem_type, true); + return ttm_bo_force_list_clean(bdev, mem_type); } EXPORT_SYMBOL(ttm_bo_evict_mm); -- cgit From cf6c467d67d319e239aec57d7ba31cb9946f29bf Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 10 Jan 2017 14:08:28 +0100 Subject: drm/ttm: add BO priorities for the LRUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This way the driver can specify a priority for a BO which has the effect that a BO is only evicted when all other BOs with a lower priority are evicted first. Reviewed-by: Sinclair Yeh Signed-off-by: Christian König Reviewed-by: Roger.He Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 67 +++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 23 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 89bbcf0300f4..0b287d8947a4 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -242,13 +242,13 @@ EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo) { - return bo->bdev->man[bo->mem.mem_type].lru.prev; + return bo->bdev->man[bo->mem.mem_type].lru[bo->priority].prev; } EXPORT_SYMBOL(ttm_bo_default_lru_tail); struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo) { - return bo->glob->swap_lru.prev; + return bo->glob->swap_lru[bo->priority].prev; } EXPORT_SYMBOL(ttm_bo_default_swap_lru_tail); @@ -741,20 +741,27 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev, struct ttm_mem_type_manager *man = &bdev->man[mem_type]; struct ttm_buffer_object *bo; int ret = -EBUSY, put_count; + unsigned i; spin_lock(&glob->lru_lock); - list_for_each_entry(bo, &man->lru, lru) { - ret = __ttm_bo_reserve(bo, false, true, NULL); - if (ret) - continue; + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + list_for_each_entry(bo, &man->lru[i], lru) { + ret = __ttm_bo_reserve(bo, false, true, NULL); + if (ret) + continue; - if (place && !bdev->driver->eviction_valuable(bo, place)) { - __ttm_bo_unreserve(bo); - ret = -EBUSY; - continue; + if (place && !bdev->driver->eviction_valuable(bo, + place)) { + __ttm_bo_unreserve(bo); + ret = -EBUSY; + continue; + } + + break; } - break; + if (!ret) + break; } if (ret) { @@ -1197,6 +1204,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, } atomic_inc(&bo->glob->bo_count); drm_vma_node_reset(&bo->vma_node); + bo->priority = 0; /* * For ttm_bo_type_device buffers, allocate @@ -1297,18 +1305,21 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, struct ttm_bo_global *glob = bdev->glob; struct dma_fence *fence; int ret; + unsigned i; /* * Can't use standard list traversal since we're unlocking. */ spin_lock(&glob->lru_lock); - while (!list_empty(&man->lru)) { - spin_unlock(&glob->lru_lock); - ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); - if (ret) - return ret; - spin_lock(&glob->lru_lock); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + while (!list_empty(&man->lru[i])) { + spin_unlock(&glob->lru_lock); + ret = ttm_mem_evict_first(bdev, mem_type, NULL, false, false); + if (ret) + return ret; + spin_lock(&glob->lru_lock); + } } spin_unlock(&glob->lru_lock); @@ -1385,6 +1396,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, { int ret = -EINVAL; struct ttm_mem_type_manager *man; + unsigned i; BUG_ON(type >= TTM_NUM_MEM_TYPES); man = &bdev->man[type]; @@ -1410,7 +1422,8 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, man->use_type = true; man->size = p_size; - INIT_LIST_HEAD(&man->lru); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&man->lru[i]); man->move = NULL; return 0; @@ -1442,6 +1455,7 @@ int ttm_bo_global_init(struct drm_global_reference *ref) container_of(ref, struct ttm_bo_global_ref, ref); struct ttm_bo_global *glob = ref->object; int ret; + unsigned i; mutex_init(&glob->device_list_mutex); spin_lock_init(&glob->lru_lock); @@ -1453,7 +1467,8 @@ int ttm_bo_global_init(struct drm_global_reference *ref) goto out_no_drp; } - INIT_LIST_HEAD(&glob->swap_lru); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + INIT_LIST_HEAD(&glob->swap_lru[i]); INIT_LIST_HEAD(&glob->device_list); ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); @@ -1512,8 +1527,9 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) if (list_empty(&bdev->ddestroy)) TTM_DEBUG("Delayed destroy list was clean\n"); - if (list_empty(&bdev->man[0].lru)) - TTM_DEBUG("Swap list was clean\n"); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) + if (list_empty(&bdev->man[0].lru[0])) + TTM_DEBUG("Swap list %d was clean\n", i); spin_unlock(&glob->lru_lock); drm_vma_offset_manager_destroy(&bdev->vma_manager); @@ -1665,10 +1681,15 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) int ret = -EBUSY; int put_count; uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); + unsigned i; spin_lock(&glob->lru_lock); - list_for_each_entry(bo, &glob->swap_lru, swap) { - ret = __ttm_bo_reserve(bo, false, true, NULL); + for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { + list_for_each_entry(bo, &glob->swap_lru[i], swap) { + ret = __ttm_bo_reserve(bo, false, true, NULL); + if (!ret) + break; + } if (!ret) break; } -- cgit From 260498f2cdfcf3a3a266b212a86c9059623ffa32 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 12 Jan 2017 11:50:13 +0100 Subject: drm/ttm: revert "implement LRU add callbacks v2" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The additional housekeeping had too much CPU overhead, let's use the BO priorities instead. agd: also revert hibmc changes Reviewed-by: Sinclair Yeh Signed-off-by: Christian König Reviewed-and-Tested-by: Roger.He Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 0b287d8947a4..8c5c64d52af2 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -163,6 +163,7 @@ static void ttm_bo_release_list(struct kref *list_kref) void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; + struct ttm_mem_type_manager *man; lockdep_assert_held(&bo->resv->lock.base); @@ -170,11 +171,13 @@ void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) BUG_ON(!list_empty(&bo->lru)); - list_add(&bo->lru, bdev->driver->lru_tail(bo)); + man = &bdev->man[bo->mem.mem_type]; + list_add_tail(&bo->lru, &man->lru[bo->priority]); kref_get(&bo->list_kref); if (bo->ttm && !(bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { - list_add(&bo->swap, bdev->driver->swap_lru_tail(bo)); + list_add_tail(&bo->swap, + &bo->glob->swap_lru[bo->priority]); kref_get(&bo->list_kref); } } @@ -240,18 +243,6 @@ void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) } EXPORT_SYMBOL(ttm_bo_move_to_lru_tail); -struct list_head *ttm_bo_default_lru_tail(struct ttm_buffer_object *bo) -{ - return bo->bdev->man[bo->mem.mem_type].lru[bo->priority].prev; -} -EXPORT_SYMBOL(ttm_bo_default_lru_tail); - -struct list_head *ttm_bo_default_swap_lru_tail(struct ttm_buffer_object *bo) -{ - return bo->glob->swap_lru[bo->priority].prev; -} -EXPORT_SYMBOL(ttm_bo_default_swap_lru_tail); - /* * Call bo->mutex locked. */ -- cgit From 896d630d240e90850fdb094f75445823fe0e0849 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 12 Jan 2017 11:54:11 +0100 Subject: drm/ttm: revert "add optional LRU removal callback v2" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the custom LRU management the callback is not used any more. agd: fix trivial warning Reviewed-by: Sinclair Yeh Signed-off-by: Christian König Reviewed-and-Tested-by: Roger.He Signed-off-by: Alex Deucher --- drivers/gpu/drm/ttm/ttm_bo.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'drivers/gpu/drm/ttm') diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index 8c5c64d52af2..e7ab0264001d 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -186,12 +186,8 @@ EXPORT_SYMBOL(ttm_bo_add_to_lru); int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; int put_count = 0; - if (bdev->driver->lru_removal) - bdev->driver->lru_removal(bo); - if (!list_empty(&bo->swap)) { list_del_init(&bo->swap); ++put_count; @@ -201,6 +197,11 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) ++put_count; } + /* + * TODO: Add a driver hook to delete from + * driver-specific LRU's here. + */ + return put_count; } @@ -229,14 +230,10 @@ EXPORT_SYMBOL(ttm_bo_del_sub_from_lru); void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; int put_count = 0; lockdep_assert_held(&bo->resv->lock.base); - if (bdev->driver->lru_removal) - bdev->driver->lru_removal(bo); - put_count = ttm_bo_del_from_lru(bo); ttm_bo_list_ref_sub(bo, put_count, true); ttm_bo_add_to_lru(bo); -- cgit