From 2440ff2c9151120c8ae27de6565b11831ee07e08 Mon Sep 17 00:00:00 2001 From: Junwei Zhang Date: Sat, 10 Oct 2015 08:48:42 +0800 Subject: drm/amdgpu: add timer to fence to detect scheduler lockup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I67e987db0efdca28faa80b332b75571192130d33 Signed-off-by: Junwei Zhang Reviewed-by: David Zhou Reviewed-by: Christian König --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index b3fc26c59787..fcad7e060938 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -628,8 +628,20 @@ int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring) init_waitqueue_head(&ring->fence_drv.fence_queue); if (amdgpu_enable_scheduler) { + long timeout = msecs_to_jiffies(amdgpu_lockup_timeout); + if (timeout == 0) { + /* + * FIXME: + * Delayed workqueue cannot use it directly, + * so the scheduler will not use delayed workqueue if + * MAX_SCHEDULE_TIMEOUT is set. + * Currently keep it simple and silly. + */ + timeout = MAX_SCHEDULE_TIMEOUT; + } r = amd_sched_init(&ring->sched, &amdgpu_sched_ops, - amdgpu_sched_hw_submission, ring->name); + amdgpu_sched_hw_submission, + timeout, ring->name); if (r) { DRM_ERROR("Failed to create scheduler on ring %s.\n", ring->name); -- cgit From b7e4dad3e1fc5d3909737fb72e57aedeb0072c7f Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 1 Sep 2015 10:50:26 +0200 Subject: drm/amdgpu: remove old lockup detection infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It didn't worked to well anyway. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Reviewed-by: Junwei Zhang --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index fcad7e060938..4010aa6b4e53 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -269,17 +269,6 @@ static void amdgpu_fence_check_lockup(struct work_struct *work) if (amdgpu_fence_activity(ring)) { wake_up_all(&ring->fence_drv.fence_queue); } - else if (amdgpu_ring_is_lockup(ring)) { - /* good news we believe it's a lockup */ - dev_warn(ring->adev->dev, "GPU lockup (current fence id " - "0x%016llx last fence id 0x%016llx on ring %d)\n", - (uint64_t)atomic64_read(&fence_drv->last_seq), - fence_drv->sync_seq[ring->idx], ring->idx); - - /* remember that we need an reset */ - ring->adev->needs_reset = true; - wake_up_all(&ring->fence_drv.fence_queue); - } up_read(&ring->adev->exclusive_lock); } @@ -380,7 +369,6 @@ static bool amdgpu_fence_enable_signaling(struct fence *f) */ static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq) { - struct amdgpu_device *adev = ring->adev; bool signaled = false; BUG_ON(!ring); @@ -391,8 +379,7 @@ static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq) return 0; wait_event(ring->fence_drv.fence_queue, ( - (signaled = amdgpu_fence_seq_signaled(ring, seq)) - || adev->needs_reset)); + (signaled = amdgpu_fence_seq_signaled(ring, seq)))); if (signaled) return 0; @@ -939,11 +926,6 @@ signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, if (amdgpu_test_signaled_any(array, count)) break; - if (adev->needs_reset) { - t = -EDEADLK; - break; - } - t = schedule_timeout(t); if (t > 0 && intr && signal_pending(current)) -- cgit From 0c418f10104d4aa1d6b83698790898dc9ef1c12d Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 1 Sep 2015 15:13:53 +0200 Subject: drm/amdgpu: remove the exclusive lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finally getting rid of it. Signed-off-by: Christian König --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 4010aa6b4e53..1fadc15e64ae 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -260,16 +260,8 @@ static void amdgpu_fence_check_lockup(struct work_struct *work) lockup_work.work); ring = fence_drv->ring; - if (!down_read_trylock(&ring->adev->exclusive_lock)) { - /* just reschedule the check if a reset is going on */ - amdgpu_fence_schedule_check(ring); - return; - } - - if (amdgpu_fence_activity(ring)) { + if (amdgpu_fence_activity(ring)) wake_up_all(&ring->fence_drv.fence_queue); - } - up_read(&ring->adev->exclusive_lock); } /** @@ -317,18 +309,15 @@ static bool amdgpu_fence_is_signaled(struct fence *f) { struct amdgpu_fence *fence = to_amdgpu_fence(f); struct amdgpu_ring *ring = fence->ring; - struct amdgpu_device *adev = ring->adev; if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) return true; - if (down_read_trylock(&adev->exclusive_lock)) { - amdgpu_fence_process(ring); - up_read(&adev->exclusive_lock); + amdgpu_fence_process(ring); + + if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) + return true; - if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) - return true; - } return false; } -- cgit From ce16b0e5a32a157abd6446214e8b91c55064204e Mon Sep 17 00:00:00 2001 From: Junwei Zhang Date: Sun, 6 Sep 2015 13:55:03 +0800 Subject: drm/amdgpu: remove the unnecessary parameter adev for amdgpu_fence_wait_any() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Junwei Zhang Reviewed-by: Christian König --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 1fadc15e64ae..003a219943f1 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -857,16 +857,12 @@ static void amdgpu_fence_wait_cb(struct fence *fence, struct fence_cb *cb) static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, signed long t) { - struct amdgpu_fence *fence = to_amdgpu_fence(f); - struct amdgpu_device *adev = fence->ring->adev; - - return amdgpu_fence_wait_any(adev, &f, 1, intr, t); + return amdgpu_fence_wait_any(&f, 1, intr, t); } /** * Wait the fence array with timeout * - * @adev: amdgpu device * @array: the fence array with amdgpu fence pointer * @count: the number of the fence array * @intr: when sleep, set the current task interruptable or not @@ -874,8 +870,7 @@ static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, * * It will return when any fence is signaled or timeout. */ -signed long amdgpu_fence_wait_any(struct amdgpu_device *adev, - struct fence **array, uint32_t count, +signed long amdgpu_fence_wait_any(struct fence **array, uint32_t count, bool intr, signed long t) { struct amdgpu_wait_cb *cb; -- cgit From 318cd340c5573a1f021f5b7711893133fe5e8480 Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 15 Oct 2015 17:58:09 +0200 Subject: drm/amdgpu: remove unneeded fence functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit amdgpu_fence_default_wait isn't needed any more the default wait does the same thing and amdgpu_test_signaled is dead as well. Signed-off-by: Christian König Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 003a219943f1..663caa9d1175 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -822,11 +822,6 @@ static const char *amdgpu_fence_get_timeline_name(struct fence *f) return (const char *)fence->ring->name; } -static inline bool amdgpu_test_signaled(struct amdgpu_fence *fence) -{ - return test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->base.flags); -} - static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count) { int idx; @@ -854,12 +849,6 @@ static void amdgpu_fence_wait_cb(struct fence *fence, struct fence_cb *cb) wake_up_process(wait->task); } -static signed long amdgpu_fence_default_wait(struct fence *f, bool intr, - signed long t) -{ - return amdgpu_fence_wait_any(&f, 1, intr, t); -} - /** * Wait the fence array with timeout * @@ -936,6 +925,6 @@ const struct fence_ops amdgpu_fence_ops = { .get_timeline_name = amdgpu_fence_get_timeline_name, .enable_signaling = amdgpu_fence_enable_signaling, .signaled = amdgpu_fence_is_signaled, - .wait = amdgpu_fence_default_wait, + .wait = fence_default_wait, .release = NULL, }; -- cgit From ee327caf1a2cdba9313167c36db2d7ff02d534bc Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 20 Oct 2015 17:38:07 +0200 Subject: drm/amdgpu: switch to common fence_wait_any_timeout v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No need to duplicate the functionality any more. v2: fix handling if no fence is available. Signed-off-by: Christian König Reviewed-by: Alex Deucher (v1) --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 98 ------------------------------- 1 file changed, 98 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 663caa9d1175..c4bb28292b9f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -822,104 +822,6 @@ static const char *amdgpu_fence_get_timeline_name(struct fence *f) return (const char *)fence->ring->name; } -static bool amdgpu_test_signaled_any(struct fence **fences, uint32_t count) -{ - int idx; - struct fence *fence; - - for (idx = 0; idx < count; ++idx) { - fence = fences[idx]; - if (fence) { - if (test_bit(FENCE_FLAG_SIGNALED_BIT, &fence->flags)) - return true; - } - } - return false; -} - -struct amdgpu_wait_cb { - struct fence_cb base; - struct task_struct *task; -}; - -static void amdgpu_fence_wait_cb(struct fence *fence, struct fence_cb *cb) -{ - struct amdgpu_wait_cb *wait = - container_of(cb, struct amdgpu_wait_cb, base); - wake_up_process(wait->task); -} - -/** - * Wait the fence array with timeout - * - * @array: the fence array with amdgpu fence pointer - * @count: the number of the fence array - * @intr: when sleep, set the current task interruptable or not - * @t: timeout to wait - * - * It will return when any fence is signaled or timeout. - */ -signed long amdgpu_fence_wait_any(struct fence **array, uint32_t count, - bool intr, signed long t) -{ - struct amdgpu_wait_cb *cb; - struct fence *fence; - unsigned idx; - - BUG_ON(!array); - - cb = kcalloc(count, sizeof(struct amdgpu_wait_cb), GFP_KERNEL); - if (cb == NULL) { - t = -ENOMEM; - goto err_free_cb; - } - - for (idx = 0; idx < count; ++idx) { - fence = array[idx]; - if (fence) { - cb[idx].task = current; - if (fence_add_callback(fence, - &cb[idx].base, amdgpu_fence_wait_cb)) { - /* The fence is already signaled */ - goto fence_rm_cb; - } - } - } - - while (t > 0) { - if (intr) - set_current_state(TASK_INTERRUPTIBLE); - else - set_current_state(TASK_UNINTERRUPTIBLE); - - /* - * amdgpu_test_signaled_any must be called after - * set_current_state to prevent a race with wake_up_process - */ - if (amdgpu_test_signaled_any(array, count)) - break; - - t = schedule_timeout(t); - - if (t > 0 && intr && signal_pending(current)) - t = -ERESTARTSYS; - } - - __set_current_state(TASK_RUNNING); - -fence_rm_cb: - for (idx = 0; idx < count; ++idx) { - fence = array[idx]; - if (fence && cb[idx].base.func) - fence_remove_callback(fence, &cb[idx].base); - } - -err_free_cb: - kfree(cb); - - return t; -} - const struct fence_ops amdgpu_fence_ops = { .get_driver_name = amdgpu_fence_get_driver_name, .get_timeline_name = amdgpu_fence_get_timeline_name, -- cgit From 6ef68c17d40e1e7e291ca513627a0d2a13ae095f Mon Sep 17 00:00:00 2001 From: Christian König Date: Thu, 22 Oct 2015 15:16:22 +0200 Subject: drm/amdgpu: remove amdgpu_fence_ref/unref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just move the remaining users to fence_put/get. Signed-off-by: Christian König Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 30 ------------------------------ 1 file changed, 30 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index c4bb28292b9f..b912539420e4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -416,36 +416,6 @@ int amdgpu_fence_wait_empty(struct amdgpu_ring *ring) return amdgpu_fence_ring_wait_seq(ring, seq); } -/** - * amdgpu_fence_ref - take a ref on a fence - * - * @fence: amdgpu fence object - * - * Take a reference on a fence (all asics). - * Returns the fence. - */ -struct amdgpu_fence *amdgpu_fence_ref(struct amdgpu_fence *fence) -{ - fence_get(&fence->base); - return fence; -} - -/** - * amdgpu_fence_unref - remove a ref on a fence - * - * @fence: amdgpu fence object - * - * Remove a reference on a fence (all asics). - */ -void amdgpu_fence_unref(struct amdgpu_fence **fence) -{ - struct amdgpu_fence *tmp = *fence; - - *fence = NULL; - if (tmp) - fence_put(&tmp->base); -} - /** * amdgpu_fence_count_emitted - get the count of emitted fences * -- cgit From 24372447e5cfe65d43a462acbf201c7882551b84 Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 3 Nov 2015 11:26:42 +0100 Subject: drm/amdgpu: fix fence fallback check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interrupts are notorious unreliable, enable the fallback at a couple of more places. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index b912539420e4..2c485a22e14b 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -342,6 +342,7 @@ static bool amdgpu_fence_enable_signaling(struct fence *f) fence->fence_wake.func = amdgpu_fence_check_signaled; __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake); fence_get(f); + amdgpu_fence_schedule_check(ring); FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); return true; } @@ -367,6 +368,7 @@ static int amdgpu_fence_ring_wait_seq(struct amdgpu_ring *ring, uint64_t seq) if (atomic64_read(&ring->fence_drv.last_seq) >= seq) return 0; + amdgpu_fence_schedule_check(ring); wait_event(ring->fence_drv.fence_queue, ( (signaled = amdgpu_fence_seq_signaled(ring, seq)))); -- cgit From a95e264254dca5b6bfb331d5902930d0787bd7e1 Mon Sep 17 00:00:00 2001 From: Christian König Date: Tue, 3 Nov 2015 12:21:57 +0100 Subject: drm/amdgpu: group together common fence implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And also add some missing function documentation. No functional change. Signed-off-by: Christian König Reviewed-by: Chunming Zhou Reviewed-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c | 206 ++++++++++++++++-------------- 1 file changed, 109 insertions(+), 97 deletions(-) (limited to 'drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c index 2c485a22e14b..257d72205bb5 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c @@ -136,42 +136,6 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, void *owner, return 0; } -/** - * amdgpu_fence_check_signaled - callback from fence_queue - * - * this function is called with fence_queue lock held, which is also used - * for the fence locking itself, so unlocked variants are used for - * fence_signal, and remove_wait_queue. - */ -static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) -{ - struct amdgpu_fence *fence; - struct amdgpu_device *adev; - u64 seq; - int ret; - - fence = container_of(wait, struct amdgpu_fence, fence_wake); - adev = fence->ring->adev; - - /* - * We cannot use amdgpu_fence_process here because we're already - * in the waitqueue, in a call from wake_up_all. - */ - seq = atomic64_read(&fence->ring->fence_drv.last_seq); - if (seq >= fence->seq) { - ret = fence_signal_locked(&fence->base); - if (!ret) - FENCE_TRACE(&fence->base, "signaled from irq context\n"); - else - FENCE_TRACE(&fence->base, "was already signaled\n"); - - __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake); - fence_put(&fence->base); - } else - FENCE_TRACE(&fence->base, "pending\n"); - return 0; -} - /** * amdgpu_fence_activity - check for fence activity * @@ -305,48 +269,6 @@ static bool amdgpu_fence_seq_signaled(struct amdgpu_ring *ring, u64 seq) return false; } -static bool amdgpu_fence_is_signaled(struct fence *f) -{ - struct amdgpu_fence *fence = to_amdgpu_fence(f); - struct amdgpu_ring *ring = fence->ring; - - if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) - return true; - - amdgpu_fence_process(ring); - - if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) - return true; - - return false; -} - -/** - * amdgpu_fence_enable_signaling - enable signalling on fence - * @fence: fence - * - * This function is called with fence_queue lock held, and adds a callback - * to fence_queue that checks if this fence is signaled, and if so it - * signals the fence and removes itself. - */ -static bool amdgpu_fence_enable_signaling(struct fence *f) -{ - struct amdgpu_fence *fence = to_amdgpu_fence(f); - struct amdgpu_ring *ring = fence->ring; - - if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) - return false; - - fence->fence_wake.flags = 0; - fence->fence_wake.private = NULL; - fence->fence_wake.func = amdgpu_fence_check_signaled; - __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake); - fence_get(f); - amdgpu_fence_schedule_check(ring); - FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); - return true; -} - /* * amdgpu_ring_wait_seq_timeout - wait for seq of the specific ring to signal * @ring: ring to wait on for the seq number @@ -733,6 +655,115 @@ void amdgpu_fence_driver_force_completion(struct amdgpu_device *adev) } } +/* + * Common fence implementation + */ + +static const char *amdgpu_fence_get_driver_name(struct fence *fence) +{ + return "amdgpu"; +} + +static const char *amdgpu_fence_get_timeline_name(struct fence *f) +{ + struct amdgpu_fence *fence = to_amdgpu_fence(f); + return (const char *)fence->ring->name; +} + +/** + * amdgpu_fence_is_signaled - test if fence is signaled + * + * @f: fence to test + * + * Test the fence sequence number if it is already signaled. If it isn't + * signaled start fence processing. Returns True if the fence is signaled. + */ +static bool amdgpu_fence_is_signaled(struct fence *f) +{ + struct amdgpu_fence *fence = to_amdgpu_fence(f); + struct amdgpu_ring *ring = fence->ring; + + if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) + return true; + + amdgpu_fence_process(ring); + + if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) + return true; + + return false; +} + +/** + * amdgpu_fence_check_signaled - callback from fence_queue + * + * this function is called with fence_queue lock held, which is also used + * for the fence locking itself, so unlocked variants are used for + * fence_signal, and remove_wait_queue. + */ +static int amdgpu_fence_check_signaled(wait_queue_t *wait, unsigned mode, int flags, void *key) +{ + struct amdgpu_fence *fence; + struct amdgpu_device *adev; + u64 seq; + int ret; + + fence = container_of(wait, struct amdgpu_fence, fence_wake); + adev = fence->ring->adev; + + /* + * We cannot use amdgpu_fence_process here because we're already + * in the waitqueue, in a call from wake_up_all. + */ + seq = atomic64_read(&fence->ring->fence_drv.last_seq); + if (seq >= fence->seq) { + ret = fence_signal_locked(&fence->base); + if (!ret) + FENCE_TRACE(&fence->base, "signaled from irq context\n"); + else + FENCE_TRACE(&fence->base, "was already signaled\n"); + + __remove_wait_queue(&fence->ring->fence_drv.fence_queue, &fence->fence_wake); + fence_put(&fence->base); + } else + FENCE_TRACE(&fence->base, "pending\n"); + return 0; +} + +/** + * amdgpu_fence_enable_signaling - enable signalling on fence + * @fence: fence + * + * This function is called with fence_queue lock held, and adds a callback + * to fence_queue that checks if this fence is signaled, and if so it + * signals the fence and removes itself. + */ +static bool amdgpu_fence_enable_signaling(struct fence *f) +{ + struct amdgpu_fence *fence = to_amdgpu_fence(f); + struct amdgpu_ring *ring = fence->ring; + + if (atomic64_read(&ring->fence_drv.last_seq) >= fence->seq) + return false; + + fence->fence_wake.flags = 0; + fence->fence_wake.private = NULL; + fence->fence_wake.func = amdgpu_fence_check_signaled; + __add_wait_queue(&ring->fence_drv.fence_queue, &fence->fence_wake); + fence_get(f); + amdgpu_fence_schedule_check(ring); + FENCE_TRACE(&fence->base, "armed on ring %i!\n", ring->idx); + return true; +} + +const struct fence_ops amdgpu_fence_ops = { + .get_driver_name = amdgpu_fence_get_driver_name, + .get_timeline_name = amdgpu_fence_get_timeline_name, + .enable_signaling = amdgpu_fence_enable_signaling, + .signaled = amdgpu_fence_is_signaled, + .wait = fence_default_wait, + .release = NULL, +}; /* * Fence debugfs @@ -783,22 +814,3 @@ int amdgpu_debugfs_fence_init(struct amdgpu_device *adev) #endif } -static const char *amdgpu_fence_get_driver_name(struct fence *fence) -{ - return "amdgpu"; -} - -static const char *amdgpu_fence_get_timeline_name(struct fence *f) -{ - struct amdgpu_fence *fence = to_amdgpu_fence(f); - return (const char *)fence->ring->name; -} - -const struct fence_ops amdgpu_fence_ops = { - .get_driver_name = amdgpu_fence_get_driver_name, - .get_timeline_name = amdgpu_fence_get_timeline_name, - .enable_signaling = amdgpu_fence_enable_signaling, - .signaled = amdgpu_fence_is_signaled, - .wait = fence_default_wait, - .release = NULL, -}; -- cgit