From 3b0781595431acafe3db6596e12deb46975d91dd Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Fri, 21 Jul 2023 08:41:27 -0600 Subject: futex: move FUTEX2_VALID_MASK to futex.h We need this for validating the futex2 flags outside of the normal futex syscalls. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 2 ++ kernel/futex/syscalls.c | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index a06030a1a27b..a173a9d501e1 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -52,6 +52,8 @@ static inline unsigned int futex_to_flags(unsigned int op) return flags; } +#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE) + /* FUTEX2_ to FLAGS_ */ static inline unsigned int futex2_to_flags(unsigned int flags2) { diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 8200d86d30e1..2b5cafdfdc50 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -179,8 +179,6 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3); } -#define FUTEX2_VALID_MASK (FUTEX2_SIZE_MASK | FUTEX2_PRIVATE) - /** * futex_parse_waitv - Parse a waitv array from userspace * @futexv: Kernel side list of waiters to be filled -- cgit From 12a4be50aff30ee8f2c6a64020c82a4e997e8d6c Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Thu, 8 Jun 2023 11:56:06 -0600 Subject: futex: factor out the futex wake handling In preparation for having another waker that isn't futex_wake_mark(), add a wake handler in futex_q. No extra data is associated with the handler outside of struct futex_q itself. futex_wake_mark() is defined as the standard wakeup helper, now set through futex_q_init like other defaults. Normal sync futex waiting relies on wake_q holding tasks that should be woken up. This is what futex_wake_mark() does, it'll unqueue the futex and add the associated task to the wake queue. For async usage of futex waiting, rather than having tasks sleeping on the futex, we'll need to deal with a futex wake differently. For the planned io_uring case, that means posting a completion event for the task in question. Having a definable wake handler can help support that use case. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 5 +++++ kernel/futex/requeue.c | 3 ++- kernel/futex/waitwake.c | 6 +++--- 3 files changed, 10 insertions(+), 4 deletions(-) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index a173a9d501e1..547f509b2c87 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -139,11 +139,15 @@ struct futex_pi_state { union futex_key key; } __randomize_layout; +struct futex_q; +typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); + /** * struct futex_q - The hashed futex queue entry, one per waiting task * @list: priority-sorted list of tasks waiting on this futex * @task: the task waiting on the futex * @lock_ptr: the hash bucket lock + * @wake: the wake handler for this queue * @key: the key the futex is hashed on * @pi_state: optional priority inheritance state * @rt_waiter: rt_waiter storage for use with requeue_pi @@ -168,6 +172,7 @@ struct futex_q { struct task_struct *task; spinlock_t *lock_ptr; + futex_wake_fn *wake; union futex_key key; struct futex_pi_state *pi_state; struct rt_mutex_waiter *rt_waiter; diff --git a/kernel/futex/requeue.c b/kernel/futex/requeue.c index a0a79954f506..9dc789399a1a 100644 --- a/kernel/futex/requeue.c +++ b/kernel/futex/requeue.c @@ -58,6 +58,7 @@ enum { const struct futex_q futex_q_init = { /* list gets initialized in futex_queue()*/ + .wake = futex_wake_mark, .key = FUTEX_KEY_INIT, .bitset = FUTEX_BITSET_MATCH_ANY, .requeue_state = ATOMIC_INIT(Q_REQUEUE_PI_NONE), @@ -593,7 +594,7 @@ retry_private: /* Plain futexes just wake or requeue and are done */ if (!requeue_pi) { if (++task_count <= nr_wake) - futex_wake_mark(&wake_q, this); + this->wake(&wake_q, this); else requeue_futex(this, hb1, hb2, &key2); continue; diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 37860f794bf7..35c6a637a4bb 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -177,7 +177,7 @@ int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset) if (!(this->bitset & bitset)) continue; - futex_wake_mark(&wake_q, this); + this->wake(&wake_q, this); if (++ret >= nr_wake) break; } @@ -292,7 +292,7 @@ retry_private: ret = -EINVAL; goto out_unlock; } - futex_wake_mark(&wake_q, this); + this->wake(&wake_q, this); if (++ret >= nr_wake) break; } @@ -306,7 +306,7 @@ retry_private: ret = -EINVAL; goto out_unlock; } - futex_wake_mark(&wake_q, this); + this->wake(&wake_q, this); if (++op_ret >= nr_wake2) break; } -- cgit From e52c43403c9b839a30a9cfc4b75109581389d764 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 12 Jul 2023 09:14:52 -0600 Subject: futex: abstract out a __futex_wake_mark() helper Move the unqueue and lock_ptr clear into a helper that futex_wake_mark() calls. Add it to the public functions as well, in preparation for using it outside the core futex code. Suggested-by: Peter Zijlstra Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 1 + kernel/futex/waitwake.c | 33 ++++++++++++++++++++++----------- 2 files changed, 23 insertions(+), 11 deletions(-) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 547f509b2c87..33835b81e0c3 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -219,6 +219,7 @@ extern int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags, struct futex_q *q, struct futex_hash_bucket **hb); extern void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, struct hrtimer_sleeper *timeout); +extern bool __futex_wake_mark(struct futex_q *q); extern void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q); extern int fault_in_user_writeable(u32 __user *uaddr); diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 35c6a637a4bb..6fcf5f723719 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -106,20 +106,11 @@ * double_lock_hb() and double_unlock_hb(), respectively. */ -/* - * The hash bucket lock must be held when this is called. - * Afterwards, the futex_q must not be accessed. Callers - * must ensure to later call wake_up_q() for the actual - * wakeups to occur. - */ -void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) +bool __futex_wake_mark(struct futex_q *q) { - struct task_struct *p = q->task; - if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n")) - return; + return false; - get_task_struct(p); __futex_unqueue(q); /* * The waiting task can free the futex_q as soon as q->lock_ptr = NULL @@ -130,6 +121,26 @@ void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) */ smp_store_release(&q->lock_ptr, NULL); + return true; +} + +/* + * The hash bucket lock must be held when this is called. + * Afterwards, the futex_q must not be accessed. Callers + * must ensure to later call wake_up_q() for the actual + * wakeups to occur. + */ +void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q) +{ + struct task_struct *p = q->task; + + get_task_struct(p); + + if (!__futex_wake_mark(q)) { + put_task_struct(p); + return; + } + /* * Queue the task for later wakeup for after we've released * the hb->lock. -- cgit From 8af1692616d993c93a080865a7f19506733aa462 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 13 Jun 2023 15:44:42 -0600 Subject: futex: add wake_data to struct futex_q With handling multiple futex_q for waitv, we cannot easily go from the futex_q to data related to that request or queue. Add a wake_data argument that belongs to the wake handler assigned. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 33835b81e0c3..76f6c2e0f539 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -148,6 +148,7 @@ typedef void (futex_wake_fn)(struct wake_q_head *wake_q, struct futex_q *q); * @task: the task waiting on the futex * @lock_ptr: the hash bucket lock * @wake: the wake handler for this queue + * @wake_data: data associated with the wake handler * @key: the key the futex is hashed on * @pi_state: optional priority inheritance state * @rt_waiter: rt_waiter storage for use with requeue_pi @@ -173,6 +174,7 @@ struct futex_q { struct task_struct *task; spinlock_t *lock_ptr; futex_wake_fn *wake; + void *wake_data; union futex_key key; struct futex_pi_state *pi_state; struct rt_mutex_waiter *rt_waiter; -- cgit From 5177c0cb306a8628bafbf1e6b7aa7e1b7436b8dc Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 13 Jun 2023 08:31:58 -0600 Subject: futex: make futex_parse_waitv() available as a helper To make it more generically useful, augment it with allowing the caller to pass in the wake handler and wake data. Convert the futex_waitv() syscall, passing in the default handlers. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 5 +++++ kernel/futex/syscalls.c | 16 +++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 76f6c2e0f539..6b6a6b3da103 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -361,6 +361,11 @@ struct futex_vector { struct futex_q q; }; +extern int futex_parse_waitv(struct futex_vector *futexv, + struct futex_waitv __user *uwaitv, + unsigned int nr_futexes, futex_wake_fn *wake, + void *wake_data); + extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to); diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c index 2b5cafdfdc50..4b6da9116aa6 100644 --- a/kernel/futex/syscalls.c +++ b/kernel/futex/syscalls.c @@ -184,12 +184,15 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val, * @futexv: Kernel side list of waiters to be filled * @uwaitv: Userspace list to be parsed * @nr_futexes: Length of futexv + * @wake: Wake to call when futex is woken + * @wake_data: Data for the wake handler * * Return: Error code on failure, 0 on success */ -static int futex_parse_waitv(struct futex_vector *futexv, - struct futex_waitv __user *uwaitv, - unsigned int nr_futexes) +int futex_parse_waitv(struct futex_vector *futexv, + struct futex_waitv __user *uwaitv, + unsigned int nr_futexes, futex_wake_fn *wake, + void *wake_data) { struct futex_waitv aux; unsigned int i; @@ -214,6 +217,8 @@ static int futex_parse_waitv(struct futex_vector *futexv, futexv[i].w.val = aux.val; futexv[i].w.uaddr = aux.uaddr; futexv[i].q = futex_q_init; + futexv[i].q.wake = wake; + futexv[i].q.wake_data = wake_data; } return 0; @@ -306,7 +311,8 @@ SYSCALL_DEFINE5(futex_waitv, struct futex_waitv __user *, waiters, goto destroy_timer; } - ret = futex_parse_waitv(futexv, waiters, nr_futexes); + ret = futex_parse_waitv(futexv, waiters, nr_futexes, futex_wake_mark, + NULL); if (!ret) ret = futex_wait_multiple(futexv, nr_futexes, timeout ? &to : NULL); @@ -421,7 +427,7 @@ SYSCALL_DEFINE4(futex_requeue, if (!waiters) return -EINVAL; - ret = futex_parse_waitv(futexes, waiters, 2); + ret = futex_parse_waitv(futexes, waiters, 2, futex_wake_mark, NULL); if (ret) return ret; -- cgit From e9a56c9325ef28d5481712e85dd5d3f8b2a68e88 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Tue, 13 Jun 2023 08:34:08 -0600 Subject: futex: make the vectored futex operations available Rename unqueue_multiple() as futex_unqueue_multiple(), and make both that and futex_wait_multiple_setup() available for external users. This is in preparation for wiring up vectored waits in io_uring. Acked-by: Peter Zijlstra (Intel) Signed-off-by: Jens Axboe --- kernel/futex/futex.h | 5 +++++ kernel/futex/waitwake.c | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) (limited to 'kernel/futex') diff --git a/kernel/futex/futex.h b/kernel/futex/futex.h index 6b6a6b3da103..8b195d06f4e8 100644 --- a/kernel/futex/futex.h +++ b/kernel/futex/futex.h @@ -366,6 +366,11 @@ extern int futex_parse_waitv(struct futex_vector *futexv, unsigned int nr_futexes, futex_wake_fn *wake, void *wake_data); +extern int futex_wait_multiple_setup(struct futex_vector *vs, int count, + int *woken); + +extern int futex_unqueue_multiple(struct futex_vector *v, int count); + extern int futex_wait_multiple(struct futex_vector *vs, unsigned int count, struct hrtimer_sleeper *to); diff --git a/kernel/futex/waitwake.c b/kernel/futex/waitwake.c index 6fcf5f723719..61b112897a84 100644 --- a/kernel/futex/waitwake.c +++ b/kernel/futex/waitwake.c @@ -372,7 +372,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, } /** - * unqueue_multiple - Remove various futexes from their hash bucket + * futex_unqueue_multiple - Remove various futexes from their hash bucket * @v: The list of futexes to unqueue * @count: Number of futexes in the list * @@ -382,7 +382,7 @@ void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q, * - >=0 - Index of the last futex that was awoken; * - -1 - No futex was awoken */ -static int unqueue_multiple(struct futex_vector *v, int count) +int futex_unqueue_multiple(struct futex_vector *v, int count) { int ret = -1, i; @@ -410,7 +410,7 @@ static int unqueue_multiple(struct futex_vector *v, int count) * - 0 - Success * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL */ -static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) +int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken) { struct futex_hash_bucket *hb; bool retry = false; @@ -472,7 +472,7 @@ retry: * was woken, we don't return error and return this index to * userspace */ - *woken = unqueue_multiple(vs, i); + *woken = futex_unqueue_multiple(vs, i); if (*woken >= 0) return 1; @@ -557,7 +557,7 @@ int futex_wait_multiple(struct futex_vector *vs, unsigned int count, __set_current_state(TASK_RUNNING); - ret = unqueue_multiple(vs, count); + ret = futex_unqueue_multiple(vs, count); if (ret >= 0) return ret; -- cgit