Add support to wait on multiple futexes. This is the interface implemented by this syscall:

futex_waitv(struct futex_waitv *waiters, unsigned int nr_futexes, unsigned int flags, struct timespec *timo)

struct futex_waitv { __u64 val; void *uaddr; unsigned int flags; };

Given an array of struct futex_waitv, wait on each uaddr. The thread wakes if a futex_wake() is performed at any uaddr. The syscall returns immediately if any waiter has *uaddr != val. *timo is an optional timeout value for the operation. The flags argument of the syscall should be used solely for specifying the timeout clock as realtime, if needed. Flags for shared futexes, sizes, etc. should be used on the individual flags of each waiter.

Returns the array index of one of the awakened futexes. There’s no given information of how many were awakened, or any particular attribute of it (if it’s the first awakened, if it is of the smaller index...).

Signed-off-by: André Almeida andrealmeid@collabora.com --- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + include/linux/compat.h | 9 + include/linux/futex.h | 108 ++++++-- include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/futex.h | 15 ++ kernel/futex.c | 72 +----- kernel/futex2.c | 345 +++++++++++++++++++++++++ kernel/sys_ni.c | 2 + 9 files changed, 477 insertions(+), 80 deletions(-)

diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index e3b827a9c094..5573437c1914 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -453,3 +453,4 @@ 446 i386 landlock_restrict_self sys_landlock_restrict_self 447 i386 futex_wait sys_futex_wait compat_sys_futex_wait 448 i386 futex_wake sys_futex_wake +449 i386 futex_waitv sys_futex_waitv compat_sys_futex_waitv diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 63b447255df2..bad4aca3e9ba 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -370,6 +370,7 @@ 446 common landlock_restrict_self sys_landlock_restrict_self 447 common futex_wait sys_futex_wait 448 common futex_wake sys_futex_wake +449 common futex_waitv sys_futex_waitv

# # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/compat.h b/include/linux/compat.h index 5a910e0c437a..75b90e41e05b 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -368,6 +368,12 @@ struct compat_robust_list_head { compat_uptr_t list_op_pending; };

+struct compat_futex_waitv { + compat_u64 val; + compat_uptr_t uaddr; + compat_uint_t flags; +}; + #ifdef CONFIG_COMPAT_OLD_SIGACTION struct compat_old_sigaction { compat_uptr_t sa_handler; @@ -696,6 +702,9 @@ compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr, asmlinkage long compat_sys_futex_wait(void __user *uaddr, compat_u64 val, unsigned int flags, struct __kernel_timespec __user *timo); +asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, + compat_uint_t nr_futexes, compat_uint_t flags, + struct __kernel_timespec __user *timo); /* kernel/itimer.c */ asmlinkage long compat_sys_getitimer(int which, struct old_itimerval32 __user *it); diff --git a/include/linux/futex.h b/include/linux/futex.h index f0eaa05ec8bc..7afef5bb3da2 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -29,6 +29,22 @@ struct task_struct; #define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ #define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */

+/* + * Futex flags used to encode options to functions and preserve them across + * restarts. + */ +#ifdef CONFIG_MMU +# define FLAGS_SHARED 0x01 +#else +/* + * NOMMU does not have per process address space. Let the compiler optimize + * code away. + */ +# define FLAGS_SHARED 0x00 +#endif +#define FLAGS_CLOCKRT 0x02 +#define FLAGS_HAS_TIMEOUT 0x04 + union futex_key { struct { u64 i_seq; @@ -50,6 +66,63 @@ union futex_key { } both; };

+/** + * struct futex_q - The hashed futex queue entry, one per waiting task + * @list: priority-sorted list of tasks waiting on this futex + * @task: the task waiting on the futex + * @lock_ptr: the hash bucket lock + * @key: the key the futex is hashed on + * @pi_state: optional priority inheritance state + * @rt_waiter: rt_waiter storage for use with requeue_pi + * @requeue_pi_key: the requeue_pi target futex key + * @bitset: bitset for the optional bitmasked wakeup + * + * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so + * we can wake only the relevant ones (hashed queues may be shared). + * + * A futex_q has a woken state, just like tasks have TASK_RUNNING. + * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. + * The order of wakeup is always to make the first condition true, then + * the second. + * + * PI futexes are typically woken before they are removed from the hash list via + * the rt_mutex code. See unqueue_me_pi(). + */ +struct futex_q { + struct plist_node list; + + struct task_struct *task; + spinlock_t *lock_ptr; + union futex_key key; + struct futex_pi_state *pi_state; + struct rt_mutex_waiter *rt_waiter; + union futex_key *requeue_pi_key; + u32 bitset; +} __randomize_layout; + +/** + * struct futex_vector - Auxiliary struct for futex_waitv() + * @w: Userspace provided data + * @q: Kernel side data + * + * Struct used to build an array with all data need for futex_waitv() + */ +struct futex_vector { + struct futex_waitv w; + struct futex_q q; +}; + +/* + * Hash buckets are shared by all the futex_keys that hash to the same + * location. Each key may have multiple futex_q structures, one for each task + * waiting on a futex. + */ +struct futex_hash_bucket { + atomic_t waiters; + spinlock_t lock; + struct plist_head chain; +} ____cacheline_aligned_in_smp; + #define FUTEX_KEY_INIT (union futex_key) { .both = { .ptr = 0ULL } }

#ifdef CONFIG_FUTEX @@ -59,6 +132,11 @@ enum { FUTEX_STATE_DEAD, };

+enum futex_access { + FUTEX_READ, + FUTEX_WRITE +}; + static inline void futex_init_task(struct task_struct *tsk) { tsk->robust_list = NULL; @@ -81,22 +159,22 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset); +void queue_me(struct futex_q *q, struct futex_hash_bucket *hb); +int unqueue_me(struct futex_q *q); +void queue_unlock(struct futex_hash_bucket *hb); +int get_futex_value_locked(u32 *dest, u32 __user *from); +int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, + enum futex_access rw); +struct futex_hash_bucket *queue_lock(struct futex_q *q); +struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, + struct hrtimer_sleeper *timeout, + int flags, u64 range_ns);

-/* - * Futex flags used to encode options to functions and preserve them across - * restarts. - */ -#ifdef CONFIG_MMU -# define FLAGS_SHARED 0x01 -#else -/* - * NOMMU does not have per process address space. Let the compiler optimize - * code away. - */ -# define FLAGS_SHARED 0x00 -#endif -#define FLAGS_CLOCKRT 0x02 -#define FLAGS_HAS_TIMEOUT 0x04 +static const struct futex_q futex_q_init = { + /* list gets initialized in queue_me()*/ + .key = FUTEX_KEY_INIT, + .bitset = FUTEX_BITSET_MATCH_ANY +};

#else static inline void futex_init_task(struct task_struct *tsk) { } diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index df9fe2e23ee0..57acb3a0f69f 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -876,9 +876,11 @@ __SYSCALL(__NR_landlock_restrict_self, sys_landlock_restrict_self) __SC_COMP(__NR_futex_wait, sys_futex_wait, compat_sys_futex_wait) #define __NR_futex_wake 448 __SYSCALL(__NR_futex_wake, sys_futex_wake) +#define __NR_futex_waitv 449 +__SC_COMP(__NR_futex_waitv, sys_futex_waitv, compat_sys_futex_waitv)

#undef __NR_syscalls -#define __NR_syscalls 449 +#define __NR_syscalls 450

/* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index 44750caa261e..daa135bdedda 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -45,6 +45,21 @@ #define FUTEX_32 2 #define FUTEX_SHARED_FLAG 8 #define FUTEX_SIZE_MASK 0x3 + +#define FUTEX_WAITV_MAX 128 + +/** + * struct futex_waitv - A waiter for vectorized wait + * @val: Expected value at uaddr + * @uaddr: User address to wait on + * @flags: Flags for this waiter + */ +struct futex_waitv { + __u64 val; + void __user *uaddr; + unsigned int flags; +}; + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/kernel/futex.c b/kernel/futex.c index ef7131bd8bc4..135782fc3461 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -171,57 +171,6 @@ struct futex_pi_state { union futex_key key; } __randomize_layout;

-/** - * struct futex_q - The hashed futex queue entry, one per waiting task - * @list: priority-sorted list of tasks waiting on this futex - * @task: the task waiting on the futex - * @lock_ptr: the hash bucket lock - * @key: the key the futex is hashed on - * @pi_state: optional priority inheritance state - * @rt_waiter: rt_waiter storage for use with requeue_pi - * @requeue_pi_key: the requeue_pi target futex key - * @bitset: bitset for the optional bitmasked wakeup - * - * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so - * we can wake only the relevant ones (hashed queues may be shared). - * - * A futex_q has a woken state, just like tasks have TASK_RUNNING. - * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. - * The order of wakeup is always to make the first condition true, then - * the second. - * - * PI futexes are typically woken before they are removed from the hash list via - * the rt_mutex code. See unqueue_me_pi(). - */ -struct futex_q { - struct plist_node list; - - struct task_struct *task; - spinlock_t *lock_ptr; - union futex_key key; - struct futex_pi_state *pi_state; - struct rt_mutex_waiter *rt_waiter; - union futex_key *requeue_pi_key; - u32 bitset; -} __randomize_layout; - -static const struct futex_q futex_q_init = { - /* list gets initialized in queue_me()*/ - .key = FUTEX_KEY_INIT, - .bitset = FUTEX_BITSET_MATCH_ANY -}; - -/* - * Hash buckets are shared by all the futex_keys that hash to the same - * location. Each key may have multiple futex_q structures, one for each task - * waiting on a futex. - */ -struct futex_hash_bucket { - atomic_t waiters; - spinlock_t lock; - struct plist_head chain; -} ____cacheline_aligned_in_smp; - /* * The base of the bucket array and its size are always used together * (after initialization only in hash_futex()), so ensure that they @@ -364,11 +313,6 @@ static inline int match_futex(union futex_key *key1, union futex_key *key2) && key1->both.offset == key2->both.offset); }

-enum futex_access { - FUTEX_READ, - FUTEX_WRITE -}; - /** * futex_setup_timer - set up the sleeping hrtimer. * @time: ptr to the given timeout value @@ -379,7 +323,7 @@ enum futex_access { * Return: Initialized hrtimer_sleeper structure or NULL if no timeout * value given */ -static inline struct hrtimer_sleeper * +inline struct hrtimer_sleeper * futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, int flags, u64 range_ns) { @@ -465,8 +409,8 @@ static u64 get_inode_sequence_number(struct inode *inode) * * lock_page() might sleep, the caller should not hold a spinlock. */ -static int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, - enum futex_access rw) +int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key, + enum futex_access rw) { unsigned long address = (unsigned long)uaddr; struct mm_struct *mm = current->mm; @@ -698,7 +642,7 @@ static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr, return ret; }

-static int get_futex_value_locked(u32 *dest, u32 __user *from) +inline int get_futex_value_locked(u32 *dest, u32 __user *from) { int ret;

@@ -2173,7 +2117,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, }

/* The key must be already stored in q->key. */ -static inline struct futex_hash_bucket *queue_lock(struct futex_q *q) +inline struct futex_hash_bucket *queue_lock(struct futex_q *q) __acquires(&hb->lock) { struct futex_hash_bucket *hb; @@ -2196,7 +2140,7 @@ static inline struct futex_hash_bucket *queue_lock(struct futex_q *q) return hb; }

-static inline void +inline void queue_unlock(struct futex_hash_bucket *hb) __releases(&hb->lock) { @@ -2235,7 +2179,7 @@ static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb) * state is implicit in the state of woken task (see futex_wait_requeue_pi() for * an example). */ -static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) +inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) __releases(&hb->lock) { __queue_me(q, hb); @@ -2253,7 +2197,7 @@ static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb) * - 1 - if the futex_q was still queued (and we removed unqueued it); * - 0 - if the futex_q was already removed by the waking thread */ -static int unqueue_me(struct futex_q *q) +int unqueue_me(struct futex_q *q) { spinlock_t *lock_ptr; int ret = 0; diff --git a/kernel/futex2.c b/kernel/futex2.c index 990c665280fd..cc1f31afb281 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -7,6 +7,7 @@

#include <asm/futex.h>

+#include <linux/freezer.h> #include <linux/syscalls.h>

/* @@ -14,6 +15,350 @@ */ #define FUTEX2_MASK (FUTEX_SIZE_MASK | FUTEX_SHARED_FLAG | FUTEX_CLOCK_REALTIME)

+/* Mask for each futex in futex_waitv list */ +#define FUTEXV_WAITER_MASK (FUTEX_SIZE_MASK | FUTEX_SHARED_FLAG) + +/* Mask for sys_futex_waitv flag */ +#define FUTEXV_MASK (FUTEX_CLOCK_REALTIME) + +/** + * unqueue_multiple() - Remove various futexes from their futex_hash_bucket + * @v: The list of futexes to unqueue + * @count: Number of futexes in the list + * + * Helper to unqueue a list of futexes. This can't fail. + * + * Return: + * - >=0 - Index of the last futex that was awoken; + * - -1 - No futex was awoken + */ +static int unqueue_multiple(struct futex_vector *v, int count) +{ + int ret = -1, i; + + for (i = 0; i < count; i++) { + if (!unqueue_me(&v[i].q)) + ret = i; + } + + return ret; +} + +/** + * futex_wait_multiple_setup() - Prepare to wait and enqueue multiple futexes + * @vs: The corresponding futex list + * @count: The size of the list + * @awaken: Index of the last awoken futex (return parameter) + * + * Prepare multiple futexes in a single step and enqueue them. This may fail if + * the futex list is invalid or if any futex was already awoken. On success the + * task is ready to interruptible sleep. + * + * Return: + * - 1 - One of the futexes was awaken by another thread + * - 0 - Success + * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL + */ +static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *awaken) +{ + struct futex_hash_bucket *hb; + int ret, i; + u32 uval; + + /* + * Enqueuing multiple futexes is tricky, because we need to + * enqueue each futex in the list before dealing with the next + * one to avoid deadlocking on the hash bucket. But, before + * enqueuing, we need to make sure that current->state is + * TASK_INTERRUPTIBLE, so we don't absorb any awake events, which + * cannot be done before the get_futex_key of the next key, + * because it calls get_user_pages, which can sleep. Thus, we + * fetch the list of futexes keys in two steps, by first pinning + * all the memory keys in the futex key, and only then we read + * each key and queue the corresponding futex. + */ +retry: + for (i = 0; i < count; i++) { + ret = get_futex_key(vs[i].w.uaddr, + vs[i].w.flags & FUTEX_SHARED_FLAG, + &vs[i].q.key, FUTEX_READ); + if (unlikely(ret)) + return ret; + } + + set_current_state(TASK_INTERRUPTIBLE); + + for (i = 0; i < count; i++) { + struct futex_q *q = &vs[i].q; + struct futex_waitv *waitv = &vs[i].w; + + hb = queue_lock(q); + ret = get_futex_value_locked(&uval, waitv->uaddr); + if (ret) { + /* + * We need to try to handle the fault, which + * cannot be done without sleep, so we need to + * undo all the work already done, to make sure + * we don't miss any wake ups. Therefore, clean + * up, handle the fault and retry from the + * beginning. + */ + queue_unlock(hb); + __set_current_state(TASK_RUNNING); + + *awaken = unqueue_multiple(vs, i); + if (*awaken >= 0) + return 1; + + if (get_user(uval, (u32 __user *)waitv->uaddr)) + return -EINVAL; + + goto retry; + } + + if (uval != waitv->val) { + queue_unlock(hb); + __set_current_state(TASK_RUNNING); + + /* + * If something was already awaken, we can + * safely ignore the error and succeed. + */ + *awaken = unqueue_multiple(vs, i); + if (*awaken >= 0) + return 1; + + return -EWOULDBLOCK; + } + + /* + * The bucket lock can't be held while dealing with the + * next futex. Queue each futex at this moment so hb can + * be unlocked. + */ + queue_me(&vs[i].q, hb); + } + return 0; +} + +/** + * futex_wait_multiple() - Prepare to wait on and enqueue several futexes + * @vs: The list of futexes to wait on + * @count: The number of objects + * @to: Timeout before giving up and returning to userspace + * + * Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this function + * sleeps on a group of futexes and returns on the first futex that + * triggered, or after the timeout has elapsed. + * + * Return: + * - >=0 - Hint to the futex that was awoken + * - <0 - On error + */ +static int futex_wait_multiple(struct futex_vector *vs, unsigned int count, + struct hrtimer_sleeper *to) +{ + int ret, hint = 0; + unsigned int i; + + while (1) { + ret = futex_wait_multiple_setup(vs, count, &hint); + if (ret) { + if (ret > 0) { + /* A futex was awaken during setup */ + ret = hint; + } + return ret; + } + + if (to) + hrtimer_start_expires(&to->timer, HRTIMER_MODE_ABS); + + /* + * Avoid sleeping if another thread already tried to + * wake us. + */ + for (i = 0; i < count; i++) { + if (plist_node_empty(&vs[i].q.list)) + break; + } + + if (i == count && (!to || to->task)) + freezable_schedule(); + + __set_current_state(TASK_RUNNING); + + ret = unqueue_multiple(vs, count); + if (ret >= 0) + return ret; + + if (to && !to->task) + return -ETIMEDOUT; + else if (signal_pending(current)) + return -ERESTARTSYS; + /* + * The final case is a spurious wakeup, for + * which just retry. + */ + } +} + +#ifdef CONFIG_COMPAT +/** + * compat_futex_parse_waitv - Parse a waitv array from userspace + * @futexv: Kernel side list of waiters to be filled + * @uwaitv: Userspace list to be parsed + * @nr_futexes: Length of futexv + * + * Return: Error code on failure, pointer to a prepared futexv otherwise + */ +static int compat_futex_parse_waitv(struct futex_vector *futexv, + struct compat_futex_waitv __user *uwaitv, + unsigned int nr_futexes) +{ + struct compat_futex_waitv aux; + unsigned int i; + + for (i = 0; i < nr_futexes; i++) { + if (copy_from_user(&aux, &uwaitv[i], sizeof(aux))) + return -EFAULT; + + if ((aux.flags & ~FUTEXV_WAITER_MASK) || + (aux.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + futexv[i].w.flags = aux.flags; + futexv[i].w.val = aux.val; + futexv[i].w.uaddr = compat_ptr(aux.uaddr); + futexv[i].q = futex_q_init; + } + + return 0; +} + +COMPAT_SYSCALL_DEFINE4(futex_waitv, struct compat_futex_waitv __user *, waiters, + unsigned int, nr_futexes, unsigned int, flags, + struct __kernel_timespec __user *, timo) +{ + struct hrtimer_sleeper to; + struct futex_vector *futexv; + struct timespec64 ts; + ktime_t time; + int ret; + + if (flags & ~FUTEXV_MASK) + return -EINVAL; + + if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters) + return -EINVAL; + + if (timo) { + int flag_clkid = 0; + + if (get_timespec64(&ts, timo)) + return -EFAULT; + + if (!timespec64_valid(&ts)) + return -EINVAL; + + if (flags & FUTEX_CLOCK_REALTIME) + flag_clkid = FLAGS_CLOCKRT; + + time = timespec64_to_ktime(ts); + futex_setup_timer(&time, &to, flag_clkid, 0); + } + + futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL); + if (!futexv) + return -ENOMEM; + + ret = compat_futex_parse_waitv(futexv, waiters, nr_futexes); + if (!ret) + ret = futex_wait_multiple(futexv, nr_futexes, timo ? &to : NULL); + + if (timo) { + hrtimer_cancel(&to.timer); + destroy_hrtimer_on_stack(&to.timer); + } + + kfree(futexv); + return ret; +} +#endif + +static int futex_parse_waitv(struct futex_vector *futexv, + struct futex_waitv __user *uwaitv, + unsigned int nr_futexes) +{ + struct futex_waitv aux; + unsigned int i; + + for (i = 0; i < nr_futexes; i++) { + if (copy_from_user(&aux, &uwaitv[i], sizeof(aux))) + return -EFAULT; + + if ((aux.flags & ~FUTEXV_WAITER_MASK) || + (aux.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + futexv[i].w.flags = aux.flags; + futexv[i].w.val = aux.val; + futexv[i].w.uaddr = aux.uaddr; + futexv[i].q = futex_q_init; + } + + return 0; +} + +SYSCALL_DEFINE4(futex_waitv, struct futex_waitv __user *, waiters, + unsigned int, nr_futexes, unsigned int, flags, + struct __kernel_timespec __user *, timo) +{ + struct hrtimer_sleeper to; + struct futex_vector *futexv; + struct timespec64 ts; + ktime_t time; + int ret; + + if (flags & ~FUTEXV_MASK) + return -EINVAL; + + if (!nr_futexes || nr_futexes > FUTEX_WAITV_MAX || !waiters) + return -EINVAL; + + if (timo) { + int flag_clkid = 0; + + if (get_timespec64(&ts, timo)) + return -EFAULT; + + if (!timespec64_valid(&ts)) + return -EINVAL; + + if (flags & FUTEX_CLOCK_REALTIME) + flag_clkid = FLAGS_CLOCKRT; + + time = timespec64_to_ktime(ts); + futex_setup_timer(&time, &to, flag_clkid, 0); + } + + futexv = kcalloc(nr_futexes, sizeof(*futexv), GFP_KERNEL); + if (!futexv) + return -ENOMEM; + + ret = futex_parse_waitv(futexv, waiters, nr_futexes); + if (!ret) + ret = futex_wait_multiple(futexv, nr_futexes, timo ? &to : NULL); + + if (timo) { + hrtimer_cancel(&to.timer); + destroy_hrtimer_on_stack(&to.timer); + } + + kfree(futexv); + return ret; +} + static long ksys_futex_wait(void __user *uaddr, u64 val, unsigned int flags, struct __kernel_timespec __user *timo) { diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index dbe397eaea46..93807bb7be51 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -155,6 +155,8 @@ COND_SYSCALL_COMPAT(get_robust_list); COND_SYSCALL(futex_wait); COND_SYSCALL_COMPAT(futex_wait); COND_SYSCALL(futex_wake); +COND_SYSCALL(futex_waitv); +COND_SYSCALL_COMPAT(futex_waitv);

/* kernel/hrtimer.c */

Implement requeue interface similarly to FUTEX_CMP_REQUEUE operation.

This is the syscall implemented by this patch:

futex_requeue(struct futex_requeue *rq1, struct futex_requeue *rq2, unsigned int nr_wake, unsigned int nr_requeue, u64 cmpval, unsigned int flags)

struct futex_requeue { void *uaddr; unsigned int flags; };

If (rq1->uaddr == cmpval), wake at rq1->uaddr a nr_wake number of waiters and then, remove a number of nr_requeue waiters at rq1->uaddr and add them to rq2->uaddr list. Each uaddr has its own set of flags, that must be defined at struct futex_requeue (such as size, shared, NUMA). The flags argument of the syscall is there just for the sake of extensibility, and right now it needs to be zero.

Return the number of the woken futexes + the number of requeued ones on success, error code otherwise.

Signed-off-by: André Almeida andrealmeid@collabora.com ---

The original FUTEX_CMP_REQUEUE interfaces is such as follows:

futex(*uaddr1, FUTEX_CMP_REQUEUE, nr_wake, nr_requeue, *uaddr2, cmpval);

Given that when this interface was created they was only one type of futex (as opposed to futex2, where there is shared, sizes, and NUMA), there was no way to specify individual flags for uaddr1 and 2. When FUTEX_PRIVATE was implemented, a new opcode was created as well (FUTEX_CMP_REQUEUE_PRIVATE), but they apply both futexes, so they should be of the same type regarding private/shared. This imposes a limitation on the use cases of the operation, and to overcome that at futex2, `struct futex_requeue` was created, so one can set individual flags for each futex. This flexibility is a trade-off with performance, given that now we need to perform two extra copy_from_user(). One alternative would be to use the upper half of flags bits to the first one, and the bottom half for the second futex, but this would also impose limitations, given that we would limit by half the flags possibilities. If equal futexes are common enough, the following extension could be added to overcome the current performance:

- A flag FUTEX_REQUEUE_EQUAL is added to futex2() flags; - If futex_requeue() see this flag, that means that both futexes uses the same set of attributes. - Then, the function parses the flags as of futex_wait/wake(). - *rq1 and *rq2 are used as void* (instead of struct futex_requeue) just like wait/wake().

In that way, we could avoid the copy_from_user(). --- arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + include/linux/compat.h | 10 +++ include/linux/futex.h | 3 + include/uapi/asm-generic/unistd.h | 4 +- include/uapi/linux/futex.h | 10 +++ kernel/futex.c | 18 ++-- kernel/futex2.c | 111 +++++++++++++++++++++++++ kernel/sys_ni.c | 2 + 9 files changed, 150 insertions(+), 10 deletions(-)

diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 5573437c1914..f02c3da76945 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -454,3 +454,4 @@ 447 i386 futex_wait sys_futex_wait compat_sys_futex_wait 448 i386 futex_wake sys_futex_wake 449 i386 futex_waitv sys_futex_waitv compat_sys_futex_waitv +450 i386 futex_requeue sys_futex_requeue compat_sys_futex_requeue diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index bad4aca3e9ba..8815c0de5d05 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -371,6 +371,7 @@ 447 common futex_wait sys_futex_wait 448 common futex_wake sys_futex_wake 449 common futex_waitv sys_futex_waitv +450 common futex_requeue sys_futex_requeue

# # Due to a historical design error, certain syscalls are numbered differently diff --git a/include/linux/compat.h b/include/linux/compat.h index 75b90e41e05b..861616de6d75 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -374,6 +374,11 @@ struct compat_futex_waitv { compat_uint_t flags; };

+struct compat_futex_requeue { + compat_uptr_t uaddr; + compat_uint_t flags; +}; + #ifdef CONFIG_COMPAT_OLD_SIGACTION struct compat_old_sigaction { compat_uptr_t sa_handler; @@ -705,6 +710,11 @@ asmlinkage long compat_sys_futex_wait(void __user *uaddr, compat_u64 val, asmlinkage long compat_sys_futex_waitv(struct compat_futex_waitv *waiters, compat_uint_t nr_futexes, compat_uint_t flags, struct __kernel_timespec __user *timo); +asmlinkage long compat_sys_futex_requeue(struct futex_requeue __user *rq1, + struct futex_requeue __user *rq2, + unsigned int nr_wake, + unsigned int nr_requeue, + u64 cmpval, unsigned int flags); /* kernel/itimer.c */ asmlinkage long compat_sys_getitimer(int which, struct old_itimerval32 __user *it); diff --git a/include/linux/futex.h b/include/linux/futex.h index 7afef5bb3da2..95afac7b2f76 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -159,6 +159,9 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset); int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset); +int futex_requeue(u32 __user *uaddr1, unsigned int flags1, u32 __user *uaddr2, + unsigned int flags2, int nr_wake, int nr_requeue, u32 *cmpval, + int requeue_pi); void queue_me(struct futex_q *q, struct futex_hash_bucket *hb); int unqueue_me(struct futex_q *q); void queue_unlock(struct futex_hash_bucket *hb); diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 57acb3a0f69f..89a480490dca 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -878,9 +878,11 @@ __SC_COMP(__NR_futex_wait, sys_futex_wait, compat_sys_futex_wait) __SYSCALL(__NR_futex_wake, sys_futex_wake) #define __NR_futex_waitv 449 __SC_COMP(__NR_futex_waitv, sys_futex_waitv, compat_sys_futex_waitv) +#define __NR_futex_requeue 450 +__SC_COMP(__NR_futex_requeue, sys_futex_requeue, compat_sys_futex_requeue)

#undef __NR_syscalls -#define __NR_syscalls 450 +#define __NR_syscalls 451

/* * 32 bit systems traditionally used different diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h index daa135bdedda..7effdd568699 100644 --- a/include/uapi/linux/futex.h +++ b/include/uapi/linux/futex.h @@ -60,6 +60,16 @@ struct futex_waitv { unsigned int flags; };

+/** + * struct futex_requeue - Define an address and its flags for requeue operation + * @uaddr: User address of one of the requeue arguments + * @flags: Flags for this address + */ +struct futex_requeue { + void __user *uaddr; + unsigned int flags; +}; + /* * Support for robust futexes: the kernel cleans up held futexes at * thread exit time. diff --git a/kernel/futex.c b/kernel/futex.c index 135782fc3461..7e6d70195d8a 100644 --- a/kernel/futex.c +++ b/kernel/futex.c @@ -1843,9 +1843,9 @@ futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1, * - >=0 - on success, the number of tasks requeued or woken; * - <0 - on error */ -static int futex_requeue(u32 __user *uaddr1, unsigned int flags, - u32 __user *uaddr2, int nr_wake, int nr_requeue, - u32 *cmpval, int requeue_pi) +int futex_requeue(u32 __user *uaddr1, unsigned int flags1, u32 __user *uaddr2, + unsigned int flags2, int nr_wake, int nr_requeue, + u32 *cmpval, int requeue_pi) { union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT; int task_count = 0, ret; @@ -1895,10 +1895,10 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, }

retry: - ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ); + ret = get_futex_key(uaddr1, flags1 & FLAGS_SHARED, &key1, FUTEX_READ); if (unlikely(ret != 0)) return ret; - ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, + ret = get_futex_key(uaddr2, flags2 & FLAGS_SHARED, &key2, requeue_pi ? FUTEX_WRITE : FUTEX_READ); if (unlikely(ret != 0)) return ret; @@ -1930,7 +1930,7 @@ static int futex_requeue(u32 __user *uaddr1, unsigned int flags, if (ret) return ret;

- if (!(flags & FLAGS_SHARED)) + if (!(flags1 & FLAGS_SHARED)) goto retry_private;

goto retry; @@ -3661,9 +3661,9 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, case FUTEX_WAKE_BITSET: return futex_wake(uaddr, flags, val, val3); case FUTEX_REQUEUE: - return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0); + return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, NULL, 0); case FUTEX_CMP_REQUEUE: - return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0); + return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 0); case FUTEX_WAKE_OP: return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3); case FUTEX_LOCK_PI: @@ -3680,7 +3680,7 @@ long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3, uaddr2); case FUTEX_CMP_REQUEUE_PI: - return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1); + return futex_requeue(uaddr, flags, uaddr2, flags, val, val2, &val3, 1); } return -ENOSYS; } diff --git a/kernel/futex2.c b/kernel/futex2.c index cc1f31afb281..4f6d099badec 100644 --- a/kernel/futex2.c +++ b/kernel/futex2.c @@ -446,3 +446,114 @@ SYSCALL_DEFINE3(futex_wake, void __user *, uaddr, unsigned int, nr_wake,

return futex_wake(uaddr, futex_flags, nr_wake, FUTEX_BITSET_MATCH_ANY); } + +#ifdef CONFIG_COMPAT +static int compat_futex_parse_requeue(struct compat_futex_requeue __user *rq, + void __user **uaddr, unsigned int *flags) +{ + struct compat_futex_requeue aux; + unsigned int futex_flags = 0; + + if (copy_from_user(&aux, rq, sizeof(*rq))) + return -EFAULT; + + if (aux.flags & ~FUTEXV_WAITER_MASK || + (aux.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + if (aux.flags & FUTEX_SHARED_FLAG) + futex_flags |= FLAGS_SHARED; + + *uaddr = compat_ptr(aux.uaddr); + *flags = futex_flags; + + return 0; +} + +COMPAT_SYSCALL_DEFINE6(futex_requeue, struct compat_futex_requeue __user *, rq1, + struct compat_futex_requeue __user *, rq2, + unsigned int, nr_wake, unsigned int, nr_requeue, + compat_u64, cmpval, unsigned int, flags) +{ + void __user *uaddr1, *uaddr2; + unsigned int flags1, flags2; + u32 val = cmpval; + int ret; + + if (flags) + return -EINVAL; + + ret = compat_futex_parse_requeue(rq1, &uaddr1, &flags1); + if (ret) + return ret; + + ret = compat_futex_parse_requeue(rq2, &uaddr2, &flags2); + if (ret) + return ret; + + return futex_requeue(uaddr1, flags1, uaddr2, flags2, nr_wake, nr_requeue, &val, 0); +} +#endif + +static int futex_parse_requeue(struct futex_requeue __user *rq, + void __user **uaddr, unsigned int *flags) +{ + struct futex_requeue aux; + unsigned int futex_flags = 0; + + if (copy_from_user(&aux, rq, sizeof(*rq))) + return -EFAULT; + + if (aux.flags & ~FUTEXV_WAITER_MASK || + (aux.flags & FUTEX_SIZE_MASK) != FUTEX_32) + return -EINVAL; + + if (aux.flags & FUTEX_SHARED_FLAG) + futex_flags |= FLAGS_SHARED; + + *uaddr = aux.uaddr; + *flags = futex_flags; + + return 0; +} + +/** + * sys_futex_requeue - Wake futexes at rq1 and requeue from rq1 to rq2 + * @rq1: Address of futexes to be waken/dequeued + * @rq2: Address for the futexes to be enqueued + * @nr_wake: Number of futexes waiting in uaddr1 to be woken up + * @nr_requeue: Number of futexes to be requeued from uaddr1 to uaddr2 + * @cmpval: Expected value at uaddr1 + * @flags: Reserved flags arg for requeue operation expansion. Must be 0. + * + * If (rq1->uaddr == cmpval), wake at uaddr1->uaddr a nr_wake number of + * waiters and then, remove a number of nr_requeue waiters at rq1->uaddr + * and add then to rq2->uaddr list. Each uaddr has its own set of flags, + * that must be defined at struct futex_requeue (such as size, shared, NUMA). + * + * Return the number of the woken futexes + the number of requeued ones on + * success, error code otherwise. + */ +SYSCALL_DEFINE6(futex_requeue, struct futex_requeue __user *, rq1, + struct futex_requeue __user *, rq2, + unsigned int, nr_wake, unsigned int, nr_requeue, + u64, cmpval, unsigned int, flags) +{ + void __user *uaddr1, *uaddr2; + unsigned int flags1, flags2; + u32 val = cmpval; + int ret; + + if (flags) + return -EINVAL; + + ret = futex_parse_requeue(rq1, &uaddr1, &flags1); + if (ret) + return ret; + + ret = futex_parse_requeue(rq2, &uaddr2, &flags2); + if (ret) + return ret; + + return futex_requeue(uaddr1, flags1, uaddr2, flags2, nr_wake, nr_requeue, &val, 0); +} diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c index 93807bb7be51..20a425b79fca 100644 --- a/kernel/sys_ni.c +++ b/kernel/sys_ni.c @@ -157,6 +157,8 @@ COND_SYSCALL_COMPAT(futex_wait); COND_SYSCALL(futex_wake); COND_SYSCALL(futex_waitv); COND_SYSCALL_COMPAT(futex_waitv); +COND_SYSCALL(futex_requeue); +COND_SYSCALL_COMPAT(futex_requeue);

/* kernel/hrtimer.c */

Adapt existing futex wait timeout file to test the same mechanism for futex2. futex2 accepts only absolute 64bit timers, but supports both monotonic and realtime clocks.

Signed-off-by: André Almeida andrealmeid@collabora.com --- .../futex/functional/futex_wait_timeout.c | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c index 1f8f6daaf1e7..d20f54745c2e 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c +++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c @@ -17,6 +17,14 @@

#include <pthread.h> #include "futextest.h" + +#include <errno.h> +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "futex2test.h" #include "logging.h"

#define TEST_NAME "futex-wait-timeout" @@ -92,8 +100,8 @@ static int futex_get_abs_timeout(clockid_t clockid, struct timespec *to, int main(int argc, char *argv[]) { futex_t f1 = FUTEX_INITIALIZER; + struct timespec to = {.tv_sec = 0, .tv_nsec = timeout_ns}; int res, ret = RET_PASS; - struct timespec to; pthread_t thread; int c;

@@ -118,7 +126,7 @@ int main(int argc, char *argv[]) }

ksft_print_header(); - ksft_set_plan(7); + ksft_set_plan(9); ksft_print_msg("%s: Block on a futex and wait for timeout\n", basename(argv[0])); ksft_print_msg("\tArguments: timeout=%ldns\n", timeout_ns); @@ -175,6 +183,18 @@ int main(int argc, char *argv[]) res = futex_lock_pi(&futex_pi, NULL, 0, FUTEX_CLOCK_REALTIME); test_timeout(res, &ret, "futex_lock_pi invalid timeout flag", ENOSYS);

+ /* setting absolute monotonic timeout for futex2 */ + if (futex_get_abs_timeout(CLOCK_MONOTONIC, &to, timeout_ns)) + return RET_FAIL; + res = futex2_wait(&f1, f1, FUTEX_32, &to); + test_timeout(res, &ret, "futex2_wait monotonic", ETIMEDOUT); + + /* setting absolute realtime timeout for futex2 */ + if (futex_get_abs_timeout(CLOCK_REALTIME, &to, timeout_ns)) + return RET_FAIL; + res = futex2_wait(&f1, f1, FUTEX_32 | FUTEX_CLOCK_REALTIME, &to); + test_timeout(res, &ret, "futex2_wait realtime", ETIMEDOUT); + ksft_print_cnts(); return ret; }

Create a new file to test the waitv mechanism. Test both private and shared futexes. Wake the last futex in the array, and check if the return value from futex_waitv() is the right index.

Signed-off-by: André Almeida andrealmeid@collabora.com --- .../selftests/futex/functional/.gitignore | 1 + .../selftests/futex/functional/Makefile | 3 +- .../selftests/futex/functional/futex2_waitv.c | 154 ++++++++++++++++++ .../testing/selftests/futex/functional/run.sh | 3 + .../selftests/futex/include/futex2test.h | 17 ++ 5 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/futex/functional/futex2_waitv.c

diff --git a/tools/testing/selftests/futex/functional/.gitignore b/tools/testing/selftests/futex/functional/.gitignore index 3e2d577c0595..22c572de8d10 100644 --- a/tools/testing/selftests/futex/functional/.gitignore +++ b/tools/testing/selftests/futex/functional/.gitignore @@ -9,3 +9,4 @@ futex_wait_wouldblock futex_wait futex_requeue futex2_wait +futex2_waitv diff --git a/tools/testing/selftests/futex/functional/Makefile b/tools/testing/selftests/futex/functional/Makefile index e4e4aa2e0368..240b53d8cb07 100644 --- a/tools/testing/selftests/futex/functional/Makefile +++ b/tools/testing/selftests/futex/functional/Makefile @@ -19,7 +19,8 @@ TEST_GEN_FILES := \ futex_wait_private_mapped_file \ futex_wait \ futex_requeue \ - futex2_wait + futex2_wait \ + futex2_waitv

TEST_PROGS := run.sh

diff --git a/tools/testing/selftests/futex/functional/futex2_waitv.c b/tools/testing/selftests/futex/functional/futex2_waitv.c new file mode 100644 index 000000000000..0f625a0657d5 --- /dev/null +++ b/tools/testing/selftests/futex/functional/futex2_waitv.c @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/****************************************************************************** + * + * Copyright Collabora Ltd., 2021 + * + * DESCRIPTION + * Test waitv/wake mechanism of futex2, using 32bit sized futexes. + * + * AUTHOR + * André Almeida andrealmeid@collabora.com + * + * HISTORY + * 2021-Feb-5: Initial version by André andrealmeid@collabora.com + * + *****************************************************************************/ + +#include <errno.h> +#include <error.h> +#include <getopt.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <pthread.h> +#include <sys/shm.h> +#include "futex2test.h" +#include "logging.h" + +#define TEST_NAME "futex2-wait" +#define WAKE_WAIT_US 10000 +#define NR_FUTEXES 30 +struct futex_waitv waitv[NR_FUTEXES]; +u_int32_t futexes[NR_FUTEXES] = {0}; + +void usage(char *prog) +{ + printf("Usage: %s\n", prog); + printf(" -c Use color\n"); + printf(" -h Display this help message\n"); + printf(" -v L Verbosity level: %d=QUIET %d=CRITICAL %d=INFO\n", + VQUIET, VCRITICAL, VINFO); +} + +void *waiterfn(void *arg) +{ + struct timespec64 to64; + int res; + + /* setting absolute timeout for futex2 */ + if (gettime64(CLOCK_MONOTONIC, &to64)) + error("gettime64 failed\n", errno); + + to64.tv_sec++; + + res = futex2_waitv(waitv, NR_FUTEXES, 0, &to64); + if (res < 0) { + ksft_test_result_fail("futex2_waitv returned: %d %s\n", + errno, strerror(errno)); + } else if (res != NR_FUTEXES - 1) { + ksft_test_result_fail("futex2_waitv returned: %d, expecting %d\n", + res, NR_FUTEXES - 1); + } + + return NULL; +} + +int main(int argc, char *argv[]) +{ + pthread_t waiter; + int res, ret = RET_PASS; + int c, i; + + while ((c = getopt(argc, argv, "cht:v:")) != -1) { + switch (c) { + case 'c': + log_color(1); + break; + case 'h': + usage(basename(argv[0])); + exit(0); + case 'v': + log_verbosity(atoi(optarg)); + break; + default: + usage(basename(argv[0])); + exit(1); + } + } + + ksft_print_header(); + ksft_set_plan(2); + ksft_print_msg("%s: Test FUTEX2_WAITV\n", + basename(argv[0])); + + for (i = 0; i < NR_FUTEXES; i++) { + waitv[i].uaddr = &futexes[i]; + waitv[i].flags = FUTEX_32; + waitv[i].val = 0; + } + + /* Private waitv */ + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + res = futex2_wake(waitv[NR_FUTEXES - 1].uaddr, 1, FUTEX_32); + if (res != 1) { + ksft_test_result_fail("futex2_waitv private returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_waitv private\n"); + } + + /* Shared waitv */ + for (i = 0; i < NR_FUTEXES; i++) { + int shm_id = shmget(IPC_PRIVATE, 4096, IPC_CREAT | 0666); + + if (shm_id < 0) { + perror("shmget"); + exit(1); + } + + unsigned int *shared_data = shmat(shm_id, NULL, 0); + + *shared_data = 0; + waitv[i].uaddr = shared_data; + waitv[i].flags = FUTEX_32 | FUTEX_SHARED_FLAG; + waitv[i].val = 0; + } + + if (pthread_create(&waiter, NULL, waiterfn, NULL)) + error("pthread_create failed\n", errno); + + usleep(WAKE_WAIT_US); + + res = futex2_wake(waitv[NR_FUTEXES - 1].uaddr, 1, FUTEX_32 | FUTEX_SHARED_FLAG); + if (res != 1) { + ksft_test_result_fail("futex2_waitv shared returned: %d %s\n", + res ? errno : res, + res ? strerror(errno) : ""); + ret = RET_FAIL; + } else { + ksft_test_result_pass("futex2_waitv shared\n"); + } + + for (i = 0; i < NR_FUTEXES; i++) + shmdt(waitv[i].uaddr); + + ksft_print_cnts(); + return ret; +} diff --git a/tools/testing/selftests/futex/functional/run.sh b/tools/testing/selftests/futex/functional/run.sh index dbe82275617c..6d30a30547da 100755 --- a/tools/testing/selftests/futex/functional/run.sh +++ b/tools/testing/selftests/futex/functional/run.sh @@ -82,3 +82,6 @@ echo

echo ./futex2_wait $COLOR + +echo +./futex2_waitv $COLOR diff --git a/tools/testing/selftests/futex/include/futex2test.h b/tools/testing/selftests/futex/include/futex2test.h index e724d56b917e..0ed3b20935be 100644 --- a/tools/testing/selftests/futex/include/futex2test.h +++ b/tools/testing/selftests/futex/include/futex2test.h @@ -28,6 +28,10 @@ # define FUTEX_32 2 #endif

+#ifndef FUTEX_SHARED_FLAG +#define FUTEX_SHARED_FLAG 8 +#endif + /* * - Y2038 section for 32-bit applications - * @@ -77,3 +81,16 @@ static inline int futex2_wake(volatile void *uaddr, unsigned int nr, unsigned lo { return syscall(__NR_futex_wake, uaddr, nr, flags); } + +/** + * futex2_waitv - Wait at multiple futexes, wake on any + * @waiters: Array of waiters + * @nr_waiters: Length of waiters array + * @flags: Operation flags + * @timo: Optional timeout for operation + */ +static inline int futex2_waitv(volatile struct futex_waitv *waiters, unsigned long nr_waiters, + unsigned long flags, struct timespec64 *timo) +{ + return syscall(__NR_futex_waitv, waiters, nr_waiters, flags, timo); +}

Add support at the existing futex benchmarking code base to enable futex2 calls. `perf bench` tests can be used not only as a way to measure the performance of implementation, but also as stress testing for the kernel infrastructure.

Signed-off-by: André Almeida andrealmeid@collabora.com --- tools/arch/x86/include/asm/unistd_64.h | 12 ++++++ tools/perf/bench/bench.h | 4 ++ tools/perf/bench/futex-hash.c | 24 +++++++++-- tools/perf/bench/futex-requeue.c | 57 ++++++++++++++++++++------ tools/perf/bench/futex-wake-parallel.c | 41 +++++++++++++++--- tools/perf/bench/futex-wake.c | 37 +++++++++++++---- tools/perf/bench/futex.h | 47 +++++++++++++++++++++ tools/perf/builtin-bench.c | 18 ++++++-- 8 files changed, 206 insertions(+), 34 deletions(-)

diff --git a/tools/arch/x86/include/asm/unistd_64.h b/tools/arch/x86/include/asm/unistd_64.h index 4205ed4158bf..191d43d84f04 100644 --- a/tools/arch/x86/include/asm/unistd_64.h +++ b/tools/arch/x86/include/asm/unistd_64.h @@ -17,3 +17,15 @@ #ifndef __NR_setns #define __NR_setns 308 #endif + +#ifndef __NR_futex_wait +# define __NR_futex_wait 447 +#endif + +#ifndef __NR_futex_wake +# define __NR_futex_wake 448 +#endif + +#ifndef __NR_futex_requeue +# define __NR_futex_requeue 450 +#endif diff --git a/tools/perf/bench/bench.h b/tools/perf/bench/bench.h index eac36afab2b3..12346844b354 100644 --- a/tools/perf/bench/bench.h +++ b/tools/perf/bench/bench.h @@ -38,9 +38,13 @@ int bench_mem_memcpy(int argc, const char **argv); int bench_mem_memset(int argc, const char **argv); int bench_mem_find_bit(int argc, const char **argv); int bench_futex_hash(int argc, const char **argv); +int bench_futex2_hash(int argc, const char **argv); int bench_futex_wake(int argc, const char **argv); +int bench_futex2_wake(int argc, const char **argv); int bench_futex_wake_parallel(int argc, const char **argv); +int bench_futex2_wake_parallel(int argc, const char **argv); int bench_futex_requeue(int argc, const char **argv); +int bench_futex2_requeue(int argc, const char **argv); /* pi futexes */ int bench_futex_lock_pi(int argc, const char **argv); int bench_epoll_wait(int argc, const char **argv); diff --git a/tools/perf/bench/futex-hash.c b/tools/perf/bench/futex-hash.c index b65373ce5c4f..1068749af40c 100644 --- a/tools/perf/bench/futex-hash.c +++ b/tools/perf/bench/futex-hash.c @@ -33,7 +33,7 @@ static unsigned int nthreads = 0; static unsigned int nsecs = 10; /* amount of futexes per thread */ static unsigned int nfutexes = 1024; -static bool fshared = false, done = false, silent = false; +static bool fshared = false, done = false, silent = false, futex2 = false; static int futex_flag = 0;

struct timeval bench__start, bench__end, bench__runtime; @@ -85,7 +85,10 @@ static void *workerfn(void *arg) * such as internal waitqueue handling, thus enlarging * the critical region protected by hb->lock. */ - ret = futex_wait(&w->futex[i], 1234, NULL, futex_flag); + if (!futex2) + ret = futex_wait(&w->futex[i], 1234, NULL, futex_flag); + else + ret = futex2_wait(&w->futex[i], 1234, futex_flag, NULL); if (!silent && (!ret || errno != EAGAIN || errno != EWOULDBLOCK)) warn("Non-expected futex return call"); @@ -116,7 +119,7 @@ static void print_summary(void) (int)bench__runtime.tv_sec); }

-int bench_futex_hash(int argc, const char **argv) +static int __bench_futex_hash(int argc, const char **argv) { int ret = 0; cpu_set_t cpuset; @@ -148,7 +151,9 @@ int bench_futex_hash(int argc, const char **argv) if (!worker) goto errmem;

- if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG;

printf("Run summary [PID %d]: %d threads, each operating on %d [%s] futexes for %d secs.\n\n", @@ -228,3 +233,14 @@ int bench_futex_hash(int argc, const char **argv) errmem: err(EXIT_FAILURE, "calloc"); } + +int bench_futex_hash(int argc, const char **argv) +{ + return __bench_futex_hash(argc, argv); +} + +int bench_futex2_hash(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_hash(argc, argv); +} diff --git a/tools/perf/bench/futex-requeue.c b/tools/perf/bench/futex-requeue.c index 5fa23295ee5f..ee5e0da5ba50 100644 --- a/tools/perf/bench/futex-requeue.c +++ b/tools/perf/bench/futex-requeue.c @@ -2,8 +2,8 @@ /* * Copyright (C) 2013 Davidlohr Bueso davidlohr@hp.com * - * futex-requeue: Block a bunch of threads on futex1 and requeue them - * on futex2, N at a time. + * futex-requeue: Block a bunch of threads on addr1 and requeue them + * on addr2, N at a time. * * This program is particularly useful to measure the latency of nthread * requeues without waking up any tasks -- thus mimicking a regular futex_wait. @@ -28,7 +28,10 @@ #include <stdlib.h> #include <sys/time.h>

-static u_int32_t futex1 = 0, futex2 = 0; +static u_int32_t addr1 = 0, addr2 = 0; + +static struct futex_requeue rq1 = { .uaddr = &addr1, .flags = FUTEX_32 }; +static struct futex_requeue rq2 = { .uaddr = &addr2, .flags = FUTEX_32 };

/* * How many tasks to requeue at a time. @@ -37,7 +40,7 @@ static u_int32_t futex1 = 0, futex2 = 0; static unsigned int nrequeue = 1;

static pthread_t *worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; static struct stats requeuetime_stats, requeued_stats; @@ -79,7 +82,11 @@ static void *workerfn(void *arg __maybe_unused) pthread_cond_wait(&thread_worker, &thread_lock); pthread_mutex_unlock(&thread_lock);

- futex_wait(&futex1, 0, NULL, futex_flag); + if (!futex2) + futex_wait(&addr1, 0, NULL, futex_flag); + else + futex2_wait(&addr1, 0, futex_flag, NULL); + return NULL; }

@@ -111,7 +118,7 @@ static void toggle_done(int sig __maybe_unused, done = true; }

-int bench_futex_requeue(int argc, const char **argv) +static int __bench_futex_requeue(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -139,15 +146,20 @@ int bench_futex_requeue(int argc, const char **argv) if (!worker) err(EXIT_FAILURE, "calloc");

- if (!fshared) + if (futex2) { + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + rq1.flags |= FUTEX_SHARED_FLAG * fshared; + rq2.flags |= FUTEX_SHARED_FLAG * fshared; + } else if (!fshared) { futex_flag = FUTEX_PRIVATE_FLAG; + }

if (nrequeue > nthreads) nrequeue = nthreads;

printf("Run summary [PID %d]: Requeuing %d threads (from [%s] %p to %p), " "%d at a time.\n\n", getpid(), nthreads, - fshared ? "shared":"private", &futex1, &futex2, nrequeue); + fshared ? "shared":"private", &addr1, &addr2, nrequeue);

init_stats(&requeued_stats); init_stats(&requeuetime_stats); @@ -176,11 +188,15 @@ int bench_futex_requeue(int argc, const char **argv) gettimeofday(&start, NULL); while (nrequeued < nthreads) { /* - * Do not wakeup any tasks blocked on futex1, allowing + * Do not wakeup any tasks blocked on addr1, allowing * us to really measure futex_wait functionality. */ - nrequeued += futex_cmp_requeue(&futex1, 0, &futex2, 0, - nrequeue, futex_flag); + if (!futex2) + nrequeued += futex_cmp_requeue(&addr1, 0, &addr2, + 0, nrequeue, futex_flag); + else + nrequeued += futex2_requeue(&rq1, &rq2, + 0, nrequeue, 0, 0); }

gettimeofday(&end, NULL); @@ -194,8 +210,12 @@ int bench_futex_requeue(int argc, const char **argv) j + 1, nrequeued, nthreads, runtime.tv_usec / (double)USEC_PER_MSEC); }

- /* everybody should be blocked on futex2, wake'em up */ - nrequeued = futex_wake(&futex2, nrequeued, futex_flag); + /* everybody should be blocked on addr2, wake'em up */ + if (!futex2) + nrequeued = futex_wake(&addr2, nrequeued, futex_flag); + else + nrequeued = futex2_wake(&addr2, nrequeued, futex_flag); + if (nthreads != nrequeued) warnx("couldn't wakeup all tasks (%d/%d)", nrequeued, nthreads);

@@ -220,3 +240,14 @@ int bench_futex_requeue(int argc, const char **argv) usage_with_options(bench_futex_requeue_usage, options); exit(EXIT_FAILURE); } + +int bench_futex_requeue(int argc, const char **argv) +{ + return __bench_futex_requeue(argc, argv); +} + +int bench_futex2_requeue(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_requeue(argc, argv); +} diff --git a/tools/perf/bench/futex-wake-parallel.c b/tools/perf/bench/futex-wake-parallel.c index 6e6f5247e1fe..cac90fc0bfb3 100644 --- a/tools/perf/bench/futex-wake-parallel.c +++ b/tools/perf/bench/futex-wake-parallel.c @@ -17,6 +17,12 @@ int bench_futex_wake_parallel(int argc __maybe_unused, const char **argv __maybe pr_err("%s: pthread_barrier_t unavailable, disabling this test...\n", __func__); return 0; } + +int bench_futex2_wake_parallel(int argc __maybe_unused, const char **argv __maybe_unused) +{ + pr_err("%s: pthread_barrier_t unavailable, disabling this test...\n", __func__); + return 0; +} #else /* HAVE_PTHREAD_BARRIER */ /* For the CLR_() macros */ #include <string.h> @@ -47,7 +53,7 @@ static unsigned int nwakes = 1; static u_int32_t futex = 0;

static pthread_t *blocked_worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static unsigned int nblocked_threads = 0, nwaking_threads = 0; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; @@ -78,7 +84,11 @@ static void *waking_workerfn(void *arg)

gettimeofday(&start, NULL);

- waker->nwoken = futex_wake(&futex, nwakes, futex_flag); + if (!futex2) + waker->nwoken = futex_wake(&futex, nwakes, futex_flag); + else + waker->nwoken = futex2_wake(&futex, nwakes, futex_flag); + if (waker->nwoken != nwakes) warnx("couldn't wakeup all tasks (%d/%d)", waker->nwoken, nwakes); @@ -129,8 +139,13 @@ static void *blocked_workerfn(void *arg __maybe_unused) pthread_mutex_unlock(&thread_lock);

while (1) { /* handle spurious wakeups */ - if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR) - break; + if (!futex2) { + if (futex_wait(&futex, 0, NULL, futex_flag) != EINTR) + break; + } else { + if (futex2_wait(&futex, 0, futex_flag, NULL) != EINTR) + break; + } }

pthread_exit(NULL); @@ -217,7 +232,7 @@ static void toggle_done(int sig __maybe_unused, done = true; }

-int bench_futex_wake_parallel(int argc, const char **argv) +static int __bench_futex_wake_parallel(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -261,7 +276,9 @@ int bench_futex_wake_parallel(int argc, const char **argv) if (!blocked_worker) err(EXIT_FAILURE, "calloc");

- if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG;

printf("Run summary [PID %d]: blocking on %d threads (at [%s] " @@ -321,4 +338,16 @@ int bench_futex_wake_parallel(int argc, const char **argv) free(blocked_worker); return ret; } + +int bench_futex_wake_parallel(int argc, const char **argv) +{ + return __bench_futex_wake_parallel(argc, argv); +} + +int bench_futex2_wake_parallel(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_wake_parallel(argc, argv); +} + #endif /* HAVE_PTHREAD_BARRIER */ diff --git a/tools/perf/bench/futex-wake.c b/tools/perf/bench/futex-wake.c index 6d217868f53c..546d2818eed8 100644 --- a/tools/perf/bench/futex-wake.c +++ b/tools/perf/bench/futex-wake.c @@ -38,7 +38,7 @@ static u_int32_t futex1 = 0; static unsigned int nwakes = 1;

pthread_t *worker; -static bool done = false, silent = false, fshared = false; +static bool done = false, silent = false, fshared = false, futex2 = false; static pthread_mutex_t thread_lock; static pthread_cond_t thread_parent, thread_worker; static struct stats waketime_stats, wakeup_stats; @@ -68,8 +68,13 @@ static void *workerfn(void *arg __maybe_unused) pthread_mutex_unlock(&thread_lock);

while (1) { - if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR) - break; + if (!futex2) { + if (futex_wait(&futex1, 0, NULL, futex_flag) != EINTR) + break; + } else { + if (futex2_wait(&futex1, 0, futex_flag, NULL) != EINTR) + break; + } }

pthread_exit(NULL); @@ -117,7 +122,7 @@ static void toggle_done(int sig __maybe_unused, done = true; }

-int bench_futex_wake(int argc, const char **argv) +static int __bench_futex_wake(int argc, const char **argv) { int ret = 0; unsigned int i, j; @@ -147,7 +152,9 @@ int bench_futex_wake(int argc, const char **argv) if (!worker) err(EXIT_FAILURE, "calloc");

- if (!fshared) + if (futex2) + futex_flag = FUTEX_32 | (fshared * FUTEX_SHARED_FLAG); + else if (!fshared) futex_flag = FUTEX_PRIVATE_FLAG;

printf("Run summary [PID %d]: blocking on %d threads (at [%s] futex %p), " @@ -179,9 +186,14 @@ int bench_futex_wake(int argc, const char **argv)

/* Ok, all threads are patiently blocked, start waking folks up */ gettimeofday(&start, NULL); - while (nwoken != nthreads) - nwoken += futex_wake(&futex1, nwakes, futex_flag); + while (nwoken != nthreads) { + if (!futex2) + nwoken += futex_wake(&futex1, nwakes, futex_flag); + else + nwoken += futex2_wake(&futex1, nwakes, futex_flag); + } gettimeofday(&end, NULL); + timersub(&end, &start, &runtime);

update_stats(&wakeup_stats, nwoken); @@ -211,3 +223,14 @@ int bench_futex_wake(int argc, const char **argv) free(worker); return ret; } + +int bench_futex_wake(int argc, const char **argv) +{ + return __bench_futex_wake(argc, argv); +} + +int bench_futex2_wake(int argc, const char **argv) +{ + futex2 = true; + return __bench_futex_wake(argc, argv); +} diff --git a/tools/perf/bench/futex.h b/tools/perf/bench/futex.h index 31b53cc7d5bc..6b2213cf3f64 100644 --- a/tools/perf/bench/futex.h +++ b/tools/perf/bench/futex.h @@ -86,4 +86,51 @@ futex_cmp_requeue(u_int32_t *uaddr, u_int32_t val, u_int32_t *uaddr2, int nr_wak return futex(uaddr, FUTEX_CMP_REQUEUE, nr_wake, nr_requeue, uaddr2, val, opflags); } + +/** + * futex2_wait - Wait at uaddr if *uaddr == val, until timo. + * @uaddr: User address to wait for + * @val: Expected value at uaddr + * @flags: Operation options + * @timo: Optional timeout + * + * Return: 0 on success, error code otherwise + */ +static inline int futex2_wait(volatile void *uaddr, unsigned long val, + unsigned long flags, struct timespec *timo) +{ + return syscall(__NR_futex_wait, uaddr, val, flags, timo); +} + +/** + * futex2_wake - Wake a number of waiters waiting at uaddr + * @uaddr: Address to wake + * @nr: Number of waiters to wake + * @flags: Operation options + * + * Return: number of waked futexes + */ +static inline int futex2_wake(volatile void *uaddr, unsigned int nr, unsigned long flags) +{ + return syscall(__NR_futex_wake, uaddr, nr, flags); +} + +/** + * futex2_requeue - Requeue waiters from an address to another one + * @uaddr1: Address where waiters are currently waiting on + * @uaddr2: New address to wait + * @nr_wake: Number of waiters at uaddr1 to be wake + * @nr_requeue: After waking nr_wake, number of waiters to be requeued + * @cmpval: Expected value at uaddr1 + * @flags: Operation options + * + * Return: waked futexes + requeued futexes at uaddr1 + */ +static inline int futex2_requeue(volatile struct futex_requeue *uaddr1, + volatile struct futex_requeue *uaddr2, + unsigned int nr_wake, unsigned int nr_requeue, + unsigned int cmpval, unsigned long flags) +{ + return syscall(__NR_futex_requeue, uaddr1, uaddr2, nr_wake, nr_requeue, cmpval, flags); +} #endif /* _FUTEX_H */ diff --git a/tools/perf/builtin-bench.c b/tools/perf/builtin-bench.c index 62a7b7420a44..e41a95ad2db6 100644 --- a/tools/perf/builtin-bench.c +++ b/tools/perf/builtin-bench.c @@ -12,10 +12,11 @@ * * sched ... scheduler and IPC performance * syscall ... System call performance - * mem ... memory access performance - * numa ... NUMA scheduling and MM performance - * futex ... Futex performance - * epoll ... Event poll performance + * mem ... memory access performance + * numa ... NUMA scheduling and MM performance + * futex ... Futex performance + * futex2 ... Futex2 performance + * epoll ... Event poll performance */ #include <subcmd/parse-options.h> #include "builtin.h" @@ -75,6 +76,14 @@ static struct bench futex_benchmarks[] = { { NULL, NULL, NULL } };

+static struct bench futex2_benchmarks[] = { + { "hash", "Benchmark for futex2 hash table", bench_futex2_hash }, + { "wake", "Benchmark for futex2 wake calls", bench_futex2_wake }, + { "wake-parallel", "Benchmark for parallel futex2 wake calls", bench_futex2_wake_parallel }, + { "requeue", "Benchmark for futex2 requeue calls", bench_futex2_requeue }, + { NULL, NULL, NULL } +}; + #ifdef HAVE_EVENTFD_SUPPORT static struct bench epoll_benchmarks[] = { { "wait", "Benchmark epoll concurrent epoll_waits", bench_epoll_wait }, @@ -105,6 +114,7 @@ static struct collection collections[] = { { "numa", "NUMA scheduling and MM benchmarks", numa_benchmarks }, #endif {"futex", "Futex stressing benchmarks", futex_benchmarks }, + {"futex2", "Futex2 stressing benchmarks", futex2_benchmarks }, #ifdef HAVE_EVENTFD_SUPPORT {"epoll", "Epoll stressing benchmarks", epoll_benchmarks }, #endif