~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/futex/waitwake.c

Version: ~ [ linux-6.1-rc7 ] ~ [ linux-6.0.10 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.80 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.156 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.225 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.267 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.300 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.334 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 
  3 #include <linux/sched/task.h>
  4 #include <linux/sched/signal.h>
  5 #include <linux/freezer.h>
  6 
  7 #include "futex.h"
  8 
  9 /*
 10  * READ this before attempting to hack on futexes!
 11  *
 12  * Basic futex operation and ordering guarantees
 13  * =============================================
 14  *
 15  * The waiter reads the futex value in user space and calls
 16  * futex_wait(). This function computes the hash bucket and acquires
 17  * the hash bucket lock. After that it reads the futex user space value
 18  * again and verifies that the data has not changed. If it has not changed
 19  * it enqueues itself into the hash bucket, releases the hash bucket lock
 20  * and schedules.
 21  *
 22  * The waker side modifies the user space value of the futex and calls
 23  * futex_wake(). This function computes the hash bucket and acquires the
 24  * hash bucket lock. Then it looks for waiters on that futex in the hash
 25  * bucket and wakes them.
 26  *
 27  * In futex wake up scenarios where no tasks are blocked on a futex, taking
 28  * the hb spinlock can be avoided and simply return. In order for this
 29  * optimization to work, ordering guarantees must exist so that the waiter
 30  * being added to the list is acknowledged when the list is concurrently being
 31  * checked by the waker, avoiding scenarios like the following:
 32  *
 33  * CPU 0                               CPU 1
 34  * val = *futex;
 35  * sys_futex(WAIT, futex, val);
 36  *   futex_wait(futex, val);
 37  *   uval = *futex;
 38  *                                     *futex = newval;
 39  *                                     sys_futex(WAKE, futex);
 40  *                                       futex_wake(futex);
 41  *                                       if (queue_empty())
 42  *                                         return;
 43  *   if (uval == val)
 44  *      lock(hash_bucket(futex));
 45  *      queue();
 46  *     unlock(hash_bucket(futex));
 47  *     schedule();
 48  *
 49  * This would cause the waiter on CPU 0 to wait forever because it
 50  * missed the transition of the user space value from val to newval
 51  * and the waker did not find the waiter in the hash bucket queue.
 52  *
 53  * The correct serialization ensures that a waiter either observes
 54  * the changed user space value before blocking or is woken by a
 55  * concurrent waker:
 56  *
 57  * CPU 0                                 CPU 1
 58  * val = *futex;
 59  * sys_futex(WAIT, futex, val);
 60  *   futex_wait(futex, val);
 61  *
 62  *   waiters++; (a)
 63  *   smp_mb(); (A) <-- paired with -.
 64  *                                  |
 65  *   lock(hash_bucket(futex));      |
 66  *                                  |
 67  *   uval = *futex;                 |
 68  *                                  |        *futex = newval;
 69  *                                  |        sys_futex(WAKE, futex);
 70  *                                  |          futex_wake(futex);
 71  *                                  |
 72  *                                  `--------> smp_mb(); (B)
 73  *   if (uval == val)
 74  *     queue();
 75  *     unlock(hash_bucket(futex));
 76  *     schedule();                         if (waiters)
 77  *                                           lock(hash_bucket(futex));
 78  *   else                                    wake_waiters(futex);
 79  *     waiters--; (b)                        unlock(hash_bucket(futex));
 80  *
 81  * Where (A) orders the waiters increment and the futex value read through
 82  * atomic operations (see futex_hb_waiters_inc) and where (B) orders the write
 83  * to futex and the waiters read (see futex_hb_waiters_pending()).
 84  *
 85  * This yields the following case (where X:=waiters, Y:=futex):
 86  *
 87  *      X = Y = 0
 88  *
 89  *      w[X]=1          w[Y]=1
 90  *      MB              MB
 91  *      r[Y]=y          r[X]=x
 92  *
 93  * Which guarantees that x==0 && y==0 is impossible; which translates back into
 94  * the guarantee that we cannot both miss the futex variable change and the
 95  * enqueue.
 96  *
 97  * Note that a new waiter is accounted for in (a) even when it is possible that
 98  * the wait call can return error, in which case we backtrack from it in (b).
 99  * Refer to the comment in futex_q_lock().
100  *
101  * Similarly, in order to account for waiters being requeued on another
102  * address we always increment the waiters for the destination bucket before
103  * acquiring the lock. It then decrements them again  after releasing it -
104  * the code that actually moves the futex(es) between hash buckets (requeue_futex)
105  * will do the additional required waiter count housekeeping. This is done for
106  * double_lock_hb() and double_unlock_hb(), respectively.
107  */
108 
109 /*
110  * The hash bucket lock must be held when this is called.
111  * Afterwards, the futex_q must not be accessed. Callers
112  * must ensure to later call wake_up_q() for the actual
113  * wakeups to occur.
114  */
115 void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
116 {
117         struct task_struct *p = q->task;
118 
119         if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
120                 return;
121 
122         get_task_struct(p);
123         __futex_unqueue(q);
124         /*
125          * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
126          * is written, without taking any locks. This is possible in the event
127          * of a spurious wakeup, for example. A memory barrier is required here
128          * to prevent the following store to lock_ptr from getting ahead of the
129          * plist_del in __futex_unqueue().
130          */
131         smp_store_release(&q->lock_ptr, NULL);
132 
133         /*
134          * Queue the task for later wakeup for after we've released
135          * the hb->lock.
136          */
137         wake_q_add_safe(wake_q, p);
138 }
139 
140 /*
141  * Wake up waiters matching bitset queued on this futex (uaddr).
142  */
143 int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
144 {
145         struct futex_hash_bucket *hb;
146         struct futex_q *this, *next;
147         union futex_key key = FUTEX_KEY_INIT;
148         int ret;
149         DEFINE_WAKE_Q(wake_q);
150 
151         if (!bitset)
152                 return -EINVAL;
153 
154         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
155         if (unlikely(ret != 0))
156                 return ret;
157 
158         hb = futex_hash(&key);
159 
160         /* Make sure we really have tasks to wakeup */
161         if (!futex_hb_waiters_pending(hb))
162                 return ret;
163 
164         spin_lock(&hb->lock);
165 
166         plist_for_each_entry_safe(this, next, &hb->chain, list) {
167                 if (futex_match (&this->key, &key)) {
168                         if (this->pi_state || this->rt_waiter) {
169                                 ret = -EINVAL;
170                                 break;
171                         }
172 
173                         /* Check if one of the bits is set in both bitsets */
174                         if (!(this->bitset & bitset))
175                                 continue;
176 
177                         futex_wake_mark(&wake_q, this);
178                         if (++ret >= nr_wake)
179                                 break;
180                 }
181         }
182 
183         spin_unlock(&hb->lock);
184         wake_up_q(&wake_q);
185         return ret;
186 }
187 
188 static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
189 {
190         unsigned int op =         (encoded_op & 0x70000000) >> 28;
191         unsigned int cmp =        (encoded_op & 0x0f000000) >> 24;
192         int oparg = sign_extend32((encoded_op & 0x00fff000) >> 12, 11);
193         int cmparg = sign_extend32(encoded_op & 0x00000fff, 11);
194         int oldval, ret;
195 
196         if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) {
197                 if (oparg < 0 || oparg > 31) {
198                         char comm[sizeof(current->comm)];
199                         /*
200                          * kill this print and return -EINVAL when userspace
201                          * is sane again
202                          */
203                         pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
204                                         get_task_comm(comm, current), oparg);
205                         oparg &= 31;
206                 }
207                 oparg = 1 << oparg;
208         }
209 
210         pagefault_disable();
211         ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
212         pagefault_enable();
213         if (ret)
214                 return ret;
215 
216         switch (cmp) {
217         case FUTEX_OP_CMP_EQ:
218                 return oldval == cmparg;
219         case FUTEX_OP_CMP_NE:
220                 return oldval != cmparg;
221         case FUTEX_OP_CMP_LT:
222                 return oldval < cmparg;
223         case FUTEX_OP_CMP_GE:
224                 return oldval >= cmparg;
225         case FUTEX_OP_CMP_LE:
226                 return oldval <= cmparg;
227         case FUTEX_OP_CMP_GT:
228                 return oldval > cmparg;
229         default:
230                 return -ENOSYS;
231         }
232 }
233 
234 /*
235  * Wake up all waiters hashed on the physical page that is mapped
236  * to this virtual address:
237  */
238 int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
239                   int nr_wake, int nr_wake2, int op)
240 {
241         union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
242         struct futex_hash_bucket *hb1, *hb2;
243         struct futex_q *this, *next;
244         int ret, op_ret;
245         DEFINE_WAKE_Q(wake_q);
246 
247 retry:
248         ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
249         if (unlikely(ret != 0))
250                 return ret;
251         ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
252         if (unlikely(ret != 0))
253                 return ret;
254 
255         hb1 = futex_hash(&key1);
256         hb2 = futex_hash(&key2);
257 
258 retry_private:
259         double_lock_hb(hb1, hb2);
260         op_ret = futex_atomic_op_inuser(op, uaddr2);
261         if (unlikely(op_ret < 0)) {
262                 double_unlock_hb(hb1, hb2);
263 
264                 if (!IS_ENABLED(CONFIG_MMU) ||
265                     unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
266                         /*
267                          * we don't get EFAULT from MMU faults if we don't have
268                          * an MMU, but we might get them from range checking
269                          */
270                         ret = op_ret;
271                         return ret;
272                 }
273 
274                 if (op_ret == -EFAULT) {
275                         ret = fault_in_user_writeable(uaddr2);
276                         if (ret)
277                                 return ret;
278                 }
279 
280                 cond_resched();
281                 if (!(flags & FLAGS_SHARED))
282                         goto retry_private;
283                 goto retry;
284         }
285 
286         plist_for_each_entry_safe(this, next, &hb1->chain, list) {
287                 if (futex_match (&this->key, &key1)) {
288                         if (this->pi_state || this->rt_waiter) {
289                                 ret = -EINVAL;
290                                 goto out_unlock;
291                         }
292                         futex_wake_mark(&wake_q, this);
293                         if (++ret >= nr_wake)
294                                 break;
295                 }
296         }
297 
298         if (op_ret > 0) {
299                 op_ret = 0;
300                 plist_for_each_entry_safe(this, next, &hb2->chain, list) {
301                         if (futex_match (&this->key, &key2)) {
302                                 if (this->pi_state || this->rt_waiter) {
303                                         ret = -EINVAL;
304                                         goto out_unlock;
305                                 }
306                                 futex_wake_mark(&wake_q, this);
307                                 if (++op_ret >= nr_wake2)
308                                         break;
309                         }
310                 }
311                 ret += op_ret;
312         }
313 
314 out_unlock:
315         double_unlock_hb(hb1, hb2);
316         wake_up_q(&wake_q);
317         return ret;
318 }
319 
320 static long futex_wait_restart(struct restart_block *restart);
321 
322 /**
323  * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
324  * @hb:         the futex hash bucket, must be locked by the caller
325  * @q:          the futex_q to queue up on
326  * @timeout:    the prepared hrtimer_sleeper, or null for no timeout
327  */
328 void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
329                             struct hrtimer_sleeper *timeout)
330 {
331         /*
332          * The task state is guaranteed to be set before another task can
333          * wake it. set_current_state() is implemented using smp_store_mb() and
334          * futex_queue() calls spin_unlock() upon completion, both serializing
335          * access to the hash list and forcing another memory barrier.
336          */
337         set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
338         futex_queue(q, hb);
339 
340         /* Arm the timer */
341         if (timeout)
342                 hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
343 
344         /*
345          * If we have been removed from the hash list, then another task
346          * has tried to wake us, and we can skip the call to schedule().
347          */
348         if (likely(!plist_node_empty(&q->list))) {
349                 /*
350                  * If the timer has already expired, current will already be
351                  * flagged for rescheduling. Only call schedule if there
352                  * is no timeout, or if it has yet to expire.
353                  */
354                 if (!timeout || timeout->task)
355                         schedule();
356         }
357         __set_current_state(TASK_RUNNING);
358 }
359 
360 /**
361  * unqueue_multiple - Remove various futexes from their hash bucket
362  * @v:     The list of futexes to unqueue
363  * @count: Number of futexes in the list
364  *
365  * Helper to unqueue a list of futexes. This can't fail.
366  *
367  * Return:
368  *  - >=0 - Index of the last futex that was awoken;
369  *  - -1  - No futex was awoken
370  */
371 static int unqueue_multiple(struct futex_vector *v, int count)
372 {
373         int ret = -1, i;
374 
375         for (i = 0; i < count; i++) {
376                 if (!futex_unqueue(&v[i].q))
377                         ret = i;
378         }
379 
380         return ret;
381 }
382 
383 /**
384  * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
385  * @vs:         The futex list to wait on
386  * @count:      The size of the list
387  * @woken:      Index of the last woken futex, if any. Used to notify the
388  *              caller that it can return this index to userspace (return parameter)
389  *
390  * Prepare multiple futexes in a single step and enqueue them. This may fail if
391  * the futex list is invalid or if any futex was already awoken. On success the
392  * task is ready to interruptible sleep.
393  *
394  * Return:
395  *  -  1 - One of the futexes was woken by another thread
396  *  -  0 - Success
397  *  - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
398  */
399 static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
400 {
401         struct futex_hash_bucket *hb;
402         bool retry = false;
403         int ret, i;
404         u32 uval;
405 
406         /*
407          * Enqueuing multiple futexes is tricky, because we need to enqueue
408          * each futex on the list before dealing with the next one to avoid
409          * deadlocking on the hash bucket. But, before enqueuing, we need to
410          * make sure that current->state is TASK_INTERRUPTIBLE, so we don't
411          * lose any wake events, which cannot be done before the get_futex_key
412          * of the next key, because it calls get_user_pages, which can sleep.
413          * Thus, we fetch the list of futexes keys in two steps, by first
414          * pinning all the memory keys in the futex key, and only then we read
415          * each key and queue the corresponding futex.
416          *
417          * Private futexes doesn't need to recalculate hash in retry, so skip
418          * get_futex_key() when retrying.
419          */
420 retry:
421         for (i = 0; i < count; i++) {
422                 if ((vs[i].w.flags & FUTEX_PRIVATE_FLAG) && retry)
423                         continue;
424 
425                 ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),
426                                     !(vs[i].w.flags & FUTEX_PRIVATE_FLAG),
427                                     &vs[i].q.key, FUTEX_READ);
428 
429                 if (unlikely(ret))
430                         return ret;
431         }
432 
433         set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
434 
435         for (i = 0; i < count; i++) {
436                 u32 __user *uaddr = (u32 __user *)(unsigned long)vs[i].w.uaddr;
437                 struct futex_q *q = &vs[i].q;
438                 u32 val = (u32)vs[i].w.val;
439 
440                 hb = futex_q_lock(q);
441                 ret = futex_get_value_locked(&uval, uaddr);
442 
443                 if (!ret && uval == val) {
444                         /*
445                          * The bucket lock can't be held while dealing with the
446                          * next futex. Queue each futex at this moment so hb can
447                          * be unlocked.
448                          */
449                         futex_queue(q, hb);
450                         continue;
451                 }
452 
453                 futex_q_unlock(hb);
454                 __set_current_state(TASK_RUNNING);
455 
456                 /*
457                  * Even if something went wrong, if we find out that a futex
458                  * was woken, we don't return error and return this index to
459                  * userspace
460                  */
461                 *woken = unqueue_multiple(vs, i);
462                 if (*woken >= 0)
463                         return 1;
464 
465                 if (ret) {
466                         /*
467                          * If we need to handle a page fault, we need to do so
468                          * without any lock and any enqueued futex (otherwise
469                          * we could lose some wakeup). So we do it here, after
470                          * undoing all the work done so far. In success, we
471                          * retry all the work.
472                          */
473                         if (get_user(uval, uaddr))
474                                 return -EFAULT;
475 
476                         retry = true;
477                         goto retry;
478                 }
479 
480                 if (uval != val)
481                         return -EWOULDBLOCK;
482         }
483 
484         return 0;
485 }
486 
487 /**
488  * futex_sleep_multiple - Check sleeping conditions and sleep
489  * @vs:    List of futexes to wait for
490  * @count: Length of vs
491  * @to:    Timeout
492  *
493  * Sleep if and only if the timeout hasn't expired and no futex on the list has
494  * been woken up.
495  */
496 static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count,
497                                  struct hrtimer_sleeper *to)
498 {
499         if (to && !to->task)
500                 return;
501 
502         for (; count; count--, vs++) {
503                 if (!READ_ONCE(vs->q.lock_ptr))
504                         return;
505         }
506 
507         schedule();
508 }
509 
510 /**
511  * futex_wait_multiple - Prepare to wait on and enqueue several futexes
512  * @vs:         The list of futexes to wait on
513  * @count:      The number of objects
514  * @to:         Timeout before giving up and returning to userspace
515  *
516  * Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this function
517  * sleeps on a group of futexes and returns on the first futex that is
518  * wake, or after the timeout has elapsed.
519  *
520  * Return:
521  *  - >=0 - Hint to the futex that was awoken
522  *  - <0  - On error
523  */
524 int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
525                         struct hrtimer_sleeper *to)
526 {
527         int ret, hint = 0;
528 
529         if (to)
530                 hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
531 
532         while (1) {
533                 ret = futex_wait_multiple_setup(vs, count, &hint);
534                 if (ret) {
535                         if (ret > 0) {
536                                 /* A futex was woken during setup */
537                                 ret = hint;
538                         }
539                         return ret;
540                 }
541 
542                 futex_sleep_multiple(vs, count, to);
543 
544                 __set_current_state(TASK_RUNNING);
545 
546                 ret = unqueue_multiple(vs, count);
547                 if (ret >= 0)
548                         return ret;
549 
550                 if (to && !to->task)
551                         return -ETIMEDOUT;
552                 else if (signal_pending(current))
553                         return -ERESTARTSYS;
554                 /*
555                  * The final case is a spurious wakeup, for
556                  * which just retry.
557                  */
558         }
559 }
560 
561 /**
562  * futex_wait_setup() - Prepare to wait on a futex
563  * @uaddr:      the futex userspace address
564  * @val:        the expected value
565  * @flags:      futex flags (FLAGS_SHARED, etc.)
566  * @q:          the associated futex_q
567  * @hb:         storage for hash_bucket pointer to be returned to caller
568  *
569  * Setup the futex_q and locate the hash_bucket.  Get the futex value and
570  * compare it with the expected value.  Handle atomic faults internally.
571  * Return with the hb lock held on success, and unlocked on failure.
572  *
573  * Return:
574  *  -  0 - uaddr contains val and hb has been locked;
575  *  - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
576  */
577 int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
578                      struct futex_q *q, struct futex_hash_bucket **hb)
579 {
580         u32 uval;
581         int ret;
582 
583         /*
584          * Access the page AFTER the hash-bucket is locked.
585          * Order is important:
586          *
587          *   Userspace waiter: val = var; if (cond(val)) futex_wait(&var, val);
588          *   Userspace waker:  if (cond(var)) { var = new; futex_wake(&var); }
589          *
590          * The basic logical guarantee of a futex is that it blocks ONLY
591          * if cond(var) is known to be true at the time of blocking, for
592          * any cond.  If we locked the hash-bucket after testing *uaddr, that
593          * would open a race condition where we could block indefinitely with
594          * cond(var) false, which would violate the guarantee.
595          *
596          * On the other hand, we insert q and release the hash-bucket only
597          * after testing *uaddr.  This guarantees that futex_wait() will NOT
598          * absorb a wakeup if *uaddr does not match the desired values
599          * while the syscall executes.
600          */
601 retry:
602         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
603         if (unlikely(ret != 0))
604                 return ret;
605 
606 retry_private:
607         *hb = futex_q_lock(q);
608 
609         ret = futex_get_value_locked(&uval, uaddr);
610 
611         if (ret) {
612                 futex_q_unlock(*hb);
613 
614                 ret = get_user(uval, uaddr);
615                 if (ret)
616                         return ret;
617 
618                 if (!(flags & FLAGS_SHARED))
619                         goto retry_private;
620 
621                 goto retry;
622         }
623 
624         if (uval != val) {
625                 futex_q_unlock(*hb);
626                 ret = -EWOULDBLOCK;
627         }
628 
629         return ret;
630 }
631 
632 int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset)
633 {
634         struct hrtimer_sleeper timeout, *to;
635         struct restart_block *restart;
636         struct futex_hash_bucket *hb;
637         struct futex_q q = futex_q_init;
638         int ret;
639 
640         if (!bitset)
641                 return -EINVAL;
642         q.bitset = bitset;
643 
644         to = futex_setup_timer(abs_time, &timeout, flags,
645                                current->timer_slack_ns);
646 retry:
647         /*
648          * Prepare to wait on uaddr. On success, it holds hb->lock and q
649          * is initialized.
650          */
651         ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
652         if (ret)
653                 goto out;
654 
655         /* futex_queue and wait for wakeup, timeout, or a signal. */
656         futex_wait_queue(hb, &q, to);
657 
658         /* If we were woken (and unqueued), we succeeded, whatever. */
659         ret = 0;
660         if (!futex_unqueue(&q))
661                 goto out;
662         ret = -ETIMEDOUT;
663         if (to && !to->task)
664                 goto out;
665 
666         /*
667          * We expect signal_pending(current), but we might be the
668          * victim of a spurious wakeup as well.
669          */
670         if (!signal_pending(current))
671                 goto retry;
672 
673         ret = -ERESTARTSYS;
674         if (!abs_time)
675                 goto out;
676 
677         restart = &current->restart_block;
678         restart->futex.uaddr = uaddr;
679         restart->futex.val = val;
680         restart->futex.time = *abs_time;
681         restart->futex.bitset = bitset;
682         restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
683 
684         ret = set_restart_fn(restart, futex_wait_restart);
685 
686 out:
687         if (to) {
688                 hrtimer_cancel(&to->timer);
689                 destroy_hrtimer_on_stack(&to->timer);
690         }
691         return ret;
692 }
693 
694 static long futex_wait_restart(struct restart_block *restart)
695 {
696         u32 __user *uaddr = restart->futex.uaddr;
697         ktime_t t, *tp = NULL;
698 
699         if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
700                 t = restart->futex.time;
701                 tp = &t;
702         }
703         restart->fn = do_no_restart_syscall;
704 
705         return (long)futex_wait(uaddr, restart->futex.flags,
706                                 restart->futex.val, tp, restart->futex.bitset);
707 }
708 
709 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp