~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/sched/wait_bit.c

Version: ~ [ linux-4.17-rc6 ] ~ [ linux-4.16.10 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.42 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.101 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.132 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.51 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.109 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.56 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.101 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * The implementation of the wait_bit*() and related waiting APIs:
  3  */
  4 #include "sched.h"
  5 
  6 #define WAIT_TABLE_BITS 8
  7 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
  8 
  9 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
 10 
 11 wait_queue_head_t *bit_waitqueue(void *word, int bit)
 12 {
 13         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
 14         unsigned long val = (unsigned long)word << shift | bit;
 15 
 16         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
 17 }
 18 EXPORT_SYMBOL(bit_waitqueue);
 19 
 20 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
 21 {
 22         struct wait_bit_key *key = arg;
 23         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
 24 
 25         if (wait_bit->key.flags != key->flags ||
 26                         wait_bit->key.bit_nr != key->bit_nr ||
 27                         test_bit(key->bit_nr, key->flags))
 28                 return 0;
 29 
 30         return autoremove_wake_function(wq_entry, mode, sync, key);
 31 }
 32 EXPORT_SYMBOL(wake_bit_function);
 33 
 34 /*
 35  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
 36  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
 37  * permitted return codes. Nonzero return codes halt waiting and return.
 38  */
 39 int __sched
 40 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 41               wait_bit_action_f *action, unsigned mode)
 42 {
 43         int ret = 0;
 44 
 45         do {
 46                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
 47                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
 48                         ret = (*action)(&wbq_entry->key, mode);
 49         } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
 50 
 51         finish_wait(wq_head, &wbq_entry->wq_entry);
 52 
 53         return ret;
 54 }
 55 EXPORT_SYMBOL(__wait_on_bit);
 56 
 57 int __sched out_of_line_wait_on_bit(void *word, int bit,
 58                                     wait_bit_action_f *action, unsigned mode)
 59 {
 60         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 61         DEFINE_WAIT_BIT(wq_entry, word, bit);
 62 
 63         return __wait_on_bit(wq_head, &wq_entry, action, mode);
 64 }
 65 EXPORT_SYMBOL(out_of_line_wait_on_bit);
 66 
 67 int __sched out_of_line_wait_on_bit_timeout(
 68         void *word, int bit, wait_bit_action_f *action,
 69         unsigned mode, unsigned long timeout)
 70 {
 71         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 72         DEFINE_WAIT_BIT(wq_entry, word, bit);
 73 
 74         wq_entry.key.timeout = jiffies + timeout;
 75 
 76         return __wait_on_bit(wq_head, &wq_entry, action, mode);
 77 }
 78 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
 79 
 80 int __sched
 81 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 82                         wait_bit_action_f *action, unsigned mode)
 83 {
 84         int ret = 0;
 85 
 86         for (;;) {
 87                 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
 88                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
 89                         ret = action(&wbq_entry->key, mode);
 90                         /*
 91                          * See the comment in prepare_to_wait_event().
 92                          * finish_wait() does not necessarily takes wwq_head->lock,
 93                          * but test_and_set_bit() implies mb() which pairs with
 94                          * smp_mb__after_atomic() before wake_up_page().
 95                          */
 96                         if (ret)
 97                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 98                 }
 99                 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
100                         if (!ret)
101                                 finish_wait(wq_head, &wbq_entry->wq_entry);
102                         return 0;
103                 } else if (ret) {
104                         return ret;
105                 }
106         }
107 }
108 EXPORT_SYMBOL(__wait_on_bit_lock);
109 
110 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
111                                          wait_bit_action_f *action, unsigned mode)
112 {
113         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
114         DEFINE_WAIT_BIT(wq_entry, word, bit);
115 
116         return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
117 }
118 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
119 
120 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
121 {
122         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
123 
124         if (waitqueue_active(wq_head))
125                 __wake_up(wq_head, TASK_NORMAL, 1, &key);
126 }
127 EXPORT_SYMBOL(__wake_up_bit);
128 
129 /**
130  * wake_up_bit - wake up a waiter on a bit
131  * @word: the word being waited on, a kernel virtual address
132  * @bit: the bit of the word being waited on
133  *
134  * There is a standard hashed waitqueue table for generic use. This
135  * is the part of the hashtable's accessor API that wakes up waiters
136  * on a bit. For instance, if one were to have waiters on a bitflag,
137  * one would call wake_up_bit() after clearing the bit.
138  *
139  * In order for this to function properly, as it uses waitqueue_active()
140  * internally, some kind of memory barrier must be done prior to calling
141  * this. Typically, this will be smp_mb__after_atomic(), but in some
142  * cases where bitflags are manipulated non-atomically under a lock, one
143  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
144  * because spin_unlock() does not guarantee a memory barrier.
145  */
146 void wake_up_bit(void *word, int bit)
147 {
148         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
149 }
150 EXPORT_SYMBOL(wake_up_bit);
151 
152 wait_queue_head_t *__var_waitqueue(void *p)
153 {
154         return bit_wait_table + hash_ptr(p, WAIT_TABLE_BITS);
155 }
156 EXPORT_SYMBOL(__var_waitqueue);
157 
158 static int
159 var_wake_function(struct wait_queue_entry *wq_entry, unsigned int mode,
160                   int sync, void *arg)
161 {
162         struct wait_bit_key *key = arg;
163         struct wait_bit_queue_entry *wbq_entry =
164                 container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
165 
166         if (wbq_entry->key.flags != key->flags ||
167             wbq_entry->key.bit_nr != key->bit_nr)
168                 return 0;
169 
170         return autoremove_wake_function(wq_entry, mode, sync, key);
171 }
172 
173 void init_wait_var_entry(struct wait_bit_queue_entry *wbq_entry, void *var, int flags)
174 {
175         *wbq_entry = (struct wait_bit_queue_entry){
176                 .key = {
177                         .flags  = (var),
178                         .bit_nr = -1,
179                 },
180                 .wq_entry = {
181                         .private = current,
182                         .func    = var_wake_function,
183                         .entry   = LIST_HEAD_INIT(wbq_entry->wq_entry.entry),
184                 },
185         };
186 }
187 EXPORT_SYMBOL(init_wait_var_entry);
188 
189 void wake_up_var(void *var)
190 {
191         __wake_up_bit(__var_waitqueue(var), var, -1);
192 }
193 EXPORT_SYMBOL(wake_up_var);
194 
195 __sched int bit_wait(struct wait_bit_key *word, int mode)
196 {
197         schedule();
198         if (signal_pending_state(mode, current))
199                 return -EINTR;
200 
201         return 0;
202 }
203 EXPORT_SYMBOL(bit_wait);
204 
205 __sched int bit_wait_io(struct wait_bit_key *word, int mode)
206 {
207         io_schedule();
208         if (signal_pending_state(mode, current))
209                 return -EINTR;
210 
211         return 0;
212 }
213 EXPORT_SYMBOL(bit_wait_io);
214 
215 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
216 {
217         unsigned long now = READ_ONCE(jiffies);
218 
219         if (time_after_eq(now, word->timeout))
220                 return -EAGAIN;
221         schedule_timeout(word->timeout - now);
222         if (signal_pending_state(mode, current))
223                 return -EINTR;
224 
225         return 0;
226 }
227 EXPORT_SYMBOL_GPL(bit_wait_timeout);
228 
229 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
230 {
231         unsigned long now = READ_ONCE(jiffies);
232 
233         if (time_after_eq(now, word->timeout))
234                 return -EAGAIN;
235         io_schedule_timeout(word->timeout - now);
236         if (signal_pending_state(mode, current))
237                 return -EINTR;
238 
239         return 0;
240 }
241 EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
242 
243 void __init wait_bit_init(void)
244 {
245         int i;
246 
247         for (i = 0; i < WAIT_TABLE_SIZE; i++)
248                 init_waitqueue_head(bit_wait_table + i);
249 }
250 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp