~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/sched/wait_bit.c

Version: ~ [ linux-4.15-rc8 ] ~ [ linux-4.14.13 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.76 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.111 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.48 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.91 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.53 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.98 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * The implementation of the wait_bit*() and related waiting APIs:
  3  */
  4 #include <linux/wait_bit.h>
  5 #include <linux/sched/signal.h>
  6 #include <linux/sched/debug.h>
  7 #include <linux/hash.h>
  8 
  9 #define WAIT_TABLE_BITS 8
 10 #define WAIT_TABLE_SIZE (1 << WAIT_TABLE_BITS)
 11 
 12 static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
 13 
 14 wait_queue_head_t *bit_waitqueue(void *word, int bit)
 15 {
 16         const int shift = BITS_PER_LONG == 32 ? 5 : 6;
 17         unsigned long val = (unsigned long)word << shift | bit;
 18 
 19         return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
 20 }
 21 EXPORT_SYMBOL(bit_waitqueue);
 22 
 23 int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *arg)
 24 {
 25         struct wait_bit_key *key = arg;
 26         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
 27 
 28         if (wait_bit->key.flags != key->flags ||
 29                         wait_bit->key.bit_nr != key->bit_nr ||
 30                         test_bit(key->bit_nr, key->flags))
 31                 return 0;
 32         else
 33                 return autoremove_wake_function(wq_entry, mode, sync, key);
 34 }
 35 EXPORT_SYMBOL(wake_bit_function);
 36 
 37 /*
 38  * To allow interruptible waiting and asynchronous (i.e. nonblocking)
 39  * waiting, the actions of __wait_on_bit() and __wait_on_bit_lock() are
 40  * permitted return codes. Nonzero return codes halt waiting and return.
 41  */
 42 int __sched
 43 __wait_on_bit(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 44               wait_bit_action_f *action, unsigned mode)
 45 {
 46         int ret = 0;
 47 
 48         do {
 49                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
 50                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags))
 51                         ret = (*action)(&wbq_entry->key, mode);
 52         } while (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags) && !ret);
 53         finish_wait(wq_head, &wbq_entry->wq_entry);
 54         return ret;
 55 }
 56 EXPORT_SYMBOL(__wait_on_bit);
 57 
 58 int __sched out_of_line_wait_on_bit(void *word, int bit,
 59                                     wait_bit_action_f *action, unsigned mode)
 60 {
 61         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 62         DEFINE_WAIT_BIT(wq_entry, word, bit);
 63 
 64         return __wait_on_bit(wq_head, &wq_entry, action, mode);
 65 }
 66 EXPORT_SYMBOL(out_of_line_wait_on_bit);
 67 
 68 int __sched out_of_line_wait_on_bit_timeout(
 69         void *word, int bit, wait_bit_action_f *action,
 70         unsigned mode, unsigned long timeout)
 71 {
 72         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
 73         DEFINE_WAIT_BIT(wq_entry, word, bit);
 74 
 75         wq_entry.key.timeout = jiffies + timeout;
 76         return __wait_on_bit(wq_head, &wq_entry, action, mode);
 77 }
 78 EXPORT_SYMBOL_GPL(out_of_line_wait_on_bit_timeout);
 79 
 80 int __sched
 81 __wait_on_bit_lock(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
 82                         wait_bit_action_f *action, unsigned mode)
 83 {
 84         int ret = 0;
 85 
 86         for (;;) {
 87                 prepare_to_wait_exclusive(wq_head, &wbq_entry->wq_entry, mode);
 88                 if (test_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
 89                         ret = action(&wbq_entry->key, mode);
 90                         /*
 91                          * See the comment in prepare_to_wait_event().
 92                          * finish_wait() does not necessarily takes wwq_head->lock,
 93                          * but test_and_set_bit() implies mb() which pairs with
 94                          * smp_mb__after_atomic() before wake_up_page().
 95                          */
 96                         if (ret)
 97                                 finish_wait(wq_head, &wbq_entry->wq_entry);
 98                 }
 99                 if (!test_and_set_bit(wbq_entry->key.bit_nr, wbq_entry->key.flags)) {
100                         if (!ret)
101                                 finish_wait(wq_head, &wbq_entry->wq_entry);
102                         return 0;
103                 } else if (ret) {
104                         return ret;
105                 }
106         }
107 }
108 EXPORT_SYMBOL(__wait_on_bit_lock);
109 
110 int __sched out_of_line_wait_on_bit_lock(void *word, int bit,
111                                          wait_bit_action_f *action, unsigned mode)
112 {
113         struct wait_queue_head *wq_head = bit_waitqueue(word, bit);
114         DEFINE_WAIT_BIT(wq_entry, word, bit);
115 
116         return __wait_on_bit_lock(wq_head, &wq_entry, action, mode);
117 }
118 EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
119 
120 void __wake_up_bit(struct wait_queue_head *wq_head, void *word, int bit)
121 {
122         struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
123         if (waitqueue_active(wq_head))
124                 __wake_up(wq_head, TASK_NORMAL, 1, &key);
125 }
126 EXPORT_SYMBOL(__wake_up_bit);
127 
128 /**
129  * wake_up_bit - wake up a waiter on a bit
130  * @word: the word being waited on, a kernel virtual address
131  * @bit: the bit of the word being waited on
132  *
133  * There is a standard hashed waitqueue table for generic use. This
134  * is the part of the hashtable's accessor API that wakes up waiters
135  * on a bit. For instance, if one were to have waiters on a bitflag,
136  * one would call wake_up_bit() after clearing the bit.
137  *
138  * In order for this to function properly, as it uses waitqueue_active()
139  * internally, some kind of memory barrier must be done prior to calling
140  * this. Typically, this will be smp_mb__after_atomic(), but in some
141  * cases where bitflags are manipulated non-atomically under a lock, one
142  * may need to use a less regular barrier, such fs/inode.c's smp_mb(),
143  * because spin_unlock() does not guarantee a memory barrier.
144  */
145 void wake_up_bit(void *word, int bit)
146 {
147         __wake_up_bit(bit_waitqueue(word, bit), word, bit);
148 }
149 EXPORT_SYMBOL(wake_up_bit);
150 
151 /*
152  * Manipulate the atomic_t address to produce a better bit waitqueue table hash
153  * index (we're keying off bit -1, but that would produce a horrible hash
154  * value).
155  */
156 static inline wait_queue_head_t *atomic_t_waitqueue(atomic_t *p)
157 {
158         if (BITS_PER_LONG == 64) {
159                 unsigned long q = (unsigned long)p;
160                 return bit_waitqueue((void *)(q & ~1), q & 1);
161         }
162         return bit_waitqueue(p, 0);
163 }
164 
165 static int wake_atomic_t_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync,
166                                   void *arg)
167 {
168         struct wait_bit_key *key = arg;
169         struct wait_bit_queue_entry *wait_bit = container_of(wq_entry, struct wait_bit_queue_entry, wq_entry);
170         atomic_t *val = key->flags;
171 
172         if (wait_bit->key.flags != key->flags ||
173             wait_bit->key.bit_nr != key->bit_nr ||
174             atomic_read(val) != 0)
175                 return 0;
176         return autoremove_wake_function(wq_entry, mode, sync, key);
177 }
178 
179 /*
180  * To allow interruptible waiting and asynchronous (i.e. nonblocking) waiting,
181  * the actions of __wait_on_atomic_t() are permitted return codes.  Nonzero
182  * return codes halt waiting and return.
183  */
184 static __sched
185 int __wait_on_atomic_t(struct wait_queue_head *wq_head, struct wait_bit_queue_entry *wbq_entry,
186                        wait_atomic_t_action_f action, unsigned int mode)
187 {
188         atomic_t *val;
189         int ret = 0;
190 
191         do {
192                 prepare_to_wait(wq_head, &wbq_entry->wq_entry, mode);
193                 val = wbq_entry->key.flags;
194                 if (atomic_read(val) == 0)
195                         break;
196                 ret = (*action)(val, mode);
197         } while (!ret && atomic_read(val) != 0);
198         finish_wait(wq_head, &wbq_entry->wq_entry);
199         return ret;
200 }
201 
202 #define DEFINE_WAIT_ATOMIC_T(name, p)                                   \
203         struct wait_bit_queue_entry name = {                            \
204                 .key = __WAIT_ATOMIC_T_KEY_INITIALIZER(p),              \
205                 .wq_entry = {                                           \
206                         .private        = current,                      \
207                         .func           = wake_atomic_t_function,       \
208                         .entry          =                               \
209                                 LIST_HEAD_INIT((name).wq_entry.entry),  \
210                 },                                                      \
211         }
212 
213 __sched int out_of_line_wait_on_atomic_t(atomic_t *p,
214                                          wait_atomic_t_action_f action,
215                                          unsigned int mode)
216 {
217         struct wait_queue_head *wq_head = atomic_t_waitqueue(p);
218         DEFINE_WAIT_ATOMIC_T(wq_entry, p);
219 
220         return __wait_on_atomic_t(wq_head, &wq_entry, action, mode);
221 }
222 EXPORT_SYMBOL(out_of_line_wait_on_atomic_t);
223 
224 __sched int atomic_t_wait(atomic_t *counter, unsigned int mode)
225 {
226         schedule();
227         if (signal_pending_state(mode, current))
228                 return -EINTR;
229         return 0;
230 }
231 EXPORT_SYMBOL(atomic_t_wait);
232 
233 /**
234  * wake_up_atomic_t - Wake up a waiter on a atomic_t
235  * @p: The atomic_t being waited on, a kernel virtual address
236  *
237  * Wake up anyone waiting for the atomic_t to go to zero.
238  *
239  * Abuse the bit-waker function and its waitqueue hash table set (the atomic_t
240  * check is done by the waiter's wake function, not the by the waker itself).
241  */
242 void wake_up_atomic_t(atomic_t *p)
243 {
244         __wake_up_bit(atomic_t_waitqueue(p), p, WAIT_ATOMIC_T_BIT_NR);
245 }
246 EXPORT_SYMBOL(wake_up_atomic_t);
247 
248 __sched int bit_wait(struct wait_bit_key *word, int mode)
249 {
250         schedule();
251         if (signal_pending_state(mode, current))
252                 return -EINTR;
253         return 0;
254 }
255 EXPORT_SYMBOL(bit_wait);
256 
257 __sched int bit_wait_io(struct wait_bit_key *word, int mode)
258 {
259         io_schedule();
260         if (signal_pending_state(mode, current))
261                 return -EINTR;
262         return 0;
263 }
264 EXPORT_SYMBOL(bit_wait_io);
265 
266 __sched int bit_wait_timeout(struct wait_bit_key *word, int mode)
267 {
268         unsigned long now = READ_ONCE(jiffies);
269         if (time_after_eq(now, word->timeout))
270                 return -EAGAIN;
271         schedule_timeout(word->timeout - now);
272         if (signal_pending_state(mode, current))
273                 return -EINTR;
274         return 0;
275 }
276 EXPORT_SYMBOL_GPL(bit_wait_timeout);
277 
278 __sched int bit_wait_io_timeout(struct wait_bit_key *word, int mode)
279 {
280         unsigned long now = READ_ONCE(jiffies);
281         if (time_after_eq(now, word->timeout))
282                 return -EAGAIN;
283         io_schedule_timeout(word->timeout - now);
284         if (signal_pending_state(mode, current))
285                 return -EINTR;
286         return 0;
287 }
288 EXPORT_SYMBOL_GPL(bit_wait_io_timeout);
289 
290 void __init wait_bit_init(void)
291 {
292         int i;
293 
294         for (i = 0; i < WAIT_TABLE_SIZE; i++)
295                 init_waitqueue_head(bit_wait_table + i);
296 }
297 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp