~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/workqueue.c

Version: ~ [ linux-5.1-rc5 ] ~ [ linux-5.0.7 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.34 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.111 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.168 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.178 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.138 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.65 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * kernel/workqueue.c - generic async execution with shared worker pool
  3  *
  4  * Copyright (C) 2002           Ingo Molnar
  5  *
  6  *   Derived from the taskqueue/keventd code by:
  7  *     David Woodhouse <dwmw2@infradead.org>
  8  *     Andrew Morton
  9  *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
 10  *     Theodore Ts'o <tytso@mit.edu>
 11  *
 12  * Made to use alloc_percpu by Christoph Lameter.
 13  *
 14  * Copyright (C) 2010           SUSE Linux Products GmbH
 15  * Copyright (C) 2010           Tejun Heo <tj@kernel.org>
 16  *
 17  * This is the generic async execution mechanism.  Work items as are
 18  * executed in process context.  The worker pool is shared and
 19  * automatically managed.  There is one worker pool for each CPU and
 20  * one extra for works which are better served by workers which are
 21  * not bound to any specific CPU.
 22  *
 23  * Please read Documentation/workqueue.txt for details.
 24  */
 25 
 26 #include <linux/export.h>
 27 #include <linux/kernel.h>
 28 #include <linux/sched.h>
 29 #include <linux/init.h>
 30 #include <linux/signal.h>
 31 #include <linux/completion.h>
 32 #include <linux/workqueue.h>
 33 #include <linux/slab.h>
 34 #include <linux/cpu.h>
 35 #include <linux/notifier.h>
 36 #include <linux/kthread.h>
 37 #include <linux/hardirq.h>
 38 #include <linux/mempolicy.h>
 39 #include <linux/freezer.h>
 40 #include <linux/kallsyms.h>
 41 #include <linux/debug_locks.h>
 42 #include <linux/lockdep.h>
 43 #include <linux/idr.h>
 44 #include <linux/jhash.h>
 45 #include <linux/hashtable.h>
 46 #include <linux/rculist.h>
 47 #include <linux/nodemask.h>
 48 #include <linux/moduleparam.h>
 49 #include <linux/uaccess.h>
 50 
 51 #include "workqueue_internal.h"
 52 
 53 enum {
 54         /*
 55          * worker_pool flags
 56          *
 57          * A bound pool is either associated or disassociated with its CPU.
 58          * While associated (!DISASSOCIATED), all workers are bound to the
 59          * CPU and none has %WORKER_UNBOUND set and concurrency management
 60          * is in effect.
 61          *
 62          * While DISASSOCIATED, the cpu may be offline and all workers have
 63          * %WORKER_UNBOUND set and concurrency management disabled, and may
 64          * be executing on any CPU.  The pool behaves as an unbound one.
 65          *
 66          * Note that DISASSOCIATED should be flipped only while holding
 67          * manager_mutex to avoid changing binding state while
 68          * create_worker() is in progress.
 69          */
 70         POOL_MANAGE_WORKERS     = 1 << 0,       /* need to manage workers */
 71         POOL_DISASSOCIATED      = 1 << 2,       /* cpu can't serve workers */
 72         POOL_FREEZING           = 1 << 3,       /* freeze in progress */
 73 
 74         /* worker flags */
 75         WORKER_STARTED          = 1 << 0,       /* started */
 76         WORKER_DIE              = 1 << 1,       /* die die die */
 77         WORKER_IDLE             = 1 << 2,       /* is idle */
 78         WORKER_PREP             = 1 << 3,       /* preparing to run works */
 79         WORKER_CPU_INTENSIVE    = 1 << 6,       /* cpu intensive */
 80         WORKER_UNBOUND          = 1 << 7,       /* worker is unbound */
 81         WORKER_REBOUND          = 1 << 8,       /* worker was rebound */
 82 
 83         WORKER_NOT_RUNNING      = WORKER_PREP | WORKER_CPU_INTENSIVE |
 84                                   WORKER_UNBOUND | WORKER_REBOUND,
 85 
 86         NR_STD_WORKER_POOLS     = 2,            /* # standard pools per cpu */
 87 
 88         UNBOUND_POOL_HASH_ORDER = 6,            /* hashed by pool->attrs */
 89         BUSY_WORKER_HASH_ORDER  = 6,            /* 64 pointers */
 90 
 91         MAX_IDLE_WORKERS_RATIO  = 4,            /* 1/4 of busy can be idle */
 92         IDLE_WORKER_TIMEOUT     = 300 * HZ,     /* keep idle ones for 5 mins */
 93 
 94         MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
 95                                                 /* call for help after 10ms
 96                                                    (min two ticks) */
 97         MAYDAY_INTERVAL         = HZ / 10,      /* and then every 100ms */
 98         CREATE_COOLDOWN         = HZ,           /* time to breath after fail */
 99 
100         /*
101          * Rescue workers are used only on emergencies and shared by
102          * all cpus.  Give -20.
103          */
104         RESCUER_NICE_LEVEL      = -20,
105         HIGHPRI_NICE_LEVEL      = -20,
106 
107         WQ_NAME_LEN             = 24,
108 };
109 
110 /*
111  * Structure fields follow one of the following exclusion rules.
112  *
113  * I: Modifiable by initialization/destruction paths and read-only for
114  *    everyone else.
115  *
116  * P: Preemption protected.  Disabling preemption is enough and should
117  *    only be modified and accessed from the local cpu.
118  *
119  * L: pool->lock protected.  Access with pool->lock held.
120  *
121  * X: During normal operation, modification requires pool->lock and should
122  *    be done only from local cpu.  Either disabling preemption on local
123  *    cpu or grabbing pool->lock is enough for read access.  If
124  *    POOL_DISASSOCIATED is set, it's identical to L.
125  *
126  * MG: pool->manager_mutex and pool->lock protected.  Writes require both
127  *     locks.  Reads can happen under either lock.
128  *
129  * PL: wq_pool_mutex protected.
130  *
131  * PR: wq_pool_mutex protected for writes.  Sched-RCU protected for reads.
132  *
133  * WQ: wq->mutex protected.
134  *
135  * WR: wq->mutex protected for writes.  Sched-RCU protected for reads.
136  *
137  * MD: wq_mayday_lock protected.
138  */
139 
140 /* struct worker is defined in workqueue_internal.h */
141 
142 struct worker_pool {
143         spinlock_t              lock;           /* the pool lock */
144         int                     cpu;            /* I: the associated cpu */
145         int                     node;           /* I: the associated node ID */
146         int                     id;             /* I: pool ID */
147         unsigned int            flags;          /* X: flags */
148 
149         struct list_head        worklist;       /* L: list of pending works */
150         int                     nr_workers;     /* L: total number of workers */
151 
152         /* nr_idle includes the ones off idle_list for rebinding */
153         int                     nr_idle;        /* L: currently idle ones */
154 
155         struct list_head        idle_list;      /* X: list of idle workers */
156         struct timer_list       idle_timer;     /* L: worker idle timeout */
157         struct timer_list       mayday_timer;   /* L: SOS timer for workers */
158 
159         /* a workers is either on busy_hash or idle_list, or the manager */
160         DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
161                                                 /* L: hash of busy workers */
162 
163         /* see manage_workers() for details on the two manager mutexes */
164         struct mutex            manager_arb;    /* manager arbitration */
165         struct mutex            manager_mutex;  /* manager exclusion */
166         struct idr              worker_idr;     /* MG: worker IDs and iteration */
167 
168         struct workqueue_attrs  *attrs;         /* I: worker attributes */
169         struct hlist_node       hash_node;      /* PL: unbound_pool_hash node */
170         int                     refcnt;         /* PL: refcnt for unbound pools */
171 
172         /*
173          * The current concurrency level.  As it's likely to be accessed
174          * from other CPUs during try_to_wake_up(), put it in a separate
175          * cacheline.
176          */
177         atomic_t                nr_running ____cacheline_aligned_in_smp;
178 
179         /*
180          * Destruction of pool is sched-RCU protected to allow dereferences
181          * from get_work_pool().
182          */
183         struct rcu_head         rcu;
184 } ____cacheline_aligned_in_smp;
185 
186 /*
187  * The per-pool workqueue.  While queued, the lower WORK_STRUCT_FLAG_BITS
188  * of work_struct->data are used for flags and the remaining high bits
189  * point to the pwq; thus, pwqs need to be aligned at two's power of the
190  * number of flag bits.
191  */
192 struct pool_workqueue {
193         struct worker_pool      *pool;          /* I: the associated pool */
194         struct workqueue_struct *wq;            /* I: the owning workqueue */
195         int                     work_color;     /* L: current color */
196         int                     flush_color;    /* L: flushing color */
197         int                     refcnt;         /* L: reference count */
198         int                     nr_in_flight[WORK_NR_COLORS];
199                                                 /* L: nr of in_flight works */
200         int                     nr_active;      /* L: nr of active works */
201         int                     max_active;     /* L: max active works */
202         struct list_head        delayed_works;  /* L: delayed works */
203         struct list_head        pwqs_node;      /* WR: node on wq->pwqs */
204         struct list_head        mayday_node;    /* MD: node on wq->maydays */
205 
206         /*
207          * Release of unbound pwq is punted to system_wq.  See put_pwq()
208          * and pwq_unbound_release_workfn() for details.  pool_workqueue
209          * itself is also sched-RCU protected so that the first pwq can be
210          * determined without grabbing wq->mutex.
211          */
212         struct work_struct      unbound_release_work;
213         struct rcu_head         rcu;
214 } __aligned(1 << WORK_STRUCT_FLAG_BITS);
215 
216 /*
217  * Structure used to wait for workqueue flush.
218  */
219 struct wq_flusher {
220         struct list_head        list;           /* WQ: list of flushers */
221         int                     flush_color;    /* WQ: flush color waiting for */
222         struct completion       done;           /* flush completion */
223 };
224 
225 struct wq_device;
226 
227 /*
228  * The externally visible workqueue.  It relays the issued work items to
229  * the appropriate worker_pool through its pool_workqueues.
230  */
231 struct workqueue_struct {
232         struct list_head        pwqs;           /* WR: all pwqs of this wq */
233         struct list_head        list;           /* PL: list of all workqueues */
234 
235         struct mutex            mutex;          /* protects this wq */
236         int                     work_color;     /* WQ: current work color */
237         int                     flush_color;    /* WQ: current flush color */
238         atomic_t                nr_pwqs_to_flush; /* flush in progress */
239         struct wq_flusher       *first_flusher; /* WQ: first flusher */
240         struct list_head        flusher_queue;  /* WQ: flush waiters */
241         struct list_head        flusher_overflow; /* WQ: flush overflow list */
242 
243         struct list_head        maydays;        /* MD: pwqs requesting rescue */
244         struct worker           *rescuer;       /* I: rescue worker */
245 
246         int                     nr_drainers;    /* WQ: drain in progress */
247         int                     saved_max_active; /* WQ: saved pwq max_active */
248 
249         struct workqueue_attrs  *unbound_attrs; /* WQ: only for unbound wqs */
250         struct pool_workqueue   *dfl_pwq;       /* WQ: only for unbound wqs */
251 
252 #ifdef CONFIG_SYSFS
253         struct wq_device        *wq_dev;        /* I: for sysfs interface */
254 #endif
255 #ifdef CONFIG_LOCKDEP
256         struct lockdep_map      lockdep_map;
257 #endif
258         char                    name[WQ_NAME_LEN]; /* I: workqueue name */
259 
260         /* hot fields used during command issue, aligned to cacheline */
261         unsigned int            flags ____cacheline_aligned; /* WQ: WQ_* flags */
262         struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
263         struct pool_workqueue __rcu *numa_pwq_tbl[]; /* FR: unbound pwqs indexed by node */
264 };
265 
266 static struct kmem_cache *pwq_cache;
267 
268 static int wq_numa_tbl_len;             /* highest possible NUMA node id + 1 */
269 static cpumask_var_t *wq_numa_possible_cpumask;
270                                         /* possible CPUs of each node */
271 
272 static bool wq_disable_numa;
273 module_param_named(disable_numa, wq_disable_numa, bool, 0444);
274 
275 static bool wq_numa_enabled;            /* unbound NUMA affinity enabled */
276 
277 /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
278 static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
279 
280 static DEFINE_MUTEX(wq_pool_mutex);     /* protects pools and workqueues list */
281 static DEFINE_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
282 
283 static LIST_HEAD(workqueues);           /* PL: list of all workqueues */
284 static bool workqueue_freezing;         /* PL: have wqs started freezing? */
285 
286 /* the per-cpu worker pools */
287 static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
288                                      cpu_worker_pools);
289 
290 static DEFINE_IDR(worker_pool_idr);     /* PR: idr of all pools */
291 
292 /* PL: hash of all unbound pools keyed by pool->attrs */
293 static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
294 
295 /* I: attributes used when instantiating standard unbound pools on demand */
296 static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
297 
298 /* I: attributes used when instantiating ordered pools on demand */
299 static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
300 
301 struct workqueue_struct *system_wq __read_mostly;
302 EXPORT_SYMBOL(system_wq);
303 struct workqueue_struct *system_highpri_wq __read_mostly;
304 EXPORT_SYMBOL_GPL(system_highpri_wq);
305 struct workqueue_struct *system_long_wq __read_mostly;
306 EXPORT_SYMBOL_GPL(system_long_wq);
307 struct workqueue_struct *system_unbound_wq __read_mostly;
308 EXPORT_SYMBOL_GPL(system_unbound_wq);
309 struct workqueue_struct *system_freezable_wq __read_mostly;
310 EXPORT_SYMBOL_GPL(system_freezable_wq);
311 
312 static int worker_thread(void *__worker);
313 static void copy_workqueue_attrs(struct workqueue_attrs *to,
314                                  const struct workqueue_attrs *from);
315 
316 #define CREATE_TRACE_POINTS
317 #include <trace/events/workqueue.h>
318 
319 #define assert_rcu_or_pool_mutex()                                      \
320         rcu_lockdep_assert(rcu_read_lock_sched_held() ||                \
321                            lockdep_is_held(&wq_pool_mutex),             \
322                            "sched RCU or wq_pool_mutex should be held")
323 
324 #define assert_rcu_or_wq_mutex(wq)                                      \
325         rcu_lockdep_assert(rcu_read_lock_sched_held() ||                \
326                            lockdep_is_held(&wq->mutex),                 \
327                            "sched RCU or wq->mutex should be held")
328 
329 #ifdef CONFIG_LOCKDEP
330 #define assert_manager_or_pool_lock(pool)                               \
331         WARN_ONCE(debug_locks &&                                        \
332                   !lockdep_is_held(&(pool)->manager_mutex) &&           \
333                   !lockdep_is_held(&(pool)->lock),                      \
334                   "pool->manager_mutex or ->lock should be held")
335 #else
336 #define assert_manager_or_pool_lock(pool)       do { } while (0)
337 #endif
338 
339 #define for_each_cpu_worker_pool(pool, cpu)                             \
340         for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];               \
341              (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
342              (pool)++)
343 
344 /**
345  * for_each_pool - iterate through all worker_pools in the system
346  * @pool: iteration cursor
347  * @pi: integer used for iteration
348  *
349  * This must be called either with wq_pool_mutex held or sched RCU read
350  * locked.  If the pool needs to be used beyond the locking in effect, the
351  * caller is responsible for guaranteeing that the pool stays online.
352  *
353  * The if/else clause exists only for the lockdep assertion and can be
354  * ignored.
355  */
356 #define for_each_pool(pool, pi)                                         \
357         idr_for_each_entry(&worker_pool_idr, pool, pi)                  \
358                 if (({ assert_rcu_or_pool_mutex(); false; })) { }       \
359                 else
360 
361 /**
362  * for_each_pool_worker - iterate through all workers of a worker_pool
363  * @worker: iteration cursor
364  * @wi: integer used for iteration
365  * @pool: worker_pool to iterate workers of
366  *
367  * This must be called with either @pool->manager_mutex or ->lock held.
368  *
369  * The if/else clause exists only for the lockdep assertion and can be
370  * ignored.
371  */
372 #define for_each_pool_worker(worker, wi, pool)                          \
373         idr_for_each_entry(&(pool)->worker_idr, (worker), (wi))         \
374                 if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
375                 else
376 
377 /**
378  * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
379  * @pwq: iteration cursor
380  * @wq: the target workqueue
381  *
382  * This must be called either with wq->mutex held or sched RCU read locked.
383  * If the pwq needs to be used beyond the locking in effect, the caller is
384  * responsible for guaranteeing that the pwq stays online.
385  *
386  * The if/else clause exists only for the lockdep assertion and can be
387  * ignored.
388  */
389 #define for_each_pwq(pwq, wq)                                           \
390         list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node)          \
391                 if (({ assert_rcu_or_wq_mutex(wq); false; })) { }       \
392                 else
393 
394 #ifdef CONFIG_DEBUG_OBJECTS_WORK
395 
396 static struct debug_obj_descr work_debug_descr;
397 
398 static void *work_debug_hint(void *addr)
399 {
400         return ((struct work_struct *) addr)->func;
401 }
402 
403 /*
404  * fixup_init is called when:
405  * - an active object is initialized
406  */
407 static int work_fixup_init(void *addr, enum debug_obj_state state)
408 {
409         struct work_struct *work = addr;
410 
411         switch (state) {
412         case ODEBUG_STATE_ACTIVE:
413                 cancel_work_sync(work);
414                 debug_object_init(work, &work_debug_descr);
415                 return 1;
416         default:
417                 return 0;
418         }
419 }
420 
421 /*
422  * fixup_activate is called when:
423  * - an active object is activated
424  * - an unknown object is activated (might be a statically initialized object)
425  */
426 static int work_fixup_activate(void *addr, enum debug_obj_state state)
427 {
428         struct work_struct *work = addr;
429 
430         switch (state) {
431 
432         case ODEBUG_STATE_NOTAVAILABLE:
433                 /*
434                  * This is not really a fixup. The work struct was
435                  * statically initialized. We just make sure that it
436                  * is tracked in the object tracker.
437                  */
438                 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
439                         debug_object_init(work, &work_debug_descr);
440                         debug_object_activate(work, &work_debug_descr);
441                         return 0;
442                 }
443                 WARN_ON_ONCE(1);
444                 return 0;
445 
446         case ODEBUG_STATE_ACTIVE:
447                 WARN_ON(1);
448 
449         default:
450                 return 0;
451         }
452 }
453 
454 /*
455  * fixup_free is called when:
456  * - an active object is freed
457  */
458 static int work_fixup_free(void *addr, enum debug_obj_state state)
459 {
460         struct work_struct *work = addr;
461 
462         switch (state) {
463         case ODEBUG_STATE_ACTIVE:
464                 cancel_work_sync(work);
465                 debug_object_free(work, &work_debug_descr);
466                 return 1;
467         default:
468                 return 0;
469         }
470 }
471 
472 static struct debug_obj_descr work_debug_descr = {
473         .name           = "work_struct",
474         .debug_hint     = work_debug_hint,
475         .fixup_init     = work_fixup_init,
476         .fixup_activate = work_fixup_activate,
477         .fixup_free     = work_fixup_free,
478 };
479 
480 static inline void debug_work_activate(struct work_struct *work)
481 {
482         debug_object_activate(work, &work_debug_descr);
483 }
484 
485 static inline void debug_work_deactivate(struct work_struct *work)
486 {
487         debug_object_deactivate(work, &work_debug_descr);
488 }
489 
490 void __init_work(struct work_struct *work, int onstack)
491 {
492         if (onstack)
493                 debug_object_init_on_stack(work, &work_debug_descr);
494         else
495                 debug_object_init(work, &work_debug_descr);
496 }
497 EXPORT_SYMBOL_GPL(__init_work);
498 
499 void destroy_work_on_stack(struct work_struct *work)
500 {
501         debug_object_free(work, &work_debug_descr);
502 }
503 EXPORT_SYMBOL_GPL(destroy_work_on_stack);
504 
505 #else
506 static inline void debug_work_activate(struct work_struct *work) { }
507 static inline void debug_work_deactivate(struct work_struct *work) { }
508 #endif
509 
510 /* allocate ID and assign it to @pool */
511 static int worker_pool_assign_id(struct worker_pool *pool)
512 {
513         int ret;
514 
515         lockdep_assert_held(&wq_pool_mutex);
516 
517         ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
518         if (ret >= 0) {
519                 pool->id = ret;
520                 return 0;
521         }
522         return ret;
523 }
524 
525 /**
526  * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
527  * @wq: the target workqueue
528  * @node: the node ID
529  *
530  * This must be called either with pwq_lock held or sched RCU read locked.
531  * If the pwq needs to be used beyond the locking in effect, the caller is
532  * responsible for guaranteeing that the pwq stays online.
533  */
534 static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
535                                                   int node)
536 {
537         assert_rcu_or_wq_mutex(wq);
538         return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
539 }
540 
541 static unsigned int work_color_to_flags(int color)
542 {
543         return color << WORK_STRUCT_COLOR_SHIFT;
544 }
545 
546 static int get_work_color(struct work_struct *work)
547 {
548         return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
549                 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
550 }
551 
552 static int work_next_color(int color)
553 {
554         return (color + 1) % WORK_NR_COLORS;
555 }
556 
557 /*
558  * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
559  * contain the pointer to the queued pwq.  Once execution starts, the flag
560  * is cleared and the high bits contain OFFQ flags and pool ID.
561  *
562  * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
563  * and clear_work_data() can be used to set the pwq, pool or clear
564  * work->data.  These functions should only be called while the work is
565  * owned - ie. while the PENDING bit is set.
566  *
567  * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
568  * corresponding to a work.  Pool is available once the work has been
569  * queued anywhere after initialization until it is sync canceled.  pwq is
570  * available only while the work item is queued.
571  *
572  * %WORK_OFFQ_CANCELING is used to mark a work item which is being
573  * canceled.  While being canceled, a work item may have its PENDING set
574  * but stay off timer and worklist for arbitrarily long and nobody should
575  * try to steal the PENDING bit.
576  */
577 static inline void set_work_data(struct work_struct *work, unsigned long data,
578                                  unsigned long flags)
579 {
580         WARN_ON_ONCE(!work_pending(work));
581         atomic_long_set(&work->data, data | flags | work_static(work));
582 }
583 
584 static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
585                          unsigned long extra_flags)
586 {
587         set_work_data(work, (unsigned long)pwq,
588                       WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
589 }
590 
591 static void set_work_pool_and_keep_pending(struct work_struct *work,
592                                            int pool_id)
593 {
594         set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
595                       WORK_STRUCT_PENDING);
596 }
597 
598 static void set_work_pool_and_clear_pending(struct work_struct *work,
599                                             int pool_id)
600 {
601         /*
602          * The following wmb is paired with the implied mb in
603          * test_and_set_bit(PENDING) and ensures all updates to @work made
604          * here are visible to and precede any updates by the next PENDING
605          * owner.
606          */
607         smp_wmb();
608         set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
609         /*
610          * The following mb guarantees that previous clear of a PENDING bit
611          * will not be reordered with any speculative LOADS or STORES from
612          * work->current_func, which is executed afterwards.  This possible
613          * reordering can lead to a missed execution on attempt to qeueue
614          * the same @work.  E.g. consider this case:
615          *
616          *   CPU#0                         CPU#1
617          *   ----------------------------  --------------------------------
618          *
619          * 1  STORE event_indicated
620          * 2  queue_work_on() {
621          * 3    test_and_set_bit(PENDING)
622          * 4 }                             set_..._and_clear_pending() {
623          * 5                                 set_work_data() # clear bit
624          * 6                                 smp_mb()
625          * 7                               work->current_func() {
626          * 8                                  LOAD event_indicated
627          *                                 }
628          *
629          * Without an explicit full barrier speculative LOAD on line 8 can
630          * be executed before CPU#0 does STORE on line 1.  If that happens,
631          * CPU#0 observes the PENDING bit is still set and new execution of
632          * a @work is not queued in a hope, that CPU#1 will eventually
633          * finish the queued @work.  Meanwhile CPU#1 does not see
634          * event_indicated is set, because speculative LOAD was executed
635          * before actual STORE.
636          */
637         smp_mb();
638 }
639 
640 static void clear_work_data(struct work_struct *work)
641 {
642         smp_wmb();      /* see set_work_pool_and_clear_pending() */
643         set_work_data(work, WORK_STRUCT_NO_POOL, 0);
644 }
645 
646 static struct pool_workqueue *get_work_pwq(struct work_struct *work)
647 {
648         unsigned long data = atomic_long_read(&work->data);
649 
650         if (data & WORK_STRUCT_PWQ)
651                 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
652         else
653                 return NULL;
654 }
655 
656 /**
657  * get_work_pool - return the worker_pool a given work was associated with
658  * @work: the work item of interest
659  *
660  * Return the worker_pool @work was last associated with.  %NULL if none.
661  *
662  * Pools are created and destroyed under wq_pool_mutex, and allows read
663  * access under sched-RCU read lock.  As such, this function should be
664  * called under wq_pool_mutex or with preemption disabled.
665  *
666  * All fields of the returned pool are accessible as long as the above
667  * mentioned locking is in effect.  If the returned pool needs to be used
668  * beyond the critical section, the caller is responsible for ensuring the
669  * returned pool is and stays online.
670  */
671 static struct worker_pool *get_work_pool(struct work_struct *work)
672 {
673         unsigned long data = atomic_long_read(&work->data);
674         int pool_id;
675 
676         assert_rcu_or_pool_mutex();
677 
678         if (data & WORK_STRUCT_PWQ)
679                 return ((struct pool_workqueue *)
680                         (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
681 
682         pool_id = data >> WORK_OFFQ_POOL_SHIFT;
683         if (pool_id == WORK_OFFQ_POOL_NONE)
684                 return NULL;
685 
686         return idr_find(&worker_pool_idr, pool_id);
687 }
688 
689 /**
690  * get_work_pool_id - return the worker pool ID a given work is associated with
691  * @work: the work item of interest
692  *
693  * Return the worker_pool ID @work was last associated with.
694  * %WORK_OFFQ_POOL_NONE if none.
695  */
696 static int get_work_pool_id(struct work_struct *work)
697 {
698         unsigned long data = atomic_long_read(&work->data);
699 
700         if (data & WORK_STRUCT_PWQ)
701                 return ((struct pool_workqueue *)
702                         (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
703 
704         return data >> WORK_OFFQ_POOL_SHIFT;
705 }
706 
707 static void mark_work_canceling(struct work_struct *work)
708 {
709         unsigned long pool_id = get_work_pool_id(work);
710 
711         pool_id <<= WORK_OFFQ_POOL_SHIFT;
712         set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
713 }
714 
715 static bool work_is_canceling(struct work_struct *work)
716 {
717         unsigned long data = atomic_long_read(&work->data);
718 
719         return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
720 }
721 
722 /*
723  * Policy functions.  These define the policies on how the global worker
724  * pools are managed.  Unless noted otherwise, these functions assume that
725  * they're being called with pool->lock held.
726  */
727 
728 static bool __need_more_worker(struct worker_pool *pool)
729 {
730         return !atomic_read(&pool->nr_running);
731 }
732 
733 /*
734  * Need to wake up a worker?  Called from anything but currently
735  * running workers.
736  *
737  * Note that, because unbound workers never contribute to nr_running, this
738  * function will always return %true for unbound pools as long as the
739  * worklist isn't empty.
740  */
741 static bool need_more_worker(struct worker_pool *pool)
742 {
743         return !list_empty(&pool->worklist) && __need_more_worker(pool);
744 }
745 
746 /* Can I start working?  Called from busy but !running workers. */
747 static bool may_start_working(struct worker_pool *pool)
748 {
749         return pool->nr_idle;
750 }
751 
752 /* Do I need to keep working?  Called from currently running workers. */
753 static bool keep_working(struct worker_pool *pool)
754 {
755         return !list_empty(&pool->worklist) &&
756                 atomic_read(&pool->nr_running) <= 1;
757 }
758 
759 /* Do we need a new worker?  Called from manager. */
760 static bool need_to_create_worker(struct worker_pool *pool)
761 {
762         return need_more_worker(pool) && !may_start_working(pool);
763 }
764 
765 /* Do I need to be the manager? */
766 static bool need_to_manage_workers(struct worker_pool *pool)
767 {
768         return need_to_create_worker(pool) ||
769                 (pool->flags & POOL_MANAGE_WORKERS);
770 }
771 
772 /* Do we have too many workers and should some go away? */
773 static bool too_many_workers(struct worker_pool *pool)
774 {
775         bool managing = mutex_is_locked(&pool->manager_arb);
776         int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
777         int nr_busy = pool->nr_workers - nr_idle;
778 
779         /*
780          * nr_idle and idle_list may disagree if idle rebinding is in
781          * progress.  Never return %true if idle_list is empty.
782          */
783         if (list_empty(&pool->idle_list))
784                 return false;
785 
786         return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
787 }
788 
789 /*
790  * Wake up functions.
791  */
792 
793 /* Return the first worker.  Safe with preemption disabled */
794 static struct worker *first_worker(struct worker_pool *pool)
795 {
796         if (unlikely(list_empty(&pool->idle_list)))
797                 return NULL;
798 
799         return list_first_entry(&pool->idle_list, struct worker, entry);
800 }
801 
802 /**
803  * wake_up_worker - wake up an idle worker
804  * @pool: worker pool to wake worker from
805  *
806  * Wake up the first idle worker of @pool.
807  *
808  * CONTEXT:
809  * spin_lock_irq(pool->lock).
810  */
811 static void wake_up_worker(struct worker_pool *pool)
812 {
813         struct worker *worker = first_worker(pool);
814 
815         if (likely(worker))
816                 wake_up_process(worker->task);
817 }
818 
819 /**
820  * wq_worker_waking_up - a worker is waking up
821  * @task: task waking up
822  * @cpu: CPU @task is waking up to
823  *
824  * This function is called during try_to_wake_up() when a worker is
825  * being awoken.
826  *
827  * CONTEXT:
828  * spin_lock_irq(rq->lock)
829  */
830 void wq_worker_waking_up(struct task_struct *task, int cpu)
831 {
832         struct worker *worker = kthread_data(task);
833 
834         if (!(worker->flags & WORKER_NOT_RUNNING)) {
835                 WARN_ON_ONCE(worker->pool->cpu != cpu);
836                 atomic_inc(&worker->pool->nr_running);
837         }
838 }
839 
840 /**
841  * wq_worker_sleeping - a worker is going to sleep
842  * @task: task going to sleep
843  * @cpu: CPU in question, must be the current CPU number
844  *
845  * This function is called during schedule() when a busy worker is
846  * going to sleep.  Worker on the same cpu can be woken up by
847  * returning pointer to its task.
848  *
849  * CONTEXT:
850  * spin_lock_irq(rq->lock)
851  *
852  * RETURNS:
853  * Worker task on @cpu to wake up, %NULL if none.
854  */
855 struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
856 {
857         struct worker *worker = kthread_data(task), *to_wakeup = NULL;
858         struct worker_pool *pool;
859 
860         /*
861          * Rescuers, which may not have all the fields set up like normal
862          * workers, also reach here, let's not access anything before
863          * checking NOT_RUNNING.
864          */
865         if (worker->flags & WORKER_NOT_RUNNING)
866                 return NULL;
867 
868         pool = worker->pool;
869 
870         /* this can only happen on the local cpu */
871         if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
872                 return NULL;
873 
874         /*
875          * The counterpart of the following dec_and_test, implied mb,
876          * worklist not empty test sequence is in insert_work().
877          * Please read comment there.
878          *
879          * NOT_RUNNING is clear.  This means that we're bound to and
880          * running on the local cpu w/ rq lock held and preemption
881          * disabled, which in turn means that none else could be
882          * manipulating idle_list, so dereferencing idle_list without pool
883          * lock is safe.
884          */
885         if (atomic_dec_and_test(&pool->nr_running) &&
886             !list_empty(&pool->worklist))
887                 to_wakeup = first_worker(pool);
888         return to_wakeup ? to_wakeup->task : NULL;
889 }
890 
891 /**
892  * worker_set_flags - set worker flags and adjust nr_running accordingly
893  * @worker: self
894  * @flags: flags to set
895  * @wakeup: wakeup an idle worker if necessary
896  *
897  * Set @flags in @worker->flags and adjust nr_running accordingly.  If
898  * nr_running becomes zero and @wakeup is %true, an idle worker is
899  * woken up.
900  *
901  * CONTEXT:
902  * spin_lock_irq(pool->lock)
903  */
904 static inline void worker_set_flags(struct worker *worker, unsigned int flags,
905                                     bool wakeup)
906 {
907         struct worker_pool *pool = worker->pool;
908 
909         WARN_ON_ONCE(worker->task != current);
910 
911         /*
912          * If transitioning into NOT_RUNNING, adjust nr_running and
913          * wake up an idle worker as necessary if requested by
914          * @wakeup.
915          */
916         if ((flags & WORKER_NOT_RUNNING) &&
917             !(worker->flags & WORKER_NOT_RUNNING)) {
918                 if (wakeup) {
919                         if (atomic_dec_and_test(&pool->nr_running) &&
920                             !list_empty(&pool->worklist))
921                                 wake_up_worker(pool);
922                 } else
923                         atomic_dec(&pool->nr_running);
924         }
925 
926         worker->flags |= flags;
927 }
928 
929 /**
930  * worker_clr_flags - clear worker flags and adjust nr_running accordingly
931  * @worker: self
932  * @flags: flags to clear
933  *
934  * Clear @flags in @worker->flags and adjust nr_running accordingly.
935  *
936  * CONTEXT:
937  * spin_lock_irq(pool->lock)
938  */
939 static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
940 {
941         struct worker_pool *pool = worker->pool;
942         unsigned int oflags = worker->flags;
943 
944         WARN_ON_ONCE(worker->task != current);
945 
946         worker->flags &= ~flags;
947 
948         /*
949          * If transitioning out of NOT_RUNNING, increment nr_running.  Note
950          * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
951          * of multiple flags, not a single flag.
952          */
953         if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
954                 if (!(worker->flags & WORKER_NOT_RUNNING))
955                         atomic_inc(&pool->nr_running);
956 }
957 
958 /**
959  * find_worker_executing_work - find worker which is executing a work
960  * @pool: pool of interest
961  * @work: work to find worker for
962  *
963  * Find a worker which is executing @work on @pool by searching
964  * @pool->busy_hash which is keyed by the address of @work.  For a worker
965  * to match, its current execution should match the address of @work and
966  * its work function.  This is to avoid unwanted dependency between
967  * unrelated work executions through a work item being recycled while still
968  * being executed.
969  *
970  * This is a bit tricky.  A work item may be freed once its execution
971  * starts and nothing prevents the freed area from being recycled for
972  * another work item.  If the same work item address ends up being reused
973  * before the original execution finishes, workqueue will identify the
974  * recycled work item as currently executing and make it wait until the
975  * current execution finishes, introducing an unwanted dependency.
976  *
977  * This function checks the work item address and work function to avoid
978  * false positives.  Note that this isn't complete as one may construct a
979  * work function which can introduce dependency onto itself through a
980  * recycled work item.  Well, if somebody wants to shoot oneself in the
981  * foot that badly, there's only so much we can do, and if such deadlock
982  * actually occurs, it should be easy to locate the culprit work function.
983  *
984  * CONTEXT:
985  * spin_lock_irq(pool->lock).
986  *
987  * RETURNS:
988  * Pointer to worker which is executing @work if found, NULL
989  * otherwise.
990  */
991 static struct worker *find_worker_executing_work(struct worker_pool *pool,
992                                                  struct work_struct *work)
993 {
994         struct worker *worker;
995 
996         hash_for_each_possible(pool->busy_hash, worker, hentry,
997                                (unsigned long)work)
998                 if (worker->current_work == work &&
999                     worker->current_func == work->func)
1000                         return worker;
1001 
1002         return NULL;
1003 }
1004 
1005 /**
1006  * move_linked_works - move linked works to a list
1007  * @work: start of series of works to be scheduled
1008  * @head: target list to append @work to
1009  * @nextp: out paramter for nested worklist walking
1010  *
1011  * Schedule linked works starting from @work to @head.  Work series to
1012  * be scheduled starts at @work and includes any consecutive work with
1013  * WORK_STRUCT_LINKED set in its predecessor.
1014  *
1015  * If @nextp is not NULL, it's updated to point to the next work of
1016  * the last scheduled work.  This allows move_linked_works() to be
1017  * nested inside outer list_for_each_entry_safe().
1018  *
1019  * CONTEXT:
1020  * spin_lock_irq(pool->lock).
1021  */
1022 static void move_linked_works(struct work_struct *work, struct list_head *head,
1023                               struct work_struct **nextp)
1024 {
1025         struct work_struct *n;
1026 
1027         /*
1028          * Linked worklist will always end before the end of the list,
1029          * use NULL for list head.
1030          */
1031         list_for_each_entry_safe_from(work, n, NULL, entry) {
1032                 list_move_tail(&work->entry, head);
1033                 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1034                         break;
1035         }
1036 
1037         /*
1038          * If we're already inside safe list traversal and have moved
1039          * multiple works to the scheduled queue, the next position
1040          * needs to be updated.
1041          */
1042         if (nextp)
1043                 *nextp = n;
1044 }
1045 
1046 /**
1047  * get_pwq - get an extra reference on the specified pool_workqueue
1048  * @pwq: pool_workqueue to get
1049  *
1050  * Obtain an extra reference on @pwq.  The caller should guarantee that
1051  * @pwq has positive refcnt and be holding the matching pool->lock.
1052  */
1053 static void get_pwq(struct pool_workqueue *pwq)
1054 {
1055         lockdep_assert_held(&pwq->pool->lock);
1056         WARN_ON_ONCE(pwq->refcnt <= 0);
1057         pwq->refcnt++;
1058 }
1059 
1060 /**
1061  * put_pwq - put a pool_workqueue reference
1062  * @pwq: pool_workqueue to put
1063  *
1064  * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
1065  * destruction.  The caller should be holding the matching pool->lock.
1066  */
1067 static void put_pwq(struct pool_workqueue *pwq)
1068 {
1069         lockdep_assert_held(&pwq->pool->lock);
1070         if (likely(--pwq->refcnt))
1071                 return;
1072         if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1073                 return;
1074         /*
1075          * @pwq can't be released under pool->lock, bounce to
1076          * pwq_unbound_release_workfn().  This never recurses on the same
1077          * pool->lock as this path is taken only for unbound workqueues and
1078          * the release work item is scheduled on a per-cpu workqueue.  To
1079          * avoid lockdep warning, unbound pool->locks are given lockdep
1080          * subclass of 1 in get_unbound_pool().
1081          */
1082         schedule_work(&pwq->unbound_release_work);
1083 }
1084 
1085 /**
1086  * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1087  * @pwq: pool_workqueue to put (can be %NULL)
1088  *
1089  * put_pwq() with locking.  This function also allows %NULL @pwq.
1090  */
1091 static void put_pwq_unlocked(struct pool_workqueue *pwq)
1092 {
1093         if (pwq) {
1094                 /*
1095                  * As both pwqs and pools are sched-RCU protected, the
1096                  * following lock operations are safe.
1097                  */
1098                 spin_lock_irq(&pwq->pool->lock);
1099                 put_pwq(pwq);
1100                 spin_unlock_irq(&pwq->pool->lock);
1101         }
1102 }
1103 
1104 static void pwq_activate_delayed_work(struct work_struct *work)
1105 {
1106         struct pool_workqueue *pwq = get_work_pwq(work);
1107 
1108         trace_workqueue_activate_work(work);
1109         move_linked_works(work, &pwq->pool->worklist, NULL);
1110         __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1111         pwq->nr_active++;
1112 }
1113 
1114 static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1115 {
1116         struct work_struct *work = list_first_entry(&pwq->delayed_works,
1117                                                     struct work_struct, entry);
1118 
1119         pwq_activate_delayed_work(work);
1120 }
1121 
1122 /**
1123  * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1124  * @pwq: pwq of interest
1125  * @color: color of work which left the queue
1126  *
1127  * A work either has completed or is removed from pending queue,
1128  * decrement nr_in_flight of its pwq and handle workqueue flushing.
1129  *
1130  * CONTEXT:
1131  * spin_lock_irq(pool->lock).
1132  */
1133 static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1134 {
1135         /* uncolored work items don't participate in flushing or nr_active */
1136         if (color == WORK_NO_COLOR)
1137                 goto out_put;
1138 
1139         pwq->nr_in_flight[color]--;
1140 
1141         pwq->nr_active--;
1142         if (!list_empty(&pwq->delayed_works)) {
1143                 /* one down, submit a delayed one */
1144                 if (pwq->nr_active < pwq->max_active)
1145                         pwq_activate_first_delayed(pwq);
1146         }
1147 
1148         /* is flush in progress and are we at the flushing tip? */
1149         if (likely(pwq->flush_color != color))
1150                 goto out_put;
1151 
1152         /* are there still in-flight works? */
1153         if (pwq->nr_in_flight[color])
1154                 goto out_put;
1155 
1156         /* this pwq is done, clear flush_color */
1157         pwq->flush_color = -1;
1158 
1159         /*
1160          * If this was the last pwq, wake up the first flusher.  It
1161          * will handle the rest.
1162          */
1163         if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1164                 complete(&pwq->wq->first_flusher->done);
1165 out_put:
1166         put_pwq(pwq);
1167 }
1168 
1169 /**
1170  * try_to_grab_pending - steal work item from worklist and disable irq
1171  * @work: work item to steal
1172  * @is_dwork: @work is a delayed_work
1173  * @flags: place to store irq state
1174  *
1175  * Try to grab PENDING bit of @work.  This function can handle @work in any
1176  * stable state - idle, on timer or on worklist.  Return values are
1177  *
1178  *  1           if @work was pending and we successfully stole PENDING
1179  *  0           if @work was idle and we claimed PENDING
1180  *  -EAGAIN     if PENDING couldn't be grabbed at the moment, safe to busy-retry
1181  *  -ENOENT     if someone else is canceling @work, this state may persist
1182  *              for arbitrarily long
1183  *
1184  * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
1185  * interrupted while holding PENDING and @work off queue, irq must be
1186  * disabled on entry.  This, combined with delayed_work->timer being
1187  * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1188  *
1189  * On successful return, >= 0, irq is disabled and the caller is
1190  * responsible for releasing it using local_irq_restore(*@flags).
1191  *
1192  * This function is safe to call from any context including IRQ handler.
1193  */
1194 static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1195                                unsigned long *flags)
1196 {
1197         struct worker_pool *pool;
1198         struct pool_workqueue *pwq;
1199 
1200         local_irq_save(*flags);
1201 
1202         /* try to steal the timer if it exists */
1203         if (is_dwork) {
1204                 struct delayed_work *dwork = to_delayed_work(work);
1205 
1206                 /*
1207                  * dwork->timer is irqsafe.  If del_timer() fails, it's
1208                  * guaranteed that the timer is not queued anywhere and not
1209                  * running on the local CPU.
1210                  */
1211                 if (likely(del_timer(&dwork->timer)))
1212                         return 1;
1213         }
1214 
1215         /* try to claim PENDING the normal way */
1216         if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1217                 return 0;
1218 
1219         /*
1220          * The queueing is in progress, or it is already queued. Try to
1221          * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1222          */
1223         pool = get_work_pool(work);
1224         if (!pool)
1225                 goto fail;
1226 
1227         spin_lock(&pool->lock);
1228         /*
1229          * work->data is guaranteed to point to pwq only while the work
1230          * item is queued on pwq->wq, and both updating work->data to point
1231          * to pwq on queueing and to pool on dequeueing are done under
1232          * pwq->pool->lock.  This in turn guarantees that, if work->data
1233          * points to pwq which is associated with a locked pool, the work
1234          * item is currently queued on that pool.
1235          */
1236         pwq = get_work_pwq(work);
1237         if (pwq && pwq->pool == pool) {
1238                 debug_work_deactivate(work);
1239 
1240                 /*
1241                  * A delayed work item cannot be grabbed directly because
1242                  * it might have linked NO_COLOR work items which, if left
1243                  * on the delayed_list, will confuse pwq->nr_active
1244                  * management later on and cause stall.  Make sure the work
1245                  * item is activated before grabbing.
1246                  */
1247                 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1248                         pwq_activate_delayed_work(work);
1249 
1250                 list_del_init(&work->entry);
1251                 pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1252 
1253                 /* work->data points to pwq iff queued, point to pool */
1254                 set_work_pool_and_keep_pending(work, pool->id);
1255 
1256                 spin_unlock(&pool->lock);
1257                 return 1;
1258         }
1259         spin_unlock(&pool->lock);
1260 fail:
1261         local_irq_restore(*flags);
1262         if (work_is_canceling(work))
1263                 return -ENOENT;
1264         cpu_relax();
1265         return -EAGAIN;
1266 }
1267 
1268 /**
1269  * insert_work - insert a work into a pool
1270  * @pwq: pwq @work belongs to
1271  * @work: work to insert
1272  * @head: insertion point
1273  * @extra_flags: extra WORK_STRUCT_* flags to set
1274  *
1275  * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
1276  * work_struct flags.
1277  *
1278  * CONTEXT:
1279  * spin_lock_irq(pool->lock).
1280  */
1281 static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1282                         struct list_head *head, unsigned int extra_flags)
1283 {
1284         struct worker_pool *pool = pwq->pool;
1285 
1286         /* we own @work, set data and link */
1287         set_work_pwq(work, pwq, extra_flags);
1288         list_add_tail(&work->entry, head);
1289         get_pwq(pwq);
1290 
1291         /*
1292          * Ensure either wq_worker_sleeping() sees the above
1293          * list_add_tail() or we see zero nr_running to avoid workers lying
1294          * around lazily while there are works to be processed.
1295          */
1296         smp_mb();
1297 
1298         if (__need_more_worker(pool))
1299                 wake_up_worker(pool);
1300 }
1301 
1302 /*
1303  * Test whether @work is being queued from another work executing on the
1304  * same workqueue.
1305  */
1306 static bool is_chained_work(struct workqueue_struct *wq)
1307 {
1308         struct worker *worker;
1309 
1310         worker = current_wq_worker();
1311         /*
1312          * Return %true iff I'm a worker execuing a work item on @wq.  If
1313          * I'm @worker, it's safe to dereference it without locking.
1314          */
1315         return worker && worker->current_pwq->wq == wq;
1316 }
1317 
1318 static void __queue_work(int cpu, struct workqueue_struct *wq,
1319                          struct work_struct *work)
1320 {
1321         struct pool_workqueue *pwq;
1322         struct worker_pool *last_pool;
1323         struct list_head *worklist;
1324         unsigned int work_flags;
1325         unsigned int req_cpu = cpu;
1326 
1327         /*
1328          * While a work item is PENDING && off queue, a task trying to
1329          * steal the PENDING will busy-loop waiting for it to either get
1330          * queued or lose PENDING.  Grabbing PENDING and queueing should
1331          * happen with IRQ disabled.
1332          */
1333         WARN_ON_ONCE(!irqs_disabled());
1334 
1335         debug_work_activate(work);
1336 
1337         /* if dying, only works from the same workqueue are allowed */
1338         if (unlikely(wq->flags & __WQ_DRAINING) &&
1339             WARN_ON_ONCE(!is_chained_work(wq)))
1340                 return;
1341 retry:
1342         if (req_cpu == WORK_CPU_UNBOUND)
1343                 cpu = raw_smp_processor_id();
1344 
1345         /* pwq which will be used unless @work is executing elsewhere */
1346         if (!(wq->flags & WQ_UNBOUND))
1347                 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1348         else
1349                 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1350 
1351         /*
1352          * If @work was previously on a different pool, it might still be
1353          * running there, in which case the work needs to be queued on that
1354          * pool to guarantee non-reentrancy.
1355          */
1356         last_pool = get_work_pool(work);
1357         if (last_pool && last_pool != pwq->pool) {
1358                 struct worker *worker;
1359 
1360                 spin_lock(&last_pool->lock);
1361 
1362                 worker = find_worker_executing_work(last_pool, work);
1363 
1364                 if (worker && worker->current_pwq->wq == wq) {
1365                         pwq = worker->current_pwq;
1366                 } else {
1367                         /* meh... not running there, queue here */
1368                         spin_unlock(&last_pool->lock);
1369                         spin_lock(&pwq->pool->lock);
1370                 }
1371         } else {
1372                 spin_lock(&pwq->pool->lock);
1373         }
1374 
1375         /*
1376          * pwq is determined and locked.  For unbound pools, we could have
1377          * raced with pwq release and it could already be dead.  If its
1378          * refcnt is zero, repeat pwq selection.  Note that pwqs never die
1379          * without another pwq replacing it in the numa_pwq_tbl or while
1380          * work items are executing on it, so the retrying is guaranteed to
1381          * make forward-progress.
1382          */
1383         if (unlikely(!pwq->refcnt)) {
1384                 if (wq->flags & WQ_UNBOUND) {
1385                         spin_unlock(&pwq->pool->lock);
1386                         cpu_relax();
1387                         goto retry;
1388                 }
1389                 /* oops */
1390                 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1391                           wq->name, cpu);
1392         }
1393 
1394         /* pwq determined, queue */
1395         trace_workqueue_queue_work(req_cpu, pwq, work);
1396 
1397         if (WARN_ON(!list_empty(&work->entry))) {
1398                 spin_unlock(&pwq->pool->lock);
1399                 return;
1400         }
1401 
1402         pwq->nr_in_flight[pwq->work_color]++;
1403         work_flags = work_color_to_flags(pwq->work_color);
1404 
1405         if (likely(pwq->nr_active < pwq->max_active)) {
1406                 trace_workqueue_activate_work(work);
1407                 pwq->nr_active++;
1408                 worklist = &pwq->pool->worklist;
1409         } else {
1410                 work_flags |= WORK_STRUCT_DELAYED;
1411                 worklist = &pwq->delayed_works;
1412         }
1413 
1414         insert_work(pwq, work, worklist, work_flags);
1415 
1416         spin_unlock(&pwq->pool->lock);
1417 }
1418 
1419 /**
1420  * queue_work_on - queue work on specific cpu
1421  * @cpu: CPU number to execute work on
1422  * @wq: workqueue to use
1423  * @work: work to queue
1424  *
1425  * Returns %false if @work was already on a queue, %true otherwise.
1426  *
1427  * We queue the work to a specific CPU, the caller must ensure it
1428  * can't go away.
1429  */
1430 bool queue_work_on(int cpu, struct workqueue_struct *wq,
1431                    struct work_struct *work)
1432 {
1433         bool ret = false;
1434         unsigned long flags;
1435 
1436         local_irq_save(flags);
1437 
1438         if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1439                 __queue_work(cpu, wq, work);
1440                 ret = true;
1441         }
1442 
1443         local_irq_restore(flags);
1444         return ret;
1445 }
1446 EXPORT_SYMBOL(queue_work_on);
1447 
1448 void delayed_work_timer_fn(unsigned long __data)
1449 {
1450         struct delayed_work *dwork = (struct delayed_work *)__data;
1451 
1452         /* should have been called from irqsafe timer with irq already off */
1453         __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1454 }
1455 EXPORT_SYMBOL(delayed_work_timer_fn);
1456 
1457 static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1458                                 struct delayed_work *dwork, unsigned long delay)
1459 {
1460         struct timer_list *timer = &dwork->timer;
1461         struct work_struct *work = &dwork->work;
1462 
1463         WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1464                      timer->data != (unsigned long)dwork);
1465         WARN_ON_ONCE(timer_pending(timer));
1466         WARN_ON_ONCE(!list_empty(&work->entry));
1467 
1468         /*
1469          * If @delay is 0, queue @dwork->work immediately.  This is for
1470          * both optimization and correctness.  The earliest @timer can
1471          * expire is on the closest next tick and delayed_work users depend
1472          * on that there's no such delay when @delay is 0.
1473          */
1474         if (!delay) {
1475                 __queue_work(cpu, wq, &dwork->work);
1476                 return;
1477         }
1478 
1479         timer_stats_timer_set_start_info(&dwork->timer);
1480 
1481         dwork->wq = wq;
1482         dwork->cpu = cpu;
1483         timer->expires = jiffies + delay;
1484 
1485         if (unlikely(cpu != WORK_CPU_UNBOUND))
1486                 add_timer_on(timer, cpu);
1487         else
1488                 add_timer(timer);
1489 }
1490 
1491 /**
1492  * queue_delayed_work_on - queue work on specific CPU after delay
1493  * @cpu: CPU number to execute work on
1494  * @wq: workqueue to use
1495  * @dwork: work to queue
1496  * @delay: number of jiffies to wait before queueing
1497  *
1498  * Returns %false if @work was already on a queue, %true otherwise.  If
1499  * @delay is zero and @dwork is idle, it will be scheduled for immediate
1500  * execution.
1501  */
1502 bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1503                            struct delayed_work *dwork, unsigned long delay)
1504 {
1505         struct work_struct *work = &dwork->work;
1506         bool ret = false;
1507         unsigned long flags;
1508 
1509         /* read the comment in __queue_work() */
1510         local_irq_save(flags);
1511 
1512         if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1513                 __queue_delayed_work(cpu, wq, dwork, delay);
1514                 ret = true;
1515         }
1516 
1517         local_irq_restore(flags);
1518         return ret;
1519 }
1520 EXPORT_SYMBOL(queue_delayed_work_on);
1521 
1522 /**
1523  * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1524  * @cpu: CPU number to execute work on
1525  * @wq: workqueue to use
1526  * @dwork: work to queue
1527  * @delay: number of jiffies to wait before queueing
1528  *
1529  * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1530  * modify @dwork's timer so that it expires after @delay.  If @delay is
1531  * zero, @work is guaranteed to be scheduled immediately regardless of its
1532  * current state.
1533  *
1534  * Returns %false if @dwork was idle and queued, %true if @dwork was
1535  * pending and its timer was modified.
1536  *
1537  * This function is safe to call from any context including IRQ handler.
1538  * See try_to_grab_pending() for details.
1539  */
1540 bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1541                          struct delayed_work *dwork, unsigned long delay)
1542 {
1543         unsigned long flags;
1544         int ret;
1545 
1546         do {
1547                 ret = try_to_grab_pending(&dwork->work, true, &flags);
1548         } while (unlikely(ret == -EAGAIN));
1549 
1550         if (likely(ret >= 0)) {
1551                 __queue_delayed_work(cpu, wq, dwork, delay);
1552                 local_irq_restore(flags);
1553         }
1554 
1555         /* -ENOENT from try_to_grab_pending() becomes %true */
1556         return ret;
1557 }
1558 EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1559 
1560 /**
1561  * worker_enter_idle - enter idle state
1562  * @worker: worker which is entering idle state
1563  *
1564  * @worker is entering idle state.  Update stats and idle timer if
1565  * necessary.
1566  *
1567  * LOCKING:
1568  * spin_lock_irq(pool->lock).
1569  */
1570 static void worker_enter_idle(struct worker *worker)
1571 {
1572         struct worker_pool *pool = worker->pool;
1573 
1574         if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1575             WARN_ON_ONCE(!list_empty(&worker->entry) &&
1576                          (worker->hentry.next || worker->hentry.pprev)))
1577                 return;
1578 
1579         /* can't use worker_set_flags(), also called from start_worker() */
1580         worker->flags |= WORKER_IDLE;
1581         pool->nr_idle++;
1582         worker->last_active = jiffies;
1583 
1584         /* idle_list is LIFO */
1585         list_add(&worker->entry, &pool->idle_list);
1586 
1587         if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1588                 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1589 
1590         /*
1591          * Sanity check nr_running.  Because wq_unbind_fn() releases
1592          * pool->lock between setting %WORKER_UNBOUND and zapping
1593          * nr_running, the warning may trigger spuriously.  Check iff
1594          * unbind is not in progress.
1595          */
1596         WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1597                      pool->nr_workers == pool->nr_idle &&
1598                      atomic_read(&pool->nr_running));
1599 }
1600 
1601 /**
1602  * worker_leave_idle - leave idle state
1603  * @worker: worker which is leaving idle state
1604  *
1605  * @worker is leaving idle state.  Update stats.
1606  *
1607  * LOCKING:
1608  * spin_lock_irq(pool->lock).
1609  */
1610 static void worker_leave_idle(struct worker *worker)
1611 {
1612         struct worker_pool *pool = worker->pool;
1613 
1614         if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1615                 return;
1616         worker_clr_flags(worker, WORKER_IDLE);
1617         pool->nr_idle--;
1618         list_del_init(&worker->entry);
1619 }
1620 
1621 /**
1622  * worker_maybe_bind_and_lock - try to bind %current to worker_pool and lock it
1623  * @pool: target worker_pool
1624  *
1625  * Bind %current to the cpu of @pool if it is associated and lock @pool.
1626  *
1627  * Works which are scheduled while the cpu is online must at least be
1628  * scheduled to a worker which is bound to the cpu so that if they are
1629  * flushed from cpu callbacks while cpu is going down, they are
1630  * guaranteed to execute on the cpu.
1631  *
1632  * This function is to be used by unbound workers and rescuers to bind
1633  * themselves to the target cpu and may race with cpu going down or
1634  * coming online.  kthread_bind() can't be used because it may put the
1635  * worker to already dead cpu and set_cpus_allowed_ptr() can't be used
1636  * verbatim as it's best effort and blocking and pool may be
1637  * [dis]associated in the meantime.
1638  *
1639  * This function tries set_cpus_allowed() and locks pool and verifies the
1640  * binding against %POOL_DISASSOCIATED which is set during
1641  * %CPU_DOWN_PREPARE and cleared during %CPU_ONLINE, so if the worker
1642  * enters idle state or fetches works without dropping lock, it can
1643  * guarantee the scheduling requirement described in the first paragraph.
1644  *
1645  * CONTEXT:
1646  * Might sleep.  Called without any lock but returns with pool->lock
1647  * held.
1648  *
1649  * RETURNS:
1650  * %true if the associated pool is online (@worker is successfully
1651  * bound), %false if offline.
1652  */
1653 static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1654 __acquires(&pool->lock)
1655 {
1656         while (true) {
1657                 /*
1658                  * The following call may fail, succeed or succeed
1659                  * without actually migrating the task to the cpu if
1660                  * it races with cpu hotunplug operation.  Verify
1661                  * against POOL_DISASSOCIATED.
1662                  */
1663                 if (!(pool->flags & POOL_DISASSOCIATED))
1664                         set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1665 
1666                 spin_lock_irq(&pool->lock);
1667                 if (pool->flags & POOL_DISASSOCIATED)
1668                         return false;
1669                 if (task_cpu(current) == pool->cpu &&
1670                     cpumask_equal(&current->cpus_allowed, pool->attrs->cpumask))
1671                         return true;
1672                 spin_unlock_irq(&pool->lock);
1673 
1674                 /*
1675                  * We've raced with CPU hot[un]plug.  Give it a breather
1676                  * and retry migration.  cond_resched() is required here;
1677                  * otherwise, we might deadlock against cpu_stop trying to
1678                  * bring down the CPU on non-preemptive kernel.
1679                  */
1680                 cpu_relax();
1681                 cond_resched();
1682         }
1683 }
1684 
1685 static struct worker *alloc_worker(void)
1686 {
1687         struct worker *worker;
1688 
1689         worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1690         if (worker) {
1691                 INIT_LIST_HEAD(&worker->entry);
1692                 INIT_LIST_HEAD(&worker->scheduled);
1693                 /* on creation a worker is in !idle && prep state */
1694                 worker->flags = WORKER_PREP;
1695         }
1696         return worker;
1697 }
1698 
1699 /**
1700  * create_worker - create a new workqueue worker
1701  * @pool: pool the new worker will belong to
1702  *
1703  * Create a new worker which is bound to @pool.  The returned worker
1704  * can be started by calling start_worker() or destroyed using
1705  * destroy_worker().
1706  *
1707  * CONTEXT:
1708  * Might sleep.  Does GFP_KERNEL allocations.
1709  *
1710  * RETURNS:
1711  * Pointer to the newly created worker.
1712  */
1713 static struct worker *create_worker(struct worker_pool *pool)
1714 {
1715         struct worker *worker = NULL;
1716         int id = -1;
1717         char id_buf[16];
1718 
1719         lockdep_assert_held(&pool->manager_mutex);
1720 
1721         /*
1722          * ID is needed to determine kthread name.  Allocate ID first
1723          * without installing the pointer.
1724          */
1725         idr_preload(GFP_KERNEL);
1726         spin_lock_irq(&pool->lock);
1727 
1728         id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1729 
1730         spin_unlock_irq(&pool->lock);
1731         idr_preload_end();
1732         if (id < 0)
1733                 goto fail;
1734 
1735         worker = alloc_worker();
1736         if (!worker)
1737                 goto fail;
1738 
1739         worker->pool = pool;
1740         worker->id = id;
1741 
1742         if (pool->cpu >= 0)
1743                 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1744                          pool->attrs->nice < 0  ? "H" : "");
1745         else
1746                 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1747 
1748         worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1749                                               "kworker/%s", id_buf);
1750         if (IS_ERR(worker->task))
1751                 goto fail;
1752 
1753         /*
1754          * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any
1755          * online CPUs.  It'll be re-applied when any of the CPUs come up.
1756          */
1757         set_user_nice(worker->task, pool->attrs->nice);
1758         set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1759 
1760         /* prevent userland from meddling with cpumask of workqueue workers */
1761         worker->task->flags |= PF_NO_SETAFFINITY;
1762 
1763         /*
1764          * The caller is responsible for ensuring %POOL_DISASSOCIATED
1765          * remains stable across this function.  See the comments above the
1766          * flag definition for details.
1767          */
1768         if (pool->flags & POOL_DISASSOCIATED)
1769                 worker->flags |= WORKER_UNBOUND;
1770 
1771         /* successful, commit the pointer to idr */
1772         spin_lock_irq(&pool->lock);
1773         idr_replace(&pool->worker_idr, worker, worker->id);
1774         spin_unlock_irq(&pool->lock);
1775 
1776         return worker;
1777 
1778 fail:
1779         if (id >= 0) {
1780                 spin_lock_irq(&pool->lock);
1781                 idr_remove(&pool->worker_idr, id);
1782                 spin_unlock_irq(&pool->lock);
1783         }
1784         kfree(worker);
1785         return NULL;
1786 }
1787 
1788 /**
1789  * start_worker - start a newly created worker
1790  * @worker: worker to start
1791  *
1792  * Make the pool aware of @worker and start it.
1793  *
1794  * CONTEXT:
1795  * spin_lock_irq(pool->lock).
1796  */
1797 static void start_worker(struct worker *worker)
1798 {
1799         worker->flags |= WORKER_STARTED;
1800         worker->pool->nr_workers++;
1801         worker_enter_idle(worker);
1802         wake_up_process(worker->task);
1803 }
1804 
1805 /**
1806  * create_and_start_worker - create and start a worker for a pool
1807  * @pool: the target pool
1808  *
1809  * Grab the managership of @pool and create and start a new worker for it.
1810  */
1811 static int create_and_start_worker(struct worker_pool *pool)
1812 {
1813         struct worker *worker;
1814 
1815         mutex_lock(&pool->manager_mutex);
1816 
1817         worker = create_worker(pool);
1818         if (worker) {
1819                 spin_lock_irq(&pool->lock);
1820                 start_worker(worker);
1821                 spin_unlock_irq(&pool->lock);
1822         }
1823 
1824         mutex_unlock(&pool->manager_mutex);
1825 
1826         return worker ? 0 : -ENOMEM;
1827 }
1828 
1829 /**
1830  * destroy_worker - destroy a workqueue worker
1831  * @worker: worker to be destroyed
1832  *
1833  * Destroy @worker and adjust @pool stats accordingly.
1834  *
1835  * CONTEXT:
1836  * spin_lock_irq(pool->lock) which is released and regrabbed.
1837  */
1838 static void destroy_worker(struct worker *worker)
1839 {
1840         struct worker_pool *pool = worker->pool;
1841 
1842         lockdep_assert_held(&pool->manager_mutex);
1843         lockdep_assert_held(&pool->lock);
1844 
1845         /* sanity check frenzy */
1846         if (WARN_ON(worker->current_work) ||
1847             WARN_ON(!list_empty(&worker->scheduled)))
1848                 return;
1849 
1850         if (worker->flags & WORKER_STARTED)
1851                 pool->nr_workers--;
1852         if (worker->flags & WORKER_IDLE)
1853                 pool->nr_idle--;
1854 
1855         /*
1856          * Once WORKER_DIE is set, the kworker may destroy itself at any
1857          * point.  Pin to ensure the task stays until we're done with it.
1858          */
1859         get_task_struct(worker->task);
1860 
1861         list_del_init(&worker->entry);
1862         worker->flags |= WORKER_DIE;
1863 
1864         idr_remove(&pool->worker_idr, worker->id);
1865 
1866         spin_unlock_irq(&pool->lock);
1867 
1868         kthread_stop(worker->task);
1869         put_task_struct(worker->task);
1870         kfree(worker);
1871 
1872         spin_lock_irq(&pool->lock);
1873 }
1874 
1875 static void idle_worker_timeout(unsigned long __pool)
1876 {
1877         struct worker_pool *pool = (void *)__pool;
1878 
1879         spin_lock_irq(&pool->lock);
1880 
1881         if (too_many_workers(pool)) {
1882                 struct worker *worker;
1883                 unsigned long expires;
1884 
1885                 /* idle_list is kept in LIFO order, check the last one */
1886                 worker = list_entry(pool->idle_list.prev, struct worker, entry);
1887                 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1888 
1889                 if (time_before(jiffies, expires))
1890                         mod_timer(&pool->idle_timer, expires);
1891                 else {
1892                         /* it's been idle for too long, wake up manager */
1893                         pool->flags |= POOL_MANAGE_WORKERS;
1894                         wake_up_worker(pool);
1895                 }
1896         }
1897 
1898         spin_unlock_irq(&pool->lock);
1899 }
1900 
1901 static void send_mayday(struct work_struct *work)
1902 {
1903         struct pool_workqueue *pwq = get_work_pwq(work);
1904         struct workqueue_struct *wq = pwq->wq;
1905 
1906         lockdep_assert_held(&wq_mayday_lock);
1907 
1908         if (!wq->rescuer)
1909                 return;
1910 
1911         /* mayday mayday mayday */
1912         if (list_empty(&pwq->mayday_node)) {
1913                 /*
1914                  * If @pwq is for an unbound wq, its base ref may be put at
1915                  * any time due to an attribute change.  Pin @pwq until the
1916                  * rescuer is done with it.
1917                  */
1918                 get_pwq(pwq);
1919                 list_add_tail(&pwq->mayday_node, &wq->maydays);
1920                 wake_up_process(wq->rescuer->task);
1921         }
1922 }
1923 
1924 static void pool_mayday_timeout(unsigned long __pool)
1925 {
1926         struct worker_pool *pool = (void *)__pool;
1927         struct work_struct *work;
1928 
1929         spin_lock_irq(&wq_mayday_lock);         /* for wq->maydays */
1930         spin_lock(&pool->lock);
1931 
1932         if (need_to_create_worker(pool)) {
1933                 /*
1934                  * We've been trying to create a new worker but
1935                  * haven't been successful.  We might be hitting an
1936                  * allocation deadlock.  Send distress signals to
1937                  * rescuers.
1938                  */
1939                 list_for_each_entry(work, &pool->worklist, entry)
1940                         send_mayday(work);
1941         }
1942 
1943         spin_unlock(&pool->lock);
1944         spin_unlock_irq(&wq_mayday_lock);
1945 
1946         mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1947 }
1948 
1949 /**
1950  * maybe_create_worker - create a new worker if necessary
1951  * @pool: pool to create a new worker for
1952  *
1953  * Create a new worker for @pool if necessary.  @pool is guaranteed to
1954  * have at least one idle worker on return from this function.  If
1955  * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
1956  * sent to all rescuers with works scheduled on @pool to resolve
1957  * possible allocation deadlock.
1958  *
1959  * On return, need_to_create_worker() is guaranteed to be %false and
1960  * may_start_working() %true.
1961  *
1962  * LOCKING:
1963  * spin_lock_irq(pool->lock) which may be released and regrabbed
1964  * multiple times.  Does GFP_KERNEL allocations.  Called only from
1965  * manager.
1966  */
1967 static void maybe_create_worker(struct worker_pool *pool)
1968 __releases(&pool->lock)
1969 __acquires(&pool->lock)
1970 {
1971         if (!need_to_create_worker(pool))
1972                 return;
1973 restart:
1974         spin_unlock_irq(&pool->lock);
1975 
1976         /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
1977         mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1978 
1979         while (true) {
1980                 struct worker *worker;
1981 
1982                 worker = create_worker(pool);
1983                 if (worker) {
1984                         del_timer_sync(&pool->mayday_timer);
1985                         spin_lock_irq(&pool->lock);
1986                         start_worker(worker);
1987                         if (WARN_ON_ONCE(need_to_create_worker(pool)))
1988                                 goto restart;
1989                         return;
1990                 }
1991 
1992                 if (!need_to_create_worker(pool))
1993                         break;
1994 
1995                 __set_current_state(TASK_INTERRUPTIBLE);
1996                 schedule_timeout(CREATE_COOLDOWN);
1997 
1998                 if (!need_to_create_worker(pool))
1999                         break;
2000         }
2001 
2002         del_timer_sync(&pool->mayday_timer);
2003         spin_lock_irq(&pool->lock);
2004         if (need_to_create_worker(pool))
2005                 goto restart;
2006         return;
2007 }
2008 
2009 /**
2010  * maybe_destroy_worker - destroy workers which have been idle for a while
2011  * @pool: pool to destroy workers for
2012  *
2013  * Destroy @pool workers which have been idle for longer than
2014  * IDLE_WORKER_TIMEOUT.
2015  *
2016  * LOCKING:
2017  * spin_lock_irq(pool->lock) which may be released and regrabbed
2018  * multiple times.  Called only from manager.
2019  */
2020 static void maybe_destroy_workers(struct worker_pool *pool)
2021 {
2022         while (too_many_workers(pool)) {
2023                 struct worker *worker;
2024                 unsigned long expires;
2025 
2026                 worker = list_entry(pool->idle_list.prev, struct worker, entry);
2027                 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2028 
2029                 if (time_before(jiffies, expires)) {
2030                         mod_timer(&pool->idle_timer, expires);
2031                         break;
2032                 }
2033 
2034                 destroy_worker(worker);
2035         }
2036 }
2037 
2038 /**
2039  * manage_workers - manage worker pool
2040  * @worker: self
2041  *
2042  * Assume the manager role and manage the worker pool @worker belongs
2043  * to.  At any given time, there can be only zero or one manager per
2044  * pool.  The exclusion is handled automatically by this function.
2045  *
2046  * The caller can safely start processing works on false return.  On
2047  * true return, it's guaranteed that need_to_create_worker() is false
2048  * and may_start_working() is true.
2049  *
2050  * CONTEXT:
2051  * spin_lock_irq(pool->lock) which may be released and regrabbed
2052  * multiple times.  Does GFP_KERNEL allocations.
2053  *
2054  * RETURNS:
2055  * %false if the pool doesn't need management and the caller can safely
2056  * start processing works, %true if management function was performed and
2057  * the conditions that the caller verified before calling the function may
2058  * no longer be true.
2059  */
2060 static bool manage_workers(struct worker *worker)
2061 {
2062         struct worker_pool *pool = worker->pool;
2063 
2064         /*
2065          * Managership is governed by two mutexes - manager_arb and
2066          * manager_mutex.  manager_arb handles arbitration of manager role.
2067          * Anyone who successfully grabs manager_arb wins the arbitration
2068          * and becomes the manager.  mutex_trylock() on pool->manager_arb
2069          * failure while holding pool->lock reliably indicates that someone
2070          * else is managing the pool and the worker which failed trylock
2071          * can proceed to executing work items.  This means that anyone
2072          * grabbing manager_arb is responsible for actually performing
2073          * manager duties.  If manager_arb is grabbed and released without
2074          * actual management, the pool may stall indefinitely.
2075          *
2076          * manager_mutex is used for exclusion of actual management
2077          * operations.  The holder of manager_mutex can be sure that none
2078          * of management operations, including creation and destruction of
2079          * workers, won't take place until the mutex is released.  Because
2080          * manager_mutex doesn't interfere with manager role arbitration,
2081          * it is guaranteed that the pool's management, while may be
2082          * delayed, won't be disturbed by someone else grabbing
2083          * manager_mutex.
2084          */
2085         if (!mutex_trylock(&pool->manager_arb))
2086                 return false;
2087 
2088         /*
2089          * With manager arbitration won, manager_mutex would be free in
2090          * most cases.  trylock first without dropping @pool->lock.
2091          */
2092         if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2093                 spin_unlock_irq(&pool->lock);
2094                 mutex_lock(&pool->manager_mutex);
2095                 spin_lock_irq(&pool->lock);
2096         }
2097 
2098         pool->flags &= ~POOL_MANAGE_WORKERS;
2099 
2100         /*
2101          * Destroy and then create so that may_start_working() is true
2102          * on return.
2103          */
2104         maybe_destroy_workers(pool);
2105         maybe_create_worker(pool);
2106 
2107         mutex_unlock(&pool->manager_mutex);
2108         mutex_unlock(&pool->manager_arb);
2109         return true;
2110 }
2111 
2112 /**
2113  * process_one_work - process single work
2114  * @worker: self
2115  * @work: work to process
2116  *
2117  * Process @work.  This function contains all the logics necessary to
2118  * process a single work including synchronization against and
2119  * interaction with other workers on the same cpu, queueing and
2120  * flushing.  As long as context requirement is met, any worker can
2121  * call this function to process a work.
2122  *
2123  * CONTEXT:
2124  * spin_lock_irq(pool->lock) which is released and regrabbed.
2125  */
2126 static void process_one_work(struct worker *worker, struct work_struct *work)
2127 __releases(&pool->lock)
2128 __acquires(&pool->lock)
2129 {
2130         struct pool_workqueue *pwq = get_work_pwq(work);
2131         struct worker_pool *pool = worker->pool;
2132         bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2133         int work_color;
2134         struct worker *collision;
2135 #ifdef CONFIG_LOCKDEP
2136         /*
2137          * It is permissible to free the struct work_struct from
2138          * inside the function that is called from it, this we need to
2139          * take into account for lockdep too.  To avoid bogus "held
2140          * lock freed" warnings as well as problems when looking into
2141          * work->lockdep_map, make a copy and use that here.
2142          */
2143         struct lockdep_map lockdep_map;
2144 
2145         lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2146 #endif
2147         /*
2148          * Ensure we're on the correct CPU.  DISASSOCIATED test is
2149          * necessary to avoid spurious warnings from rescuers servicing the
2150          * unbound or a disassociated pool.
2151          */
2152         WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2153                      !(pool->flags & POOL_DISASSOCIATED) &&
2154                      raw_smp_processor_id() != pool->cpu);
2155 
2156         /*
2157          * A single work shouldn't be executed concurrently by
2158          * multiple workers on a single cpu.  Check whether anyone is
2159          * already processing the work.  If so, defer the work to the
2160          * currently executing one.
2161          */
2162         collision = find_worker_executing_work(pool, work);
2163         if (unlikely(collision)) {
2164                 move_linked_works(work, &collision->scheduled, NULL);
2165                 return;
2166         }
2167 
2168         /* claim and dequeue */
2169         debug_work_deactivate(work);
2170         hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2171         worker->current_work = work;
2172         worker->current_func = work->func;
2173         worker->current_pwq = pwq;
2174         work_color = get_work_color(work);
2175 
2176         list_del_init(&work->entry);
2177 
2178         /*
2179          * CPU intensive works don't participate in concurrency
2180          * management.  They're the scheduler's responsibility.
2181          */
2182         if (unlikely(cpu_intensive))
2183                 worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2184 
2185         /*
2186          * Unbound pool isn't concurrency managed and work items should be
2187          * executed ASAP.  Wake up another worker if necessary.
2188          */
2189         if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2190                 wake_up_worker(pool);
2191 
2192         /*
2193          * Record the last pool and clear PENDING which should be the last
2194          * update to @work.  Also, do this inside @pool->lock so that
2195          * PENDING and queued state changes happen together while IRQ is
2196          * disabled.
2197          */
2198         set_work_pool_and_clear_pending(work, pool->id);
2199 
2200         spin_unlock_irq(&pool->lock);
2201 
2202         lock_map_acquire_read(&pwq->wq->lockdep_map);
2203         lock_map_acquire(&lockdep_map);
2204         trace_workqueue_execute_start(work);
2205         worker->current_func(work);
2206         /*
2207          * While we must be careful to not use "work" after this, the trace
2208          * point will only record its address.
2209          */
2210         trace_workqueue_execute_end(work);
2211         lock_map_release(&lockdep_map);
2212         lock_map_release(&pwq->wq->lockdep_map);
2213 
2214         if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2215                 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2216                        "     last function: %pf\n",
2217                        current->comm, preempt_count(), task_pid_nr(current),
2218                        worker->current_func);
2219                 debug_show_held_locks(current);
2220                 dump_stack();
2221         }
2222 
2223         /*
2224          * The following prevents a kworker from hogging CPU on !PREEMPT
2225          * kernels, where a requeueing work item waiting for something to
2226          * happen could deadlock with stop_machine as such work item could
2227          * indefinitely requeue itself while all other CPUs are trapped in
2228          * stop_machine.
2229          */
2230         cond_resched();
2231 
2232         spin_lock_irq(&pool->lock);
2233 
2234         /* clear cpu intensive status */
2235         if (unlikely(cpu_intensive))
2236                 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2237 
2238         /* we're done with it, release */
2239         hash_del(&worker->hentry);
2240         worker->current_work = NULL;
2241         worker->current_func = NULL;
2242         worker->current_pwq = NULL;
2243         worker->desc_valid = false;
2244         pwq_dec_nr_in_flight(pwq, work_color);
2245 }
2246 
2247 /**
2248  * process_scheduled_works - process scheduled works
2249  * @worker: self
2250  *
2251  * Process all scheduled works.  Please note that the scheduled list
2252  * may change while processing a work, so this function repeatedly
2253  * fetches a work from the top and executes it.
2254  *
2255  * CONTEXT:
2256  * spin_lock_irq(pool->lock) which may be released and regrabbed
2257  * multiple times.
2258  */
2259 static void process_scheduled_works(struct worker *worker)
2260 {
2261         while (!list_empty(&worker->scheduled)) {
2262                 struct work_struct *work = list_first_entry(&worker->scheduled,
2263                                                 struct work_struct, entry);
2264                 process_one_work(worker, work);
2265         }
2266 }
2267 
2268 /**
2269  * worker_thread - the worker thread function
2270  * @__worker: self
2271  *
2272  * The worker thread function.  All workers belong to a worker_pool -
2273  * either a per-cpu one or dynamic unbound one.  These workers process all
2274  * work items regardless of their specific target workqueue.  The only
2275  * exception is work items which belong to workqueues with a rescuer which
2276  * will be explained in rescuer_thread().
2277  */
2278 static int worker_thread(void *__worker)
2279 {
2280         struct worker *worker = __worker;
2281         struct worker_pool *pool = worker->pool;
2282 
2283         /* tell the scheduler that this is a workqueue worker */
2284         worker->task->flags |= PF_WQ_WORKER;
2285 woke_up:
2286         spin_lock_irq(&pool->lock);
2287 
2288         /* am I supposed to die? */
2289         if (unlikely(worker->flags & WORKER_DIE)) {
2290                 spin_unlock_irq(&pool->lock);
2291                 WARN_ON_ONCE(!list_empty(&worker->entry));
2292                 worker->task->flags &= ~PF_WQ_WORKER;
2293                 return 0;
2294         }
2295 
2296         worker_leave_idle(worker);
2297 recheck:
2298         /* no more worker necessary? */
2299         if (!need_more_worker(pool))
2300                 goto sleep;
2301 
2302         /* do we need to manage? */
2303         if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2304                 goto recheck;
2305 
2306         /*
2307          * ->scheduled list can only be filled while a worker is
2308          * preparing to process a work or actually processing it.
2309          * Make sure nobody diddled with it while I was sleeping.
2310          */
2311         WARN_ON_ONCE(!list_empty(&worker->scheduled));
2312 
2313         /*
2314          * Finish PREP stage.  We're guaranteed to have at least one idle
2315          * worker or that someone else has already assumed the manager
2316          * role.  This is where @worker starts participating in concurrency
2317          * management if applicable and concurrency management is restored
2318          * after being rebound.  See rebind_workers() for details.
2319          */
2320         worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2321 
2322         do {
2323                 struct work_struct *work =
2324                         list_first_entry(&pool->worklist,
2325                                          struct work_struct, entry);
2326 
2327                 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2328                         /* optimization path, not strictly necessary */
2329                         process_one_work(worker, work);
2330                         if (unlikely(!list_empty(&worker->scheduled)))
2331                                 process_scheduled_works(worker);
2332                 } else {
2333                         move_linked_works(work, &worker->scheduled, NULL);
2334                         process_scheduled_works(worker);
2335                 }
2336         } while (keep_working(pool));
2337 
2338         worker_set_flags(worker, WORKER_PREP, false);
2339 sleep:
2340         if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2341                 goto recheck;
2342 
2343         /*
2344          * pool->lock is held and there's no work to process and no need to
2345          * manage, sleep.  Workers are woken up only while holding
2346          * pool->lock or from local cpu, so setting the current state
2347          * before releasing pool->lock is enough to prevent losing any
2348          * event.
2349          */
2350         worker_enter_idle(worker);
2351         __set_current_state(TASK_INTERRUPTIBLE);
2352         spin_unlock_irq(&pool->lock);
2353         schedule();
2354         goto woke_up;
2355 }
2356 
2357 /**
2358  * rescuer_thread - the rescuer thread function
2359  * @__rescuer: self
2360  *
2361  * Workqueue rescuer thread function.  There's one rescuer for each
2362  * workqueue which has WQ_MEM_RECLAIM set.
2363  *
2364  * Regular work processing on a pool may block trying to create a new
2365  * worker which uses GFP_KERNEL allocation which has slight chance of
2366  * developing into deadlock if some works currently on the same queue
2367  * need to be processed to satisfy the GFP_KERNEL allocation.  This is
2368  * the problem rescuer solves.
2369  *
2370  * When such condition is possible, the pool summons rescuers of all
2371  * workqueues which have works queued on the pool and let them process
2372  * those works so that forward progress can be guaranteed.
2373  *
2374  * This should happen rarely.
2375  */
2376 static int rescuer_thread(void *__rescuer)
2377 {
2378         struct worker *rescuer = __rescuer;
2379         struct workqueue_struct *wq = rescuer->rescue_wq;
2380         struct list_head *scheduled = &rescuer->scheduled;
2381         bool should_stop;
2382 
2383         set_user_nice(current, RESCUER_NICE_LEVEL);
2384 
2385         /*
2386          * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
2387          * doesn't participate in concurrency management.
2388          */
2389         rescuer->task->flags |= PF_WQ_WORKER;
2390 repeat:
2391         set_current_state(TASK_INTERRUPTIBLE);
2392 
2393         /*
2394          * By the time the rescuer is requested to stop, the workqueue
2395          * shouldn't have any work pending, but @wq->maydays may still have
2396          * pwq(s) queued.  This can happen by non-rescuer workers consuming
2397          * all the work items before the rescuer got to them.  Go through
2398          * @wq->maydays processing before acting on should_stop so that the
2399          * list is always empty on exit.
2400          */
2401         should_stop = kthread_should_stop();
2402 
2403         /* see whether any pwq is asking for help */
2404         spin_lock_irq(&wq_mayday_lock);
2405 
2406         while (!list_empty(&wq->maydays)) {
2407                 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2408                                         struct pool_workqueue, mayday_node);
2409                 struct worker_pool *pool = pwq->pool;
2410                 struct work_struct *work, *n;
2411 
2412                 __set_current_state(TASK_RUNNING);
2413                 list_del_init(&pwq->mayday_node);
2414 
2415                 spin_unlock_irq(&wq_mayday_lock);
2416 
2417                 /* migrate to the target cpu if possible */
2418                 worker_maybe_bind_and_lock(pool);
2419                 rescuer->pool = pool;
2420 
2421                 /*
2422                  * Slurp in all works issued via this workqueue and
2423                  * process'em.
2424                  */
2425                 WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2426                 list_for_each_entry_safe(work, n, &pool->worklist, entry)
2427                         if (get_work_pwq(work) == pwq)
2428                                 move_linked_works(work, scheduled, &n);
2429 
2430                 process_scheduled_works(rescuer);
2431 
2432                 /*
2433                  * Put the reference grabbed by send_mayday().  @pool won't
2434                  * go away while we're holding its lock.
2435                  */
2436                 put_pwq(pwq);
2437 
2438                 /*
2439                  * Leave this pool.  If keep_working() is %true, notify a
2440                  * regular worker; otherwise, we end up with 0 concurrency
2441                  * and stalling the execution.
2442                  */
2443                 if (keep_working(pool))
2444                         wake_up_worker(pool);
2445 
2446                 rescuer->pool = NULL;
2447                 spin_unlock(&pool->lock);
2448                 spin_lock(&wq_mayday_lock);
2449         }
2450 
2451         spin_unlock_irq(&wq_mayday_lock);
2452 
2453         if (should_stop) {
2454                 __set_current_state(TASK_RUNNING);
2455                 rescuer->task->flags &= ~PF_WQ_WORKER;
2456                 return 0;
2457         }
2458 
2459         /* rescuers should never participate in concurrency management */
2460         WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2461         schedule();
2462         goto repeat;
2463 }
2464 
2465 struct wq_barrier {
2466         struct work_struct      work;
2467         struct completion       done;
2468 };
2469 
2470 static void wq_barrier_func(struct work_struct *work)
2471 {
2472         struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2473         complete(&barr->done);
2474 }
2475 
2476 /**
2477  * insert_wq_barrier - insert a barrier work
2478  * @pwq: pwq to insert barrier into
2479  * @barr: wq_barrier to insert
2480  * @target: target work to attach @barr to
2481  * @worker: worker currently executing @target, NULL if @target is not executing
2482  *
2483  * @barr is linked to @target such that @barr is completed only after
2484  * @target finishes execution.  Please note that the ordering
2485  * guarantee is observed only with respect to @target and on the local
2486  * cpu.
2487  *
2488  * Currently, a queued barrier can't be canceled.  This is because
2489  * try_to_grab_pending() can't determine whether the work to be
2490  * grabbed is at the head of the queue and thus can't clear LINKED
2491  * flag of the previous work while there must be a valid next work
2492  * after a work with LINKED flag set.
2493  *
2494  * Note that when @worker is non-NULL, @target may be modified
2495  * underneath us, so we can't reliably determine pwq from @target.
2496  *
2497  * CONTEXT:
2498  * spin_lock_irq(pool->lock).
2499  */
2500 static void insert_wq_barrier(struct pool_workqueue *pwq,
2501                               struct wq_barrier *barr,
2502                               struct work_struct *target, struct worker *worker)
2503 {
2504         struct list_head *head;
2505         unsigned int linked = 0;
2506 
2507         /*
2508          * debugobject calls are safe here even with pool->lock locked
2509          * as we know for sure that this will not trigger any of the
2510          * checks and call back into the fixup functions where we
2511          * might deadlock.
2512          */
2513         INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2514         __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2515         init_completion(&barr->done);
2516 
2517         /*
2518          * If @target is currently being executed, schedule the
2519          * barrier to the worker; otherwise, put it after @target.
2520          */
2521         if (worker)
2522                 head = worker->scheduled.next;
2523         else {
2524                 unsigned long *bits = work_data_bits(target);
2525 
2526                 head = target->entry.next;
2527                 /* there can already be other linked works, inherit and set */
2528                 linked = *bits & WORK_STRUCT_LINKED;
2529                 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2530         }
2531 
2532         debug_work_activate(&barr->work);
2533         insert_work(pwq, &barr->work, head,
2534                     work_color_to_flags(WORK_NO_COLOR) | linked);
2535 }
2536 
2537 /**
2538  * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2539  * @wq: workqueue being flushed
2540  * @flush_color: new flush color, < 0 for no-op
2541  * @work_color: new work color, < 0 for no-op
2542  *
2543  * Prepare pwqs for workqueue flushing.
2544  *
2545  * If @flush_color is non-negative, flush_color on all pwqs should be
2546  * -1.  If no pwq has in-flight commands at the specified color, all
2547  * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
2548  * has in flight commands, its pwq->flush_color is set to
2549  * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2550  * wakeup logic is armed and %true is returned.
2551  *
2552  * The caller should have initialized @wq->first_flusher prior to
2553  * calling this function with non-negative @flush_color.  If
2554  * @flush_color is negative, no flush color update is done and %false
2555  * is returned.
2556  *
2557  * If @work_color is non-negative, all pwqs should have the same
2558  * work_color which is previous to @work_color and all will be
2559  * advanced to @work_color.
2560  *
2561  * CONTEXT:
2562  * mutex_lock(wq->mutex).
2563  *
2564  * RETURNS:
2565  * %true if @flush_color >= 0 and there's something to flush.  %false
2566  * otherwise.
2567  */
2568 static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2569                                       int flush_color, int work_color)
2570 {
2571         bool wait = false;
2572         struct pool_workqueue *pwq;
2573 
2574         if (flush_color >= 0) {
2575                 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2576                 atomic_set(&wq->nr_pwqs_to_flush, 1);
2577         }
2578 
2579         for_each_pwq(pwq, wq) {
2580                 struct worker_pool *pool = pwq->pool;
2581 
2582                 spin_lock_irq(&pool->lock);
2583 
2584                 if (flush_color >= 0) {
2585                         WARN_ON_ONCE(pwq->flush_color != -1);
2586 
2587                         if (pwq->nr_in_flight[flush_color]) {
2588                                 pwq->flush_color = flush_color;
2589                                 atomic_inc(&wq->nr_pwqs_to_flush);
2590                                 wait = true;
2591                         }
2592                 }
2593 
2594                 if (work_color >= 0) {
2595                         WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2596                         pwq->work_color = work_color;
2597                 }
2598 
2599                 spin_unlock_irq(&pool->lock);
2600         }
2601 
2602         if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2603                 complete(&wq->first_flusher->done);
2604 
2605         return wait;
2606 }
2607 
2608 /**
2609  * flush_workqueue - ensure that any scheduled work has run to completion.
2610  * @wq: workqueue to flush
2611  *
2612  * This function sleeps until all work items which were queued on entry
2613  * have finished execution, but it is not livelocked by new incoming ones.
2614  */
2615 void flush_workqueue(struct workqueue_struct *wq)
2616 {
2617         struct wq_flusher this_flusher = {
2618                 .list = LIST_HEAD_INIT(this_flusher.list),
2619                 .flush_color = -1,
2620                 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2621         };
2622         int next_color;
2623 
2624         lock_map_acquire(&wq->lockdep_map);
2625         lock_map_release(&wq->lockdep_map);
2626 
2627         mutex_lock(&wq->mutex);
2628 
2629         /*
2630          * Start-to-wait phase
2631          */
2632         next_color = work_next_color(wq->work_color);
2633 
2634         if (next_color != wq->flush_color) {
2635                 /*
2636                  * Color space is not full.  The current work_color
2637                  * becomes our flush_color and work_color is advanced
2638                  * by one.
2639                  */
2640                 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2641                 this_flusher.flush_color = wq->work_color;
2642                 wq->work_color = next_color;
2643 
2644                 if (!wq->first_flusher) {
2645                         /* no flush in progress, become the first flusher */
2646                         WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2647 
2648                         wq->first_flusher = &this_flusher;
2649 
2650                         if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2651                                                        wq->work_color)) {
2652                                 /* nothing to flush, done */
2653                                 wq->flush_color = next_color;
2654                                 wq->first_flusher = NULL;
2655                                 goto out_unlock;
2656                         }
2657                 } else {
2658                         /* wait in queue */
2659                         WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2660                         list_add_tail(&this_flusher.list, &wq->flusher_queue);
2661                         flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2662                 }
2663         } else {
2664                 /*
2665                  * Oops, color space is full, wait on overflow queue.
2666                  * The next flush completion will assign us
2667                  * flush_color and transfer to flusher_queue.
2668                  */
2669                 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2670         }
2671 
2672         mutex_unlock(&wq->mutex);
2673 
2674         wait_for_completion(&this_flusher.done);
2675 
2676         /*
2677          * Wake-up-and-cascade phase
2678          *
2679          * First flushers are responsible for cascading flushes and
2680          * handling overflow.  Non-first flushers can simply return.
2681          */
2682         if (wq->first_flusher != &this_flusher)
2683                 return;
2684 
2685         mutex_lock(&wq->mutex);
2686 
2687         /* we might have raced, check again with mutex held */
2688         if (wq->first_flusher != &this_flusher)
2689                 goto out_unlock;
2690 
2691         wq->first_flusher = NULL;
2692 
2693         WARN_ON_ONCE(!list_empty(&this_flusher.list));
2694         WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2695 
2696         while (true) {
2697                 struct wq_flusher *next, *tmp;
2698 
2699                 /* complete all the flushers sharing the current flush color */
2700                 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2701                         if (next->flush_color != wq->flush_color)
2702                                 break;
2703                         list_del_init(&next->list);
2704                         complete(&next->done);
2705                 }
2706 
2707                 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2708                              wq->flush_color != work_next_color(wq->work_color));
2709 
2710                 /* this flush_color is finished, advance by one */
2711                 wq->flush_color = work_next_color(wq->flush_color);
2712 
2713                 /* one color has been freed, handle overflow queue */
2714                 if (!list_empty(&wq->flusher_overflow)) {
2715                         /*
2716                          * Assign the same color to all overflowed
2717                          * flushers, advance work_color and append to
2718                          * flusher_queue.  This is the start-to-wait
2719                          * phase for these overflowed flushers.
2720                          */
2721                         list_for_each_entry(tmp, &wq->flusher_overflow, list)
2722                                 tmp->flush_color = wq->work_color;
2723 
2724                         wq->work_color = work_next_color(wq->work_color);
2725 
2726                         list_splice_tail_init(&wq->flusher_overflow,
2727                                               &wq->flusher_queue);
2728                         flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2729                 }
2730 
2731                 if (list_empty(&wq->flusher_queue)) {
2732                         WARN_ON_ONCE(wq->flush_color != wq->work_color);
2733                         break;
2734                 }
2735 
2736                 /*
2737                  * Need to flush more colors.  Make the next flusher
2738                  * the new first flusher and arm pwqs.
2739                  */
2740                 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2741                 WARN_ON_ONCE(wq->flush_color != next->flush_color);
2742 
2743                 list_del_init(&next->list);
2744                 wq->first_flusher = next;
2745 
2746                 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2747                         break;
2748 
2749                 /*
2750                  * Meh... this color is already done, clear first
2751                  * flusher and repeat cascading.
2752                  */
2753                 wq->first_flusher = NULL;
2754         }
2755 
2756 out_unlock:
2757         mutex_unlock(&wq->mutex);
2758 }
2759 EXPORT_SYMBOL_GPL(flush_workqueue);
2760 
2761 /**
2762  * drain_workqueue - drain a workqueue
2763  * @wq: workqueue to drain
2764  *
2765  * Wait until the workqueue becomes empty.  While draining is in progress,
2766  * only chain queueing is allowed.  IOW, only currently pending or running
2767  * work items on @wq can queue further work items on it.  @wq is flushed
2768  * repeatedly until it becomes empty.  The number of flushing is detemined
2769  * by the depth of chaining and should be relatively short.  Whine if it
2770  * takes too long.
2771  */
2772 void drain_workqueue(struct workqueue_struct *wq)
2773 {
2774         unsigned int flush_cnt = 0;
2775         struct pool_workqueue *pwq;
2776 
2777         /*
2778          * __queue_work() needs to test whether there are drainers, is much
2779          * hotter than drain_workqueue() and already looks at @wq->flags.
2780          * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2781          */
2782         mutex_lock(&wq->mutex);
2783         if (!wq->nr_drainers++)
2784                 wq->flags |= __WQ_DRAINING;
2785         mutex_unlock(&wq->mutex);
2786 reflush:
2787         flush_workqueue(wq);
2788 
2789         mutex_lock(&wq->mutex);
2790 
2791         for_each_pwq(pwq, wq) {
2792                 bool drained;
2793 
2794                 spin_lock_irq(&pwq->pool->lock);
2795                 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2796                 spin_unlock_irq(&pwq->pool->lock);
2797 
2798                 if (drained)
2799                         continue;
2800 
2801                 if (++flush_cnt == 10 ||
2802                     (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2803                         pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2804                                 wq->name, flush_cnt);
2805 
2806                 mutex_unlock(&wq->mutex);
2807                 goto reflush;
2808         }
2809 
2810         if (!--wq->nr_drainers)
2811                 wq->flags &= ~__WQ_DRAINING;
2812         mutex_unlock(&wq->mutex);
2813 }
2814 EXPORT_SYMBOL_GPL(drain_workqueue);
2815 
2816 static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2817 {
2818         struct worker *worker = NULL;
2819         struct worker_pool *pool;
2820         struct pool_workqueue *pwq;
2821 
2822         might_sleep();
2823 
2824         local_irq_disable();
2825         pool = get_work_pool(work);
2826         if (!pool) {
2827                 local_irq_enable();
2828                 return false;
2829         }
2830 
2831         spin_lock(&pool->lock);
2832         /* see the comment in try_to_grab_pending() with the same code */
2833         pwq = get_work_pwq(work);
2834         if (pwq) {
2835                 if (unlikely(pwq->pool != pool))
2836                         goto already_gone;
2837         } else {
2838                 worker = find_worker_executing_work(pool, work);
2839                 if (!worker)
2840                         goto already_gone;
2841                 pwq = worker->current_pwq;
2842         }
2843 
2844         insert_wq_barrier(pwq, barr, work, worker);
2845         spin_unlock_irq(&pool->lock);
2846 
2847         /*
2848          * If @max_active is 1 or rescuer is in use, flushing another work
2849          * item on the same workqueue may lead to deadlock.  Make sure the
2850          * flusher is not running on the same workqueue by verifying write
2851          * access.
2852          */
2853         if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
2854                 lock_map_acquire(&pwq->wq->lockdep_map);
2855         else
2856                 lock_map_acquire_read(&pwq->wq->lockdep_map);
2857         lock_map_release(&pwq->wq->lockdep_map);
2858 
2859         return true;
2860 already_gone:
2861         spin_unlock_irq(&pool->lock);
2862         return false;
2863 }
2864 
2865 /**
2866  * flush_work - wait for a work to finish executing the last queueing instance
2867  * @work: the work to flush
2868  *
2869  * Wait until @work has finished execution.  @work is guaranteed to be idle
2870  * on return if it hasn't been requeued since flush started.
2871  *
2872  * RETURNS:
2873  * %true if flush_work() waited for the work to finish execution,
2874  * %false if it was already idle.
2875  */
2876 bool flush_work(struct work_struct *work)
2877 {
2878         struct wq_barrier barr;
2879 
2880         lock_map_acquire(&work->lockdep_map);
2881         lock_map_release(&work->lockdep_map);
2882 
2883         if (start_flush_work(work, &barr)) {
2884                 wait_for_completion(&barr.done);
2885                 destroy_work_on_stack(&barr.work);
2886                 return true;
2887         } else {
2888                 return false;
2889         }
2890 }
2891 EXPORT_SYMBOL_GPL(flush_work);
2892 
2893 struct cwt_wait {
2894         wait_queue_t            wait;
2895         struct work_struct      *work;
2896 };
2897 
2898 static int cwt_wakefn(wait_queue_t *wait, unsigned mode, int sync, void *key)
2899 {
2900         struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
2901 
2902         if (cwait->work != key)
2903                 return 0;
2904         return autoremove_wake_function(wait, mode, sync, key);
2905 }
2906 
2907 static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2908 {
2909         static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
2910         unsigned long flags;
2911         int ret;
2912 
2913         do {
2914                 ret = try_to_grab_pending(work, is_dwork, &flags);
2915                 /*
2916                  * If someone else is already canceling, wait for it to
2917                  * finish.  flush_work() doesn't work for PREEMPT_NONE
2918                  * because we may get scheduled between @work's completion
2919                  * and the other canceling task resuming and clearing
2920                  * CANCELING - flush_work() will return false immediately
2921                  * as @work is no longer busy, try_to_grab_pending() will
2922                  * return -ENOENT as @work is still being canceled and the
2923                  * other canceling task won't be able to clear CANCELING as
2924                  * we're hogging the CPU.
2925                  *
2926                  * Let's wait for completion using a waitqueue.  As this
2927                  * may lead to the thundering herd problem, use a custom
2928                  * wake function which matches @work along with exclusive
2929                  * wait and wakeup.
2930                  */
2931                 if (unlikely(ret == -ENOENT)) {
2932                         struct cwt_wait cwait;
2933 
2934                         init_wait(&cwait.wait);
2935                         cwait.wait.func = cwt_wakefn;
2936                         cwait.work = work;
2937 
2938                         prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
2939                                                   TASK_UNINTERRUPTIBLE);
2940                         if (work_is_canceling(work))
2941                                 schedule();
2942                         finish_wait(&cancel_waitq, &cwait.wait);
2943                 }
2944         } while (unlikely(ret < 0));
2945 
2946         /* tell other tasks trying to grab @work to back off */
2947         mark_work_canceling(work);
2948         local_irq_restore(flags);
2949 
2950         flush_work(work);
2951         clear_work_data(work);
2952 
2953         /*
2954          * Paired with prepare_to_wait() above so that either
2955          * waitqueue_active() is visible here or !work_is_canceling() is
2956          * visible there.
2957          */
2958         smp_mb();
2959         if (waitqueue_active(&cancel_waitq))
2960                 __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
2961 
2962         return ret;
2963 }
2964 
2965 /**
2966  * cancel_work_sync - cancel a work and wait for it to finish
2967  * @work: the work to cancel
2968  *
2969  * Cancel @work and wait for its execution to finish.  This function
2970  * can be used even if the work re-queues itself or migrates to
2971  * another workqueue.  On return from this function, @work is
2972  * guaranteed to be not pending or executing on any CPU.
2973  *
2974  * cancel_work_sync(&delayed_work->work) must not be used for
2975  * delayed_work's.  Use cancel_delayed_work_sync() instead.
2976  *
2977  * The caller must ensure that the workqueue on which @work was last
2978  * queued can't be destroyed before this function returns.
2979  *
2980  * RETURNS:
2981  * %true if @work was pending, %false otherwise.
2982  */
2983 bool cancel_work_sync(struct work_struct *work)
2984 {
2985         return __cancel_work_timer(work, false);
2986 }
2987 EXPORT_SYMBOL_GPL(cancel_work_sync);
2988 
2989 /**
2990  * flush_delayed_work - wait for a dwork to finish executing the last queueing
2991  * @dwork: the delayed work to flush
2992  *
2993  * Delayed timer is cancelled and the pending work is queued for
2994  * immediate execution.  Like flush_work(), this function only
2995  * considers the last queueing instance of @dwork.
2996  *
2997  * RETURNS:
2998  * %true if flush_work() waited for the work to finish execution,
2999  * %false if it was already idle.
3000  */
3001 bool flush_delayed_work(struct delayed_work *dwork)
3002 {
3003         local_irq_disable();
3004         if (del_timer_sync(&dwork->timer))
3005                 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
3006         local_irq_enable();
3007         return flush_work(&dwork->work);
3008 }
3009 EXPORT_SYMBOL(flush_delayed_work);
3010 
3011 /**
3012  * cancel_delayed_work - cancel a delayed work
3013  * @dwork: delayed_work to cancel
3014  *
3015  * Kill off a pending delayed_work.  Returns %true if @dwork was pending
3016  * and canceled; %false if wasn't pending.  Note that the work callback
3017  * function may still be running on return, unless it returns %true and the
3018  * work doesn't re-arm itself.  Explicitly flush or use
3019  * cancel_delayed_work_sync() to wait on it.
3020  *
3021  * This function is safe to call from any context including IRQ handler.
3022  */
3023 bool cancel_delayed_work(struct delayed_work *dwork)
3024 {
3025         unsigned long flags;
3026         int ret;
3027 
3028         do {
3029                 ret = try_to_grab_pending(&dwork->work, true, &flags);
3030         } while (unlikely(ret == -EAGAIN));
3031 
3032         if (unlikely(ret < 0))
3033                 return false;
3034 
3035         set_work_pool_and_clear_pending(&dwork->work,
3036                                         get_work_pool_id(&dwork->work));
3037         local_irq_restore(flags);
3038         return ret;
3039 }
3040 EXPORT_SYMBOL(cancel_delayed_work);
3041 
3042 /**
3043  * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3044  * @dwork: the delayed work cancel
3045  *
3046  * This is cancel_work_sync() for delayed works.
3047  *
3048  * RETURNS:
3049  * %true if @dwork was pending, %false otherwise.
3050  */
3051 bool cancel_delayed_work_sync(struct delayed_work *dwork)
3052 {
3053         return __cancel_work_timer(&dwork->work, true);
3054 }
3055 EXPORT_SYMBOL(cancel_delayed_work_sync);
3056 
3057 /**
3058  * schedule_on_each_cpu - execute a function synchronously on each online CPU
3059  * @func: the function to call
3060  *
3061  * schedule_on_each_cpu() executes @func on each online CPU using the
3062  * system workqueue and blocks until all CPUs have completed.
3063  * schedule_on_each_cpu() is very slow.
3064  *
3065  * RETURNS:
3066  * 0 on success, -errno on failure.
3067  */
3068 int schedule_on_each_cpu(work_func_t func)
3069 {
3070         int cpu;
3071         struct work_struct __percpu *works;
3072 
3073         works = alloc_percpu(struct work_struct);
3074         if (!works)
3075                 return -ENOMEM;
3076 
3077         get_online_cpus();
3078 
3079         for_each_online_cpu(cpu) {
3080                 struct work_struct *work = per_cpu_ptr(works, cpu);
3081 
3082                 INIT_WORK(work, func);
3083                 schedule_work_on(cpu, work);
3084         }
3085 
3086         for_each_online_cpu(cpu)
3087                 flush_work(per_cpu_ptr(works, cpu));
3088 
3089         put_online_cpus();
3090         free_percpu(works);
3091         return 0;
3092 }
3093 
3094 /**
3095  * flush_scheduled_work - ensure that any scheduled work has run to completion.
3096  *
3097  * Forces execution of the kernel-global workqueue and blocks until its
3098  * completion.
3099  *
3100  * Think twice before calling this function!  It's very easy to get into
3101  * trouble if you don't take great care.  Either of the following situations
3102  * will lead to deadlock:
3103  *
3104  *      One of the work items currently on the workqueue needs to acquire
3105  *      a lock held by your code or its caller.
3106  *
3107  *      Your code is running in the context of a work routine.
3108  *
3109  * They will be detected by lockdep when they occur, but the first might not
3110  * occur very often.  It depends on what work items are on the workqueue and
3111  * what locks they need, which you have no control over.
3112  *
3113  * In most situations flushing the entire workqueue is overkill; you merely
3114  * need to know that a particular work item isn't queued and isn't running.
3115  * In such cases you should use cancel_delayed_work_sync() or
3116  * cancel_work_sync() instead.
3117  */
3118 void flush_scheduled_work(void)
3119 {
3120         flush_workqueue(system_wq);
3121 }
3122 EXPORT_SYMBOL(flush_scheduled_work);
3123 
3124 /**
3125  * execute_in_process_context - reliably execute the routine with user context
3126  * @fn:         the function to execute
3127  * @ew:         guaranteed storage for the execute work structure (must
3128  *              be available when the work executes)
3129  *
3130  * Executes the function immediately if process context is available,
3131  * otherwise schedules the function for delayed execution.
3132  *
3133  * Returns:     0 - function was executed
3134  *              1 - function was scheduled for execution
3135  */
3136 int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3137 {
3138         if (!in_interrupt()) {
3139                 fn(&ew->work);
3140                 return 0;
3141         }
3142 
3143         INIT_WORK(&ew->work, fn);
3144         schedule_work(&ew->work);
3145 
3146         return 1;
3147 }
3148 EXPORT_SYMBOL_GPL(execute_in_process_context);
3149 
3150 #ifdef CONFIG_SYSFS
3151 /*
3152  * Workqueues with WQ_SYSFS flag set is visible to userland via
3153  * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
3154  * following attributes.
3155  *
3156  *  per_cpu     RO bool : whether the workqueue is per-cpu or unbound
3157  *  max_active  RW int  : maximum number of in-flight work items
3158  *
3159  * Unbound workqueues have the following extra attributes.
3160  *
3161  *  id          RO int  : the associated pool ID
3162  *  nice        RW int  : nice value of the workers
3163  *  cpumask     RW mask : bitmask of allowed CPUs for the workers
3164  */
3165 struct wq_device {
3166         struct workqueue_struct         *wq;
3167         struct device                   dev;
3168 };
3169 
3170 static struct workqueue_struct *dev_to_wq(struct device *dev)
3171 {
3172         struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3173 
3174         return wq_dev->wq;
3175 }
3176 
3177 static ssize_t wq_per_cpu_show(struct device *dev,
3178                                struct device_attribute *attr, char *buf)
3179 {
3180         struct workqueue_struct *wq = dev_to_wq(dev);
3181 
3182         return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3183 }
3184 
3185 static ssize_t wq_max_active_show(struct device *dev,
3186                                   struct device_attribute *attr, char *buf)
3187 {
3188         struct workqueue_struct *wq = dev_to_wq(dev);
3189 
3190         return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3191 }
3192 
3193 static ssize_t wq_max_active_store(struct device *dev,
3194                                    struct device_attribute *attr,
3195                                    const char *buf, size_t count)
3196 {
3197         struct workqueue_struct *wq = dev_to_wq(dev);
3198         int val;
3199 
3200         if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3201                 return -EINVAL;
3202 
3203         workqueue_set_max_active(wq, val);
3204         return count;
3205 }
3206 
3207 static struct device_attribute wq_sysfs_attrs[] = {
3208         __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
3209         __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
3210         __ATTR_NULL,
3211 };
3212 
3213 static ssize_t wq_pool_ids_show(struct device *dev,
3214                                 struct device_attribute *attr, char *buf)
3215 {
3216         struct workqueue_struct *wq = dev_to_wq(dev);
3217         const char *delim = "";
3218         int node, written = 0;
3219 
3220         rcu_read_lock_sched();
3221         for_each_node(node) {
3222                 written += scnprintf(buf + written, PAGE_SIZE - written,
3223                                      "%s%d:%d", delim, node,
3224                                      unbound_pwq_by_node(wq, node)->pool->id);
3225                 delim = " ";
3226         }
3227         written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3228         rcu_read_unlock_sched();
3229 
3230         return written;
3231 }
3232 
3233 static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3234                             char *buf)
3235 {
3236         struct workqueue_struct *wq = dev_to_wq(dev);
3237         int written;
3238 
3239         mutex_lock(&wq->mutex);
3240         written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3241         mutex_unlock(&wq->mutex);
3242 
3243         return written;
3244 }
3245 
3246 /* prepare workqueue_attrs for sysfs store operations */
3247 static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3248 {
3249         struct workqueue_attrs *attrs;
3250 
3251         attrs = alloc_workqueue_attrs(GFP_KERNEL);
3252         if (!attrs)
3253                 return NULL;
3254 
3255         mutex_lock(&wq->mutex);
3256         copy_workqueue_attrs(attrs, wq->unbound_attrs);
3257         mutex_unlock(&wq->mutex);
3258         return attrs;
3259 }
3260 
3261 static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3262                              const char *buf, size_t count)
3263 {
3264         struct workqueue_struct *wq = dev_to_wq(dev);
3265         struct workqueue_attrs *attrs;
3266         int ret;
3267 
3268         attrs = wq_sysfs_prep_attrs(wq);
3269         if (!attrs)
3270                 return -ENOMEM;
3271 
3272         if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3273             attrs->nice >= -20 && attrs->nice <= 19)
3274                 ret = apply_workqueue_attrs(wq, attrs);
3275         else
3276                 ret = -EINVAL;
3277 
3278         free_workqueue_attrs(attrs);
3279         return ret ?: count;
3280 }
3281 
3282 static ssize_t wq_cpumask_show(struct device *dev,
3283                                struct device_attribute *attr, char *buf)
3284 {
3285         struct workqueue_struct *wq = dev_to_wq(dev);
3286         int written;
3287 
3288         mutex_lock(&wq->mutex);
3289         written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3290         mutex_unlock(&wq->mutex);
3291 
3292         written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3293         return written;
3294 }
3295 
3296 static ssize_t wq_cpumask_store(struct device *dev,
3297                                 struct device_attribute *attr,
3298                                 const char *buf, size_t count)
3299 {
3300         struct workqueue_struct *wq = dev_to_wq(dev);
3301         struct workqueue_attrs *attrs;
3302         int ret;
3303 
3304         attrs = wq_sysfs_prep_attrs(wq);
3305         if (!attrs)
3306                 return -ENOMEM;
3307 
3308         ret = cpumask_parse(buf, attrs->cpumask);
3309         if (!ret)
3310                 ret = apply_workqueue_attrs(wq, attrs);
3311 
3312         free_workqueue_attrs(attrs);
3313         return ret ?: count;
3314 }
3315 
3316 static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
3317                             char *buf)
3318 {
3319         struct workqueue_struct *wq = dev_to_wq(dev);
3320         int written;
3321 
3322         mutex_lock(&wq->mutex);
3323         written = scnprintf(buf, PAGE_SIZE, "%d\n",
3324                             !wq->unbound_attrs->no_numa);
3325         mutex_unlock(&wq->mutex);
3326 
3327         return written;
3328 }
3329 
3330 static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
3331                              const char *buf, size_t count)
3332 {
3333         struct workqueue_struct *wq = dev_to_wq(dev);
3334         struct workqueue_attrs *attrs;
3335         int v, ret;
3336 
3337         attrs = wq_sysfs_prep_attrs(wq);
3338         if (!attrs)
3339                 return -ENOMEM;
3340 
3341         ret = -EINVAL;
3342         if (sscanf(buf, "%d", &v) == 1) {
3343                 attrs->no_numa = !v;
3344                 ret = apply_workqueue_attrs(wq, attrs);
3345         }
3346 
3347         free_workqueue_attrs(attrs);
3348         return ret ?: count;
3349 }
3350 
3351 static struct device_attribute wq_sysfs_unbound_attrs[] = {
3352         __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3353         __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3354         __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3355         __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3356         __ATTR_NULL,
3357 };
3358 
3359 static struct bus_type wq_subsys = {
3360         .name                           = "workqueue",
3361         .dev_attrs                      = wq_sysfs_attrs,
3362 };
3363 
3364 static int __init wq_sysfs_init(void)
3365 {
3366         return subsys_virtual_register(&wq_subsys, NULL);
3367 }
3368 core_initcall(wq_sysfs_init);
3369 
3370 static void wq_device_release(struct device *dev)
3371 {
3372         struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3373 
3374         kfree(wq_dev);
3375 }
3376 
3377 /**
3378  * workqueue_sysfs_register - make a workqueue visible in sysfs
3379  * @wq: the workqueue to register
3380  *
3381  * Expose @wq in sysfs under /sys/bus/workqueue/devices.
3382  * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
3383  * which is the preferred method.
3384  *
3385  * Workqueue user should use this function directly iff it wants to apply
3386  * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
3387  * apply_workqueue_attrs() may race against userland updating the
3388  * attributes.
3389  *
3390  * Returns 0 on success, -errno on failure.
3391  */
3392 int workqueue_sysfs_register(struct workqueue_struct *wq)
3393 {
3394         struct wq_device *wq_dev;
3395         int ret;
3396 
3397         /*
3398          * Adjusting max_active or creating new pwqs by applyting
3399          * attributes breaks ordering guarantee.  Disallow exposing ordered
3400          * workqueues.
3401          */
3402         if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
3403                 return -EINVAL;
3404 
3405         wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3406         if (!wq_dev)
3407                 return -ENOMEM;
3408 
3409         wq_dev->wq = wq;
3410         wq_dev->dev.bus = &wq_subsys;
3411         wq_dev->dev.init_name = wq->name;
3412         wq_dev->dev.release = wq_device_release;
3413 
3414         /*
3415          * unbound_attrs are created separately.  Suppress uevent until
3416          * everything is ready.
3417          */
3418         dev_set_uevent_suppress(&wq_dev->dev, true);
3419 
3420         ret = device_register(&wq_dev->dev);
3421         if (ret) {
3422                 kfree(wq_dev);
3423                 wq->wq_dev = NULL;
3424                 return ret;
3425         }
3426 
3427         if (wq->flags & WQ_UNBOUND) {
3428                 struct device_attribute *attr;
3429 
3430                 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3431                         ret = device_create_file(&wq_dev->dev, attr);
3432                         if (ret) {
3433                                 device_unregister(&wq_dev->dev);
3434                                 wq->wq_dev = NULL;
3435                                 return ret;
3436                         }
3437                 }
3438         }
3439 
3440         dev_set_uevent_suppress(&wq_dev->dev, false);
3441         kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3442         return 0;
3443 }
3444 
3445 /**
3446  * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
3447  * @wq: the workqueue to unregister
3448  *
3449  * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
3450  */
3451 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3452 {
3453         struct wq_device *wq_dev = wq->wq_dev;
3454 
3455         if (!wq->wq_dev)
3456                 return;
3457 
3458         wq->wq_dev = NULL;
3459         device_unregister(&wq_dev->dev);
3460 }
3461 #else   /* CONFIG_SYSFS */
3462 static void workqueue_sysfs_unregister(struct workqueue_struct *wq)     { }
3463 #endif  /* CONFIG_SYSFS */
3464 
3465 /**
3466  * free_workqueue_attrs - free a workqueue_attrs
3467  * @attrs: workqueue_attrs to free
3468  *
3469  * Undo alloc_workqueue_attrs().
3470  */
3471 void free_workqueue_attrs(struct workqueue_attrs *attrs)
3472 {
3473         if (attrs) {
3474                 free_cpumask_var(attrs->cpumask);
3475                 kfree(attrs);
3476         }
3477 }
3478 
3479 /**
3480  * alloc_workqueue_attrs - allocate a workqueue_attrs
3481  * @gfp_mask: allocation mask to use
3482  *
3483  * Allocate a new workqueue_attrs, initialize with default settings and
3484  * return it.  Returns NULL on failure.
3485  */
3486 struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3487 {
3488         struct workqueue_attrs *attrs;
3489 
3490         attrs = kzalloc(sizeof(*attrs), gfp_mask);
3491         if (!attrs)
3492                 goto fail;
3493         if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3494                 goto fail;
3495 
3496         cpumask_copy(attrs->cpumask, cpu_possible_mask);
3497         return attrs;
3498 fail:
3499         free_workqueue_attrs(attrs);
3500         return NULL;
3501 }
3502 
3503 static void copy_workqueue_attrs(struct workqueue_attrs *to,
3504                                  const struct workqueue_attrs *from)
3505 {
3506         to->nice = from->nice;
3507         cpumask_copy(to->cpumask, from->cpumask);
3508         /*
3509          * Unlike hash and equality test, this function doesn't ignore
3510          * ->no_numa as it is used for both pool and wq attrs.  Instead,
3511          * get_unbound_pool() explicitly clears ->no_numa after copying.
3512          */
3513         to->no_numa = from->no_numa;
3514 }
3515 
3516 /* hash value of the content of @attr */
3517 static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3518 {
3519         u32 hash = 0;
3520 
3521         hash = jhash_1word(attrs->nice, hash);
3522         hash = jhash(cpumask_bits(attrs->cpumask),
3523                      BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3524         return hash;
3525 }
3526 
3527 /* content equality test */
3528 static bool wqattrs_equal(const struct workqueue_attrs *a,
3529                           const struct workqueue_attrs *b)
3530 {
3531         if (a->nice != b->nice)
3532                 return false;
3533         if (!cpumask_equal(a->cpumask, b->cpumask))
3534                 return false;
3535         return true;
3536 }
3537 
3538 /**
3539  * init_worker_pool - initialize a newly zalloc'd worker_pool
3540  * @pool: worker_pool to initialize
3541  *
3542  * Initiailize a newly zalloc'd @pool.  It also allocates @pool->attrs.
3543  * Returns 0 on success, -errno on failure.  Even on failure, all fields
3544  * inside @pool proper are initialized and put_unbound_pool() can be called
3545  * on @pool safely to release it.
3546  */
3547 static int init_worker_pool(struct worker_pool *pool)
3548 {
3549         spin_lock_init(&pool->lock);
3550         pool->id = -1;
3551         pool->cpu = -1;
3552         pool->node = NUMA_NO_NODE;
3553         pool->flags |= POOL_DISASSOCIATED;
3554         INIT_LIST_HEAD(&pool->worklist);
3555         INIT_LIST_HEAD(&pool->idle_list);
3556         hash_init(pool->busy_hash);
3557 
3558         init_timer_deferrable(&pool->idle_timer);
3559         pool->idle_timer.function = idle_worker_timeout;
3560         pool->idle_timer.data = (unsigned long)pool;
3561 
3562         setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3563                     (unsigned long)pool);
3564 
3565         mutex_init(&pool->manager_arb);
3566         mutex_init(&pool->manager_mutex);
3567         idr_init(&pool->worker_idr);
3568 
3569         INIT_HLIST_NODE(&pool->hash_node);
3570         pool->refcnt = 1;
3571 
3572         /* shouldn't fail above this point */
3573         pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3574         if (!pool->attrs)
3575                 return -ENOMEM;
3576         return 0;
3577 }
3578 
3579 static void rcu_free_pool(struct rcu_head *rcu)
3580 {
3581         struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3582 
3583         idr_destroy(&pool->worker_idr);
3584         free_workqueue_attrs(pool->attrs);
3585         kfree(pool);
3586 }
3587 
3588 /**
3589  * put_unbound_pool - put a worker_pool
3590  * @pool: worker_pool to put
3591  *
3592  * Put @pool.  If its refcnt reaches zero, it gets destroyed in sched-RCU
3593  * safe manner.  get_unbound_pool() calls this function on its failure path
3594  * and this function should be able to release pools which went through,
3595  * successfully or not, init_worker_pool().
3596  *
3597  * Should be called with wq_pool_mutex held.
3598  */
3599 static void put_unbound_pool(struct worker_pool *pool)
3600 {
3601         struct worker *worker;
3602 
3603         lockdep_assert_held(&wq_pool_mutex);
3604 
3605         if (--pool->refcnt)
3606                 return;
3607 
3608         /* sanity checks */
3609         if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3610             WARN_ON(!list_empty(&pool->worklist)))
3611                 return;
3612 
3613         /* release id and unhash */
3614         if (pool->id >= 0)
3615                 idr_remove(&worker_pool_idr, pool->id);
3616         hash_del(&pool->hash_node);
3617 
3618         /*
3619          * Become the manager and destroy all workers.  Grabbing
3620          * manager_arb prevents @pool's workers from blocking on
3621          * manager_mutex.
3622          */
3623         mutex_lock(&pool->manager_arb);
3624         mutex_lock(&pool->manager_mutex);
3625         spin_lock_irq(&pool->lock);
3626 
3627         while ((worker = first_worker(pool)))
3628                 destroy_worker(worker);
3629         WARN_ON(pool->nr_workers || pool->nr_idle);
3630 
3631         spin_unlock_irq(&pool->lock);
3632         mutex_unlock(&pool->manager_mutex);
3633         mutex_unlock(&pool->manager_arb);
3634 
3635         /* shut down the timers */
3636         del_timer_sync(&pool->idle_timer);
3637         del_timer_sync(&pool->mayday_timer);
3638 
3639         /* sched-RCU protected to allow dereferences from get_work_pool() */
3640         call_rcu_sched(&pool->rcu, rcu_free_pool);
3641 }
3642 
3643 /**
3644  * get_unbound_pool - get a worker_pool with the specified attributes
3645  * @attrs: the attributes of the worker_pool to get
3646  *
3647  * Obtain a worker_pool which has the same attributes as @attrs, bump the
3648  * reference count and return it.  If there already is a matching
3649  * worker_pool, it will be used; otherwise, this function attempts to
3650  * create a new one.  On failure, returns NULL.
3651  *
3652  * Should be called with wq_pool_mutex held.
3653  */
3654 static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3655 {
3656         u32 hash = wqattrs_hash(attrs);
3657         struct worker_pool *pool;
3658         int node;
3659 
3660         lockdep_assert_held(&wq_pool_mutex);
3661 
3662         /* do we already have a matching pool? */
3663         hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3664                 if (wqattrs_equal(pool->attrs, attrs)) {
3665                         pool->refcnt++;
3666                         goto out_unlock;
3667                 }
3668         }
3669 
3670         /* nope, create a new one */
3671         pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3672         if (!pool || init_worker_pool(pool) < 0)
3673                 goto fail;
3674 
3675         if (workqueue_freezing)
3676                 pool->flags |= POOL_FREEZING;
3677 
3678         lockdep_set_subclass(&pool->lock, 1);   /* see put_pwq() */
3679         copy_workqueue_attrs(pool->attrs, attrs);
3680 
3681         /*
3682          * no_numa isn't a worker_pool attribute, always clear it.  See
3683          * 'struct workqueue_attrs' comments for detail.
3684          */
3685         pool->attrs->no_numa = false;
3686 
3687         /* if cpumask is contained inside a NUMA node, we belong to that node */
3688         if (wq_numa_enabled) {
3689                 for_each_node(node) {
3690                         if (cpumask_subset(pool->attrs->cpumask,
3691                                            wq_numa_possible_cpumask[node])) {
3692                                 pool->node = node;
3693                                 break;
3694                         }
3695                 }
3696         }
3697 
3698         if (worker_pool_assign_id(pool) < 0)
3699                 goto fail;
3700 
3701         /* create and start the initial worker */
3702         if (create_and_start_worker(pool) < 0)
3703                 goto fail;
3704 
3705         /* install */
3706         hash_add(unbound_pool_hash, &pool->hash_node, hash);
3707 out_unlock:
3708         return pool;
3709 fail:
3710         if (pool)
3711                 put_unbound_pool(pool);
3712         return NULL;
3713 }
3714 
3715 static void rcu_free_pwq(struct rcu_head *rcu)
3716 {
3717         kmem_cache_free(pwq_cache,
3718                         container_of(rcu, struct pool_workqueue, rcu));
3719 }
3720 
3721 /*
3722  * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3723  * and needs to be destroyed.
3724  */
3725 static void pwq_unbound_release_workfn(struct work_struct *work)
3726 {
3727         struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3728                                                   unbound_release_work);
3729         struct workqueue_struct *wq = pwq->wq;
3730         struct worker_pool *pool = pwq->pool;
3731         bool is_last;
3732 
3733         if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3734                 return;
3735 
3736         /*
3737          * Unlink @pwq.  Synchronization against wq->mutex isn't strictly
3738          * necessary on release but do it anyway.  It's easier to verify
3739          * and consistent with the linking path.
3740          */
3741         mutex_lock(&wq->mutex);
3742         list_del_rcu(&pwq->pwqs_node);
3743         is_last = list_empty(&wq->pwqs);
3744         mutex_unlock(&wq->mutex);
3745 
3746         mutex_lock(&wq_pool_mutex);
3747         put_unbound_pool(pool);
3748         mutex_unlock(&wq_pool_mutex);
3749 
3750         call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3751 
3752         /*
3753          * If we're the last pwq going away, @wq is already dead and no one
3754          * is gonna access it anymore.  Free it.
3755          */
3756         if (is_last) {
3757                 free_workqueue_attrs(wq->unbound_attrs);
3758                 kfree(wq);
3759         }
3760 }
3761 
3762 /**
3763  * pwq_adjust_max_active - update a pwq's max_active to the current setting
3764  * @pwq: target pool_workqueue
3765  *
3766  * If @pwq isn't freezing, set @pwq->max_active to the associated
3767  * workqueue's saved_max_active and activate delayed work items
3768  * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero.
3769  */
3770 static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3771 {
3772         struct workqueue_struct *wq = pwq->wq;
3773         bool freezable = wq->flags & WQ_FREEZABLE;
3774 
3775         /* for @wq->saved_max_active */
3776         lockdep_assert_held(&wq->mutex);
3777 
3778         /* fast exit for non-freezable wqs */
3779         if (!freezable && pwq->max_active == wq->saved_max_active)
3780                 return;
3781 
3782         spin_lock_irq(&pwq->pool->lock);
3783 
3784         if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3785                 pwq->max_active = wq->saved_max_active;
3786 
3787                 while (!list_empty(&pwq->delayed_works) &&
3788                        pwq->nr_active < pwq->max_active)
3789                         pwq_activate_first_delayed(pwq);
3790 
3791                 /*
3792                  * Need to kick a worker after thawed or an unbound wq's
3793                  * max_active is bumped.  It's a slow path.  Do it always.
3794                  */
3795                 wake_up_worker(pwq->pool);
3796         } else {
3797                 pwq->max_active = 0;
3798         }
3799 
3800         spin_unlock_irq(&pwq->pool->lock);
3801 }
3802 
3803 /* initialize newly alloced @pwq which is associated with @wq and @pool */
3804 static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3805                      struct worker_pool *pool)
3806 {
3807         BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3808 
3809         memset(pwq, 0, sizeof(*pwq));
3810 
3811         pwq->pool = pool;
3812         pwq->wq = wq;
3813         pwq->flush_color = -1;
3814         pwq->refcnt = 1;
3815         INIT_LIST_HEAD(&pwq->delayed_works);
3816         INIT_LIST_HEAD(&pwq->pwqs_node);
3817         INIT_LIST_HEAD(&pwq->mayday_node);
3818         INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3819 }
3820 
3821 /* sync @pwq with the current state of its associated wq and link it */
3822 static void link_pwq(struct pool_workqueue *pwq)
3823 {
3824         struct workqueue_struct *wq = pwq->wq;
3825 
3826         lockdep_assert_held(&wq->mutex);
3827 
3828         /* may be called multiple times, ignore if already linked */
3829         if (!list_empty(&pwq->pwqs_node))
3830                 return;
3831 
3832         /*
3833          * Set the matching work_color.  This is synchronized with
3834          * wq->mutex to avoid confusing flush_workqueue().
3835          */
3836         pwq->work_color = wq->work_color;
3837 
3838         /* sync max_active to the current setting */
3839         pwq_adjust_max_active(pwq);
3840 
3841         /* link in @pwq */
3842         list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3843 }
3844 
3845 /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3846 static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3847                                         const struct workqueue_attrs *attrs)
3848 {
3849         struct worker_pool *pool;
3850         struct pool_workqueue *pwq;
3851 
3852         lockdep_assert_held(&wq_pool_mutex);
3853 
3854         pool = get_unbound_pool(attrs);
3855         if (!pool)
3856                 return NULL;
3857 
3858         pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3859         if (!pwq) {
3860                 put_unbound_pool(pool);
3861                 return NULL;
3862         }
3863 
3864         init_pwq(pwq, wq, pool);
3865         return pwq;
3866 }
3867 
3868 /* undo alloc_unbound_pwq(), used only in the error path */
3869 static void free_unbound_pwq(struct pool_workqueue *pwq)
3870 {
3871         lockdep_assert_held(&wq_pool_mutex);
3872 
3873         if (pwq) {
3874                 put_unbound_pool(pwq->pool);
3875                 kmem_cache_free(pwq_cache, pwq);
3876         }
3877 }
3878 
3879 /**
3880  * wq_calc_node_mask - calculate a wq_attrs' cpumask for the specified node
3881  * @attrs: the wq_attrs of interest
3882  * @node: the target NUMA node
3883  * @cpu_going_down: if >= 0, the CPU to consider as offline
3884  * @cpumask: outarg, the resulting cpumask
3885  *
3886  * Calculate the cpumask a workqueue with @attrs should use on @node.  If
3887  * @cpu_going_down is >= 0, that cpu is considered offline during
3888  * calculation.  The result is stored in @cpumask.  This function returns
3889  * %true if the resulting @cpumask is different from @attrs->cpumask,
3890  * %false if equal.
3891  *
3892  * If NUMA affinity is not enabled, @attrs->cpumask is always used.  If
3893  * enabled and @node has online CPUs requested by @attrs, the returned
3894  * cpumask is the intersection of the possible CPUs of @node and
3895  * @attrs->cpumask.
3896  *
3897  * The caller is responsible for ensuring that the cpumask of @node stays
3898  * stable.
3899  */
3900 static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3901                                  int cpu_going_down, cpumask_t *cpumask)
3902 {
3903         if (!wq_numa_enabled || attrs->no_numa)
3904                 goto use_dfl;
3905 
3906         /* does @node have any online CPUs @attrs wants? */
3907         cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3908         if (cpu_going_down >= 0)
3909                 cpumask_clear_cpu(cpu_going_down, cpumask);
3910 
3911         if (cpumask_empty(cpumask))
3912                 goto use_dfl;
3913 
3914         /* yeap, return possible CPUs in @node that @attrs wants */
3915         cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3916         return !cpumask_equal(cpumask, attrs->cpumask);
3917 
3918 use_dfl:
3919         cpumask_copy(cpumask, attrs->cpumask);
3920         return false;
3921 }
3922 
3923 /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3924 static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3925                                                    int node,
3926                                                    struct pool_workqueue *pwq)
3927 {
3928         struct pool_workqueue *old_pwq;
3929 
3930         lockdep_assert_held(&wq->mutex);
3931 
3932         /* link_pwq() can handle duplicate calls */
3933         link_pwq(pwq);
3934 
3935         old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3936         rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3937         return old_pwq;
3938 }
3939 
3940 /**
3941  * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
3942  * @wq: the target workqueue
3943  * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
3944  *
3945  * Apply @attrs to an unbound workqueue @wq.  Unless disabled, on NUMA
3946  * machines, this function maps a separate pwq to each NUMA node with
3947  * possibles CPUs in @attrs->cpumask so that work items are affine to the
3948  * NUMA node it was issued on.  Older pwqs are released as in-flight work
3949  * items finish.  Note that a work item which repeatedly requeues itself
3950  * back-to-back will stay on its current pwq.
3951  *
3952  * Performs GFP_KERNEL allocations.  Returns 0 on success and -errno on
3953  * failure.
3954  */
3955 int apply_workqueue_attrs(struct workqueue_struct *wq,
3956                           const struct workqueue_attrs *attrs)
3957 {
3958         struct workqueue_attrs *new_attrs, *tmp_attrs;
3959         struct pool_workqueue **pwq_tbl, *dfl_pwq;
3960         int node, ret;
3961 
3962         /* only unbound workqueues can change attributes */
3963         if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3964                 return -EINVAL;
3965 
3966         /* creating multiple pwqs breaks ordering guarantee */
3967         if (!list_empty(&wq->pwqs)) {
3968                 if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
3969                         return -EINVAL;
3970 
3971                 wq->flags &= ~__WQ_ORDERED;
3972         }
3973 
3974         pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3975         new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3976         tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3977         if (!pwq_tbl || !new_attrs || !tmp_attrs)
3978                 goto enomem;
3979 
3980         /* make a copy of @attrs and sanitize it */
3981         copy_workqueue_attrs(new_attrs, attrs);
3982         cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3983 
3984         /*
3985          * We may create multiple pwqs with differing cpumasks.  Make a
3986          * copy of @new_attrs which will be modified and used to obtain
3987          * pools.
3988          */
3989         copy_workqueue_attrs(tmp_attrs, new_attrs);
3990 
3991         /*
3992          * CPUs should stay stable across pwq creations and installations.
3993          * Pin CPUs, determine the target cpumask for each node and create
3994          * pwqs accordingly.
3995          */
3996         get_online_cpus();
3997 
3998         mutex_lock(&wq_pool_mutex);
3999 
4000         /*
4001          * If something goes wrong during CPU up/down, we'll fall back to
4002          * the default pwq covering whole @attrs->cpumask.  Always create
4003          * it even if we don't use it immediately.
4004          */
4005         dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
4006         if (!dfl_pwq)
4007                 goto enomem_pwq;
4008 
4009         for_each_node(node) {
4010                 if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
4011                         pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
4012                         if (!pwq_tbl[node])
4013                                 goto enomem_pwq;
4014                 } else {
4015                         dfl_pwq->refcnt++;
4016                         pwq_tbl[node] = dfl_pwq;
4017                 }
4018         }
4019 
4020         mutex_unlock(&wq_pool_mutex);
4021 
4022         /* all pwqs have been created successfully, let's install'em */
4023         mutex_lock(&wq->mutex);
4024 
4025         copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
4026 
4027         /* save the previous pwq and install the new one */
4028         for_each_node(node)
4029                 pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
4030 
4031         /* @dfl_pwq might not have been used, ensure it's linked */
4032         link_pwq(dfl_pwq);
4033         swap(wq->dfl_pwq, dfl_pwq);
4034 
4035         mutex_unlock(&wq->mutex);
4036 
4037         /* put the old pwqs */
4038         for_each_node(node)
4039                 put_pwq_unlocked(pwq_tbl[node]);
4040         put_pwq_unlocked(dfl_pwq);
4041 
4042         put_online_cpus();
4043         ret = 0;
4044         /* fall through */
4045 out_free:
4046         free_workqueue_attrs(tmp_attrs);
4047         free_workqueue_attrs(new_attrs);
4048         kfree(pwq_tbl);
4049         return ret;
4050 
4051 enomem_pwq:
4052         free_unbound_pwq(dfl_pwq);
4053         for_each_node(node)
4054                 if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
4055                         free_unbound_pwq(pwq_tbl[node]);
4056         mutex_unlock(&wq_pool_mutex);
4057         put_online_cpus();
4058 enomem:
4059         ret = -ENOMEM;
4060         goto out_free;
4061 }
4062 
4063 /**
4064  * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4065  * @wq: the target workqueue
4066  * @cpu: the CPU coming up or going down
4067  * @online: whether @cpu is coming up or going down
4068  *
4069  * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4070  * %CPU_DOWN_FAILED.  @cpu is being hot[un]plugged, update NUMA affinity of
4071  * @wq accordingly.
4072  *
4073  * If NUMA affinity can't be adjusted due to memory allocation failure, it
4074  * falls back to @wq->dfl_pwq which may not be optimal but is always
4075  * correct.
4076  *
4077  * Note that when the last allowed CPU of a NUMA node goes offline for a
4078  * workqueue with a cpumask spanning multiple nodes, the workers which were
4079  * already executing the work items for the workqueue will lose their CPU
4080  * affinity and may execute on any CPU.  This is similar to how per-cpu
4081  * workqueues behave on CPU_DOWN.  If a workqueue user wants strict
4082  * affinity, it's the user's responsibility to flush the work item from
4083  * CPU_DOWN_PREPARE.
4084  */
4085 static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4086                                    bool online)
4087 {
4088         int node = cpu_to_node(cpu);
4089         int cpu_off = online ? -1 : cpu;
4090         struct pool_workqueue *old_pwq = NULL, *pwq;
4091         struct workqueue_attrs *target_attrs;
4092         cpumask_t *cpumask;
4093 
4094         lockdep_assert_held(&wq_pool_mutex);
4095 
4096         if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))
4097                 return;
4098 
4099         /*
4100          * We don't wanna alloc/free wq_attrs for each wq for each CPU.
4101          * Let's use a preallocated one.  The following buf is protected by
4102          * CPU hotplug exclusion.
4103          */
4104         target_attrs = wq_update_unbound_numa_attrs_buf;
4105         cpumask = target_attrs->cpumask;
4106 
4107         mutex_lock(&wq->mutex);
4108         if (wq->unbound_attrs->no_numa)
4109                 goto out_unlock;
4110 
4111         copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4112         pwq = unbound_pwq_by_node(wq, node);
4113 
4114         /*
4115          * Let's determine what needs to be done.  If the target cpumask is
4116          * different from wq's, we need to compare it to @pwq's and create
4117          * a new one if they don't match.  If the target cpumask equals
4118          * wq's, the default pwq should be used.  If @pwq is already the
4119          * default one, nothing to do; otherwise, install the default one.
4120          */
4121         if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4122                 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4123                         goto out_unlock;
4124         } else {
4125                 if (pwq == wq->dfl_pwq)
4126                         goto out_unlock;
4127                 else
4128                         goto use_dfl_pwq;
4129         }
4130 
4131         mutex_unlock(&wq->mutex);
4132 
4133         /* create a new pwq */
4134         pwq = alloc_unbound_pwq(wq, target_attrs);
4135         if (!pwq) {
4136                 pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4137                            wq->name);
4138                 mutex_lock(&wq->mutex);
4139                 goto use_dfl_pwq;
4140         }
4141 
4142         /*
4143          * Install the new pwq.  As this function is called only from CPU
4144          * hotplug callbacks and applying a new attrs is wrapped with
4145          * get/put_online_cpus(), @wq->unbound_attrs couldn't have changed
4146          * inbetween.
4147          */
4148         mutex_lock(&wq->mutex);
4149         old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4150         goto out_unlock;
4151 
4152 use_dfl_pwq:
4153         spin_lock_irq(&wq->dfl_pwq->pool->lock);
4154         get_pwq(wq->dfl_pwq);
4155         spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4156         old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4157 out_unlock:
4158         mutex_unlock(&wq->mutex);
4159         put_pwq_unlocked(old_pwq);
4160 }
4161 
4162 static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4163 {
4164         bool highpri = wq->flags & WQ_HIGHPRI;
4165         int cpu, ret;
4166 
4167         if (!(wq->flags & WQ_UNBOUND)) {
4168                 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4169                 if (!wq->cpu_pwqs)
4170                         return -ENOMEM;
4171 
4172                 for_each_possible_cpu(cpu) {
4173                         struct pool_workqueue *pwq =
4174                                 per_cpu_ptr(wq->cpu_pwqs, cpu);
4175                         struct worker_pool *cpu_pools =
4176                                 per_cpu(cpu_worker_pools, cpu);
4177 
4178                         init_pwq(pwq, wq, &cpu_pools[highpri]);
4179 
4180                         mutex_lock(&wq->mutex);
4181                         link_pwq(pwq);
4182                         mutex_unlock(&wq->mutex);
4183                 }
4184                 return 0;
4185         } else if (wq->flags & __WQ_ORDERED) {
4186                 ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4187                 /* there should only be single pwq for ordering guarantee */
4188                 WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
4189                               wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4190                      "ordering guarantee broken for workqueue %s\n", wq->name);
4191                 return ret;
4192         } else {
4193                 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4194         }
4195 }
4196 
4197 static int wq_clamp_max_active(int max_active, unsigned int flags,
4198                                const char *name)
4199 {
4200         int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4201 
4202         if (max_active < 1 || max_active > lim)
4203                 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4204                         max_active, name, 1, lim);
4205 
4206         return clamp_val(max_active, 1, lim);
4207 }
4208 
4209 struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
4210                                                unsigned int flags,
4211                                                int max_active,
4212                                                struct lock_class_key *key,
4213                                                const char *lock_name, ...)
4214 {
4215         size_t tbl_size = 0;
4216         va_list args;
4217         struct workqueue_struct *wq;
4218         struct pool_workqueue *pwq;
4219 
4220         /*
4221          * Unbound && max_active == 1 used to imply ordered, which is no
4222          * longer the case on NUMA machines due to per-node pools.  While
4223          * alloc_ordered_workqueue() is the right way to create an ordered
4224          * workqueue, keep the previous behavior to avoid subtle breakages
4225          * on NUMA.
4226          */
4227         if ((flags & WQ_UNBOUND) && max_active == 1)
4228                 flags |= __WQ_ORDERED;
4229 
4230         /* allocate wq and format name */
4231         if (flags & WQ_UNBOUND)
4232                 tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4233 
4234         wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4235         if (!wq)
4236                 return NULL;
4237 
4238         if (flags & WQ_UNBOUND) {
4239                 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4240                 if (!wq->unbound_attrs)
4241                         goto err_free_wq;
4242         }
4243 
4244         va_start(args, lock_name);
4245         vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4246         va_end(args);
4247 
4248         max_active = max_active ?: WQ_DFL_ACTIVE;
4249         max_active = wq_clamp_max_active(max_active, flags, wq->name);
4250 
4251         /* init wq */
4252         wq->flags = flags;
4253         wq->saved_max_active = max_active;
4254         mutex_init(&wq->mutex);
4255         atomic_set(&wq->nr_pwqs_to_flush, 0);
4256         INIT_LIST_HEAD(&wq->pwqs);
4257         INIT_LIST_HEAD(&wq->flusher_queue);
4258         INIT_LIST_HEAD(&wq->flusher_overflow);
4259         INIT_LIST_HEAD(&wq->maydays);
4260 
4261         lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4262         INIT_LIST_HEAD(&wq->list);
4263 
4264         if (alloc_and_link_pwqs(wq) < 0)
4265                 goto err_free_wq;
4266 
4267         /*
4268          * Workqueues which may be used during memory reclaim should
4269          * have a rescuer to guarantee forward progress.
4270          */
4271         if (flags & WQ_MEM_RECLAIM) {
4272                 struct worker *rescuer;
4273 
4274                 rescuer = alloc_worker();
4275                 if (!rescuer)
4276                         goto err_destroy;
4277 
4278                 rescuer->rescue_wq = wq;
4279                 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4280                                                wq->name);
4281                 if (IS_ERR(rescuer->task)) {
4282                         kfree(rescuer);
4283                         goto err_destroy;
4284                 }
4285 
4286                 wq->rescuer = rescuer;
4287                 rescuer->task->flags |= PF_NO_SETAFFINITY;
4288                 wake_up_process(rescuer->task);
4289         }
4290 
4291         if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4292                 goto err_destroy;
4293 
4294         /*
4295          * wq_pool_mutex protects global freeze state and workqueues list.
4296          * Grab it, adjust max_active and add the new @wq to workqueues
4297          * list.
4298          */
4299         mutex_lock(&wq_pool_mutex);
4300 
4301         mutex_lock(&wq->mutex);
4302         for_each_pwq(pwq, wq)
4303                 pwq_adjust_max_active(pwq);
4304         mutex_unlock(&wq->mutex);
4305 
4306         list_add(&wq->list, &workqueues);
4307 
4308         mutex_unlock(&wq_pool_mutex);
4309 
4310         return wq;
4311 
4312 err_free_wq:
4313         free_workqueue_attrs(wq->unbound_attrs);
4314         kfree(wq);
4315         return NULL;
4316 err_destroy:
4317         destroy_workqueue(wq);
4318         return NULL;
4319 }
4320 EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4321 
4322 /**
4323  * destroy_workqueue - safely terminate a workqueue
4324  * @wq: target workqueue
4325  *
4326  * Safely destroy a workqueue. All work currently pending will be done first.
4327  */
4328 void destroy_workqueue(struct workqueue_struct *wq)
4329 {
4330         struct pool_workqueue *pwq;
4331         int node;
4332 
4333         /* drain it before proceeding with destruction */
4334         drain_workqueue(wq);
4335 
4336         /* sanity checks */
4337         mutex_lock(&wq->mutex);
4338         for_each_pwq(pwq, wq) {
4339                 int i;
4340 
4341                 for (i = 0; i < WORK_NR_COLORS; i++) {
4342                         if (WARN_ON(pwq->nr_in_flight[i])) {
4343                                 mutex_unlock(&wq->mutex);
4344                                 return;
4345                         }
4346                 }
4347 
4348                 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
4349                     WARN_ON(pwq->nr_active) ||
4350                     WARN_ON(!list_empty(&pwq->delayed_works))) {
4351                         mutex_unlock(&wq->mutex);
4352                         return;
4353                 }
4354         }
4355         mutex_unlock(&wq->mutex);
4356 
4357         /*
4358          * wq list is used to freeze wq, remove from list after
4359          * flushing is complete in case freeze races us.
4360          */
4361         mutex_lock(&wq_pool_mutex);
4362         list_del_init(&wq->list);
4363         mutex_unlock(&wq_pool_mutex);
4364 
4365         workqueue_sysfs_unregister(wq);
4366 
4367         if (wq->rescuer) {
4368                 kthread_stop(wq->rescuer->task);
4369                 kfree(wq->rescuer);
4370                 wq->rescuer = NULL;
4371         }
4372 
4373         if (!(wq->flags & WQ_UNBOUND)) {
4374                 /*
4375                  * The base ref is never dropped on per-cpu pwqs.  Directly
4376                  * free the pwqs and wq.
4377                  */
4378                 free_percpu(wq->cpu_pwqs);
4379                 kfree(wq);
4380         } else {
4381                 /*
4382                  * We're the sole accessor of @wq at this point.  Directly
4383                  * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4384                  * @wq will be freed when the last pwq is released.
4385                  */
4386                 for_each_node(node) {
4387                         pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4388                         RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4389                         put_pwq_unlocked(pwq);
4390                 }
4391 
4392                 /*
4393                  * Put dfl_pwq.  @wq may be freed any time after dfl_pwq is
4394                  * put.  Don't access it afterwards.
4395                  */
4396                 pwq = wq->dfl_pwq;
4397                 wq->dfl_pwq = NULL;
4398                 put_pwq_unlocked(pwq);
4399         }
4400 }
4401 EXPORT_SYMBOL_GPL(destroy_workqueue);
4402 
4403 /**
4404  * workqueue_set_max_active - adjust max_active of a workqueue
4405  * @wq: target workqueue
4406  * @max_active: new max_active value.
4407  *
4408  * Set max_active of @wq to @max_active.
4409  *
4410  * CONTEXT:
4411  * Don't call from IRQ context.
4412  */
4413 void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4414 {
4415         struct pool_workqueue *pwq;
4416 
4417         /* disallow meddling with max_active for ordered workqueues */
4418         if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4419                 return;
4420 
4421         max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4422 
4423         mutex_lock(&wq->mutex);
4424 
4425         wq->flags &= ~__WQ_ORDERED;
4426         wq->saved_max_active = max_active;
4427 
4428         for_each_pwq(pwq, wq)
4429                 pwq_adjust_max_active(pwq);
4430 
4431         mutex_unlock(&wq->mutex);
4432 }
4433 EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4434 
4435 /**
4436  * current_is_workqueue_rescuer - is %current workqueue rescuer?
4437  *
4438  * Determine whether %current is a workqueue rescuer.  Can be used from
4439  * work functions to determine whether it's being run off the rescuer task.
4440  */
4441 bool current_is_workqueue_rescuer(void)
4442 {
4443         struct worker *worker = current_wq_worker();
4444 
4445         return worker && worker->rescue_wq;
4446 }
4447 
4448 /**
4449  * workqueue_congested - test whether a workqueue is congested
4450  * @cpu: CPU in question
4451  * @wq: target workqueue
4452  *
4453  * Test whether @wq's cpu workqueue for @cpu is congested.  There is
4454  * no synchronization around this function and the test result is
4455  * unreliable and only useful as advisory hints or for debugging.
4456  *
4457  * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4458  * Note that both per-cpu and unbound workqueues may be associated with
4459  * multiple pool_workqueues which have separate congested states.  A
4460  * workqueue being congested on one CPU doesn't mean the workqueue is also
4461  * contested on other CPUs / NUMA nodes.
4462  *
4463  * RETURNS:
4464  * %true if congested, %false otherwise.
4465  */
4466 bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4467 {
4468         struct pool_workqueue *pwq;
4469         bool ret;
4470 
4471         rcu_read_lock_sched();
4472 
4473         if (cpu == WORK_CPU_UNBOUND)
4474                 cpu = smp_processor_id();
4475 
4476         if (!(wq->flags & WQ_UNBOUND))
4477                 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4478         else
4479                 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4480 
4481         ret = !list_empty(&pwq->delayed_works);
4482         rcu_read_unlock_sched();
4483 
4484         return ret;
4485 }
4486 EXPORT_SYMBOL_GPL(workqueue_congested);
4487 
4488 /**
4489  * work_busy - test whether a work is currently pending or running
4490  * @work: the work to be tested
4491  *
4492  * Test whether @work is currently pending or running.  There is no
4493  * synchronization around this function and the test result is
4494  * unreliable and only useful as advisory hints or for debugging.
4495  *
4496  * RETURNS:
4497  * OR'd bitmask of WORK_BUSY_* bits.
4498  */
4499 unsigned int work_busy(struct work_struct *work)
4500 {
4501         struct worker_pool *pool;
4502         unsigned long flags;
4503         unsigned int ret = 0;
4504 
4505         if (work_pending(work))
4506                 ret |= WORK_BUSY_PENDING;
4507 
4508         local_irq_save(flags);
4509         pool = get_work_pool(work);
4510         if (pool) {
4511                 spin_lock(&pool->lock);
4512                 if (find_worker_executing_work(pool, work))
4513                         ret |= WORK_BUSY_RUNNING;
4514                 spin_unlock(&pool->lock);
4515         }
4516         local_irq_restore(flags);
4517 
4518         return ret;
4519 }
4520 EXPORT_SYMBOL_GPL(work_busy);
4521 
4522 /**
4523  * set_worker_desc - set description for the current work item
4524  * @fmt: printf-style format string
4525  * @...: arguments for the format string
4526  *
4527  * This function can be called by a running work function to describe what
4528  * the work item is about.  If the worker task gets dumped, this
4529  * information will be printed out together to help debugging.  The
4530  * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4531  */
4532 void set_worker_desc(const char *fmt, ...)
4533 {
4534         struct worker *worker = current_wq_worker();
4535         va_list args;
4536 
4537         if (worker) {
4538                 va_start(args, fmt);
4539                 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4540                 va_end(args);
4541                 worker->desc_valid = true;
4542         }
4543 }
4544 
4545 /**
4546  * print_worker_info - print out worker information and description
4547  * @log_lvl: the log level to use when printing
4548  * @task: target task
4549  *
4550  * If @task is a worker and currently executing a work item, print out the
4551  * name of the workqueue being serviced and worker description set with
4552  * set_worker_desc() by the currently executing work item.
4553  *
4554  * This function can be safely called on any task as long as the
4555  * task_struct itself is accessible.  While safe, this function isn't
4556  * synchronized and may print out mixups or garbages of limited length.
4557  */
4558 void print_worker_info(const char *log_lvl, struct task_struct *task)
4559 {
4560         work_func_t *fn = NULL;
4561         char name[WQ_NAME_LEN] = { };
4562         char desc[WORKER_DESC_LEN] = { };
4563         struct pool_workqueue *pwq = NULL;
4564         struct workqueue_struct *wq = NULL;
4565         bool desc_valid = false;
4566         struct worker *worker;
4567 
4568         if (!(task->flags & PF_WQ_WORKER))
4569                 return;
4570 
4571         /*
4572          * This function is called without any synchronization and @task
4573          * could be in any state.  Be careful with dereferences.
4574          */
4575         worker = probe_kthread_data(task);
4576 
4577         /*
4578          * Carefully copy the associated workqueue's workfn and name.  Keep
4579          * the original last '\0' in case the original contains garbage.
4580          */
4581         probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4582         probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4583         probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4584         probe_kernel_read(name, wq->name, sizeof(name) - 1);
4585 
4586         /* copy worker description */
4587         probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4588         if (desc_valid)
4589                 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4590 
4591         if (fn || name[0] || desc[0]) {
4592                 printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4593                 if (desc[0])
4594                         pr_cont(" (%s)", desc);
4595                 pr_cont("\n");
4596         }
4597 }
4598 
4599 /*
4600  * CPU hotplug.
4601  *
4602  * There are two challenges in supporting CPU hotplug.  Firstly, there
4603  * are a lot of assumptions on strong associations among work, pwq and
4604  * pool which make migrating pending and scheduled works very
4605  * difficult to implement without impacting hot paths.  Secondly,
4606  * worker pools serve mix of short, long and very long running works making
4607  * blocked draining impractical.
4608  *
4609  * This is solved by allowing the pools to be disassociated from the CPU
4610  * running as an unbound one and allowing it to be reattached later if the
4611  * cpu comes back online.
4612  */
4613 
4614 static void wq_unbind_fn(struct work_struct *work)
4615 {
4616         int cpu = smp_processor_id();
4617         struct worker_pool *pool;
4618         struct worker *worker;
4619         int wi;
4620 
4621         for_each_cpu_worker_pool(pool, cpu) {
4622                 WARN_ON_ONCE(cpu != smp_processor_id());
4623 
4624                 mutex_lock(&pool->manager_mutex);
4625                 spin_lock_irq(&pool->lock);
4626 
4627                 /*
4628                  * We've blocked all manager operations.  Make all workers
4629                  * unbound and set DISASSOCIATED.  Before this, all workers
4630                  * except for the ones which are still executing works from
4631                  * before the last CPU down must be on the cpu.  After
4632                  * this, they may become diasporas.
4633                  */
4634                 for_each_pool_worker(worker, wi, pool)
4635                         worker->flags |= WORKER_UNBOUND;
4636 
4637                 pool->flags |= POOL_DISASSOCIATED;
4638 
4639                 spin_unlock_irq(&pool->lock);
4640                 mutex_unlock(&pool->manager_mutex);
4641 
4642                 /*
4643                  * Call schedule() so that we cross rq->lock and thus can
4644                  * guarantee sched callbacks see the %WORKER_UNBOUND flag.
4645                  * This is necessary as scheduler callbacks may be invoked
4646                  * from other cpus.
4647                  */
4648                 schedule();
4649 
4650                 /*
4651                  * Sched callbacks are disabled now.  Zap nr_running.
4652                  * After this, nr_running stays zero and need_more_worker()
4653                  * and keep_working() are always true as long as the
4654                  * worklist is not empty.  This pool now behaves as an
4655                  * unbound (in terms of concurrency management) pool which
4656                  * are served by workers tied to the pool.
4657                  */
4658                 atomic_set(&pool->nr_running, 0);
4659 
4660                 /*
4661                  * With concurrency management just turned off, a busy
4662                  * worker blocking could lead to lengthy stalls.  Kick off
4663                  * unbound chain execution of currently pending work items.
4664                  */
4665                 spin_lock_irq(&pool->lock);
4666                 wake_up_worker(pool);
4667                 spin_unlock_irq(&pool->lock);
4668         }
4669 }
4670 
4671 /**
4672  * rebind_workers - rebind all workers of a pool to the associated CPU
4673  * @pool: pool of interest
4674  *
4675  * @pool->cpu is coming online.  Rebind all workers to the CPU.
4676  */
4677 static void rebind_workers(struct worker_pool *pool)
4678 {
4679         struct worker *worker;
4680         int wi;
4681 
4682         lockdep_assert_held(&pool->manager_mutex);
4683 
4684         /*
4685          * Restore CPU affinity of all workers.  As all idle workers should
4686          * be on the run-queue of the associated CPU before any local
4687          * wake-ups for concurrency management happen, restore CPU affinty
4688          * of all workers first and then clear UNBOUND.  As we're called
4689          * from CPU_ONLINE, the following shouldn't fail.
4690          */
4691         for_each_pool_worker(worker, wi, pool)
4692                 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4693                                                   pool->attrs->cpumask) < 0);
4694 
4695         spin_lock_irq(&pool->lock);
4696 
4697         for_each_pool_worker(worker, wi, pool) {
4698                 unsigned int worker_flags = worker->flags;
4699 
4700                 /*
4701                  * A bound idle worker should actually be on the runqueue
4702                  * of the associated CPU for local wake-ups targeting it to
4703                  * work.  Kick all idle workers so that they migrate to the
4704                  * associated CPU.  Doing this in the same loop as
4705                  * replacing UNBOUND with REBOUND is safe as no worker will
4706                  * be bound before @pool->lock is released.
4707                  */
4708                 if (worker_flags & WORKER_IDLE)
4709                         wake_up_process(worker->task);
4710 
4711                 /*
4712                  * We want to clear UNBOUND but can't directly call
4713                  * worker_clr_flags() or adjust nr_running.  Atomically
4714                  * replace UNBOUND with another NOT_RUNNING flag REBOUND.
4715                  * @worker will clear REBOUND using worker_clr_flags() when
4716                  * it initiates the next execution cycle thus restoring
4717                  * concurrency management.  Note that when or whether
4718                  * @worker clears REBOUND doesn't affect correctness.
4719                  *
4720                  * ACCESS_ONCE() is necessary because @worker->flags may be
4721                  * tested without holding any lock in
4722                  * wq_worker_waking_up().  Without it, NOT_RUNNING test may
4723                  * fail incorrectly leading to premature concurrency
4724                  * management operations.
4725                  */
4726                 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4727                 worker_flags |= WORKER_REBOUND;
4728                 worker_flags &= ~WORKER_UNBOUND;
4729                 ACCESS_ONCE(worker->flags) = worker_flags;
4730         }
4731 
4732         spin_unlock_irq(&pool->lock);
4733 }
4734 
4735 /**
4736  * restore_unbound_workers_cpumask - restore cpumask of unbound workers
4737  * @pool: unbound pool of interest
4738  * @cpu: the CPU which is coming up
4739  *
4740  * An unbound pool may end up with a cpumask which doesn't have any online
4741  * CPUs.  When a worker of such pool get scheduled, the scheduler resets
4742  * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
4743  * online CPU before, cpus_allowed of all its workers should be restored.
4744  */
4745 static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4746 {
4747         static cpumask_t cpumask;
4748         struct worker *worker;
4749         int wi;
4750 
4751         lockdep_assert_held(&pool->manager_mutex);
4752 
4753         /* is @cpu allowed for @pool? */
4754         if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4755                 return;
4756 
4757         /* is @cpu the only online CPU? */
4758         cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4759         if (cpumask_weight(&cpumask) != 1)
4760                 return;
4761 
4762         /* as we're called from CPU_ONLINE, the following shouldn't fail */
4763         for_each_pool_worker(worker, wi, pool)
4764                 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4765                                                   pool->attrs->cpumask) < 0);
4766 }
4767 
4768 /*
4769  * Workqueues should be brought up before normal priority CPU notifiers.
4770  * This will be registered high priority CPU notifier.
4771  */
4772 static int __cpuinit workqueue_cpu_up_callback(struct notifier_block *nfb,
4773                                                unsigned long action,
4774                                                void *hcpu)
4775 {
4776         int cpu = (unsigned long)hcpu;
4777         struct worker_pool *pool;
4778         struct workqueue_struct *wq;
4779         int pi;
4780 
4781         switch (action & ~CPU_TASKS_FROZEN) {
4782         case CPU_UP_PREPARE:
4783                 for_each_cpu_worker_pool(pool, cpu) {
4784                         if (pool->nr_workers)
4785                                 continue;
4786                         if (create_and_start_worker(pool) < 0)
4787                                 return NOTIFY_BAD;
4788                 }
4789                 break;
4790 
4791         case CPU_DOWN_FAILED:
4792         case CPU_ONLINE:
4793                 mutex_lock(&wq_pool_mutex);
4794 
4795                 for_each_pool(pool, pi) {
4796                         mutex_lock(&pool->manager_mutex);
4797 
4798                         if (pool->cpu == cpu) {
4799                                 spin_lock_irq(&pool->lock);
4800                                 pool->flags &= ~POOL_DISASSOCIATED;
4801                                 spin_unlock_irq(&pool->lock);
4802 
4803                                 rebind_workers(pool);
4804                         } else if (pool->cpu < 0) {
4805                                 restore_unbound_workers_cpumask(pool, cpu);
4806                         }
4807 
4808                         mutex_unlock(&pool->manager_mutex);
4809                 }
4810 
4811                 /* update NUMA affinity of unbound workqueues */
4812                 list_for_each_entry(wq, &workqueues, list)
4813                         wq_update_unbound_numa(wq, cpu, true);
4814 
4815                 mutex_unlock(&wq_pool_mutex);
4816                 break;
4817         }
4818         return NOTIFY_OK;
4819 }
4820 
4821 /*
4822  * Workqueues should be brought down after normal priority CPU notifiers.
4823  * This will be registered as low priority CPU notifier.
4824  */
4825 static int __cpuinit workqueue_cpu_down_callback(struct notifier_block *nfb,
4826                                                  unsigned long action,
4827                                                  void *hcpu)
4828 {
4829         int cpu = (unsigned long)hcpu;
4830         struct work_struct unbind_work;
4831         struct workqueue_struct *wq;
4832 
4833         switch (action & ~CPU_TASKS_FROZEN) {
4834         case CPU_DOWN_PREPARE:
4835                 /* unbinding per-cpu workers should happen on the local CPU */
4836                 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4837                 queue_work_on(cpu, system_highpri_wq, &unbind_work);
4838 
4839                 /* update NUMA affinity of unbound workqueues */
4840                 mutex_lock(&wq_pool_mutex);
4841                 list_for_each_entry(wq, &workqueues, list)
4842                         wq_update_unbound_numa(wq, cpu, false);
4843                 mutex_unlock(&wq_pool_mutex);
4844 
4845                 /* wait for per-cpu unbinding to finish */
4846                 flush_work(&unbind_work);
4847                 break;
4848         }
4849         return NOTIFY_OK;
4850 }
4851 
4852 #ifdef CONFIG_SMP
4853 
4854 struct work_for_cpu {
4855         struct work_struct work;
4856         long (*fn)(void *);
4857         void *arg;
4858         long ret;
4859 };
4860 
4861 static void work_for_cpu_fn(struct work_struct *work)
4862 {
4863         struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4864 
4865         wfc->ret = wfc->fn(wfc->arg);
4866 }
4867 
4868 /**
4869  * work_on_cpu - run a function in user context on a particular cpu
4870  * @cpu: the cpu to run on
4871  * @fn: the function to run
4872  * @arg: the function arg
4873  *
4874  * This will return the value @fn returns.
4875  * It is up to the caller to ensure that the cpu doesn't go offline.
4876  * The caller must not hold any locks which would prevent @fn from completing.
4877  */
4878 long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4879 {
4880         struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4881 
4882         INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4883         schedule_work_on(cpu, &wfc.work);
4884         flush_work(&wfc.work);
4885         return wfc.ret;
4886 }
4887 EXPORT_SYMBOL_GPL(work_on_cpu);
4888 #endif /* CONFIG_SMP */
4889 
4890 #ifdef CONFIG_FREEZER
4891 
4892 /**
4893  * freeze_workqueues_begin - begin freezing workqueues
4894  *
4895  * Start freezing workqueues.  After this function returns, all freezable
4896  * workqueues will queue new works to their delayed_works list instead of
4897  * pool->worklist.
4898  *
4899  * CONTEXT:
4900  * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4901  */
4902 void freeze_workqueues_begin(void)
4903 {
4904         struct worker_pool *pool;
4905         struct workqueue_struct *wq;
4906         struct pool_workqueue *pwq;
4907         int pi;
4908 
4909         mutex_lock(&wq_pool_mutex);
4910 
4911         WARN_ON_ONCE(workqueue_freezing);
4912         workqueue_freezing = true;
4913 
4914         /* set FREEZING */
4915         for_each_pool(pool, pi) {
4916                 spin_lock_irq(&pool->lock);
4917                 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4918                 pool->flags |= POOL_FREEZING;
4919                 spin_unlock_irq(&pool->lock);
4920         }
4921 
4922         list_for_each_entry(wq, &workqueues, list) {
4923                 mutex_lock(&wq->mutex);
4924                 for_each_pwq(pwq, wq)
4925                         pwq_adjust_max_active(pwq);
4926                 mutex_unlock(&wq->mutex);
4927         }
4928 
4929         mutex_unlock(&wq_pool_mutex);
4930 }
4931 
4932 /**
4933  * freeze_workqueues_busy - are freezable workqueues still busy?
4934  *
4935  * Check whether freezing is complete.  This function must be called
4936  * between freeze_workqueues_begin() and thaw_workqueues().
4937  *
4938  * CONTEXT:
4939  * Grabs and releases wq_pool_mutex.
4940  *
4941  * RETURNS:
4942  * %true if some freezable workqueues are still busy.  %false if freezing
4943  * is complete.
4944  */
4945 bool freeze_workqueues_busy(void)
4946 {
4947         bool busy = false;
4948         struct workqueue_struct *wq;
4949         struct pool_workqueue *pwq;
4950 
4951         mutex_lock(&wq_pool_mutex);
4952 
4953         WARN_ON_ONCE(!workqueue_freezing);
4954 
4955         list_for_each_entry(wq, &workqueues, list) {
4956                 if (!(wq->flags & WQ_FREEZABLE))
4957                         continue;
4958                 /*
4959                  * nr_active is monotonically decreasing.  It's safe
4960                  * to peek without lock.
4961                  */
4962                 rcu_read_lock_sched();
4963                 for_each_pwq(pwq, wq) {
4964                         WARN_ON_ONCE(pwq->nr_active < 0);
4965                         if (pwq->nr_active) {
4966                                 busy = true;
4967                                 rcu_read_unlock_sched();
4968                                 goto out_unlock;
4969                         }
4970                 }
4971                 rcu_read_unlock_sched();
4972         }
4973 out_unlock:
4974         mutex_unlock(&wq_pool_mutex);
4975         return busy;
4976 }
4977 
4978 /**
4979  * thaw_workqueues - thaw workqueues
4980  *
4981  * Thaw workqueues.  Normal queueing is restored and all collected
4982  * frozen works are transferred to their respective pool worklists.
4983  *
4984  * CONTEXT:
4985  * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
4986  */
4987 void thaw_workqueues(void)
4988 {
4989         struct workqueue_struct *wq;
4990         struct pool_workqueue *pwq;
4991         struct worker_pool *pool;
4992         int pi;
4993 
4994         mutex_lock(&wq_pool_mutex);
4995 
4996         if (!workqueue_freezing)
4997                 goto out_unlock;
4998 
4999         /* clear FREEZING */
5000         for_each_pool(pool, pi) {
5001                 spin_lock_irq(&pool->lock);
5002                 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
5003                 pool->flags &= ~POOL_FREEZING;
5004                 spin_unlock_irq(&pool->lock);
5005         }
5006 
5007         /* restore max_active and repopulate worklist */
5008         list_for_each_entry(wq, &workqueues, list) {
5009                 mutex_lock(&wq->mutex);
5010                 for_each_pwq(pwq, wq)
5011                         pwq_adjust_max_active(pwq);
5012                 mutex_unlock(&wq->mutex);
5013         }
5014 
5015         workqueue_freezing = false;
5016 out_unlock:
5017         mutex_unlock(&wq_pool_mutex);
5018 }
5019 #endif /* CONFIG_FREEZER */
5020 
5021 static void __init wq_numa_init(void)
5022 {
5023         cpumask_var_t *tbl;
5024         int node, cpu;
5025 
5026         /* determine NUMA pwq table len - highest node id + 1 */
5027         for_each_node(node)
5028                 wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
5029 
5030         if (num_possible_nodes() <= 1)
5031                 return;
5032 
5033         if (wq_disable_numa) {
5034                 pr_info("workqueue: NUMA affinity support disabled\n");
5035                 return;
5036         }
5037 
5038         wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
5039         BUG_ON(!wq_update_unbound_numa_attrs_buf);
5040 
5041         /*
5042          * We want masks of possible CPUs of each node which isn't readily
5043          * available.  Build one from cpu_to_node() which should have been
5044          * fully initialized by now.
5045          */
5046         tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
5047         BUG_ON(!tbl);
5048 
5049         for_each_node(node)
5050                 BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
5051                                 node_online(node) ? node : NUMA_NO_NODE));
5052 
5053         for_each_possible_cpu(cpu) {
5054                 node = cpu_to_node(cpu);
5055                 if (WARN_ON(node == NUMA_NO_NODE)) {
5056                         pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5057                         /* happens iff arch is bonkers, let's just proceed */
5058                         return;
5059                 }
5060                 cpumask_set_cpu(cpu, tbl[node]);
5061         }
5062 
5063         wq_numa_possible_cpumask = tbl;
5064         wq_numa_enabled = true;
5065 }
5066 
5067 static int __init init_workqueues(void)
5068 {
5069         int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5070         int i, cpu;
5071 
5072         /* make sure we have enough bits for OFFQ pool ID */
5073         BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
5074                      WORK_CPU_END * NR_STD_WORKER_POOLS);
5075 
5076         WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5077 
5078         pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5079 
5080         cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
5081         hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
5082 
5083         wq_numa_init();
5084 
5085         /* initialize CPU pools */
5086         for_each_possible_cpu(cpu) {
5087                 struct worker_pool *pool;
5088 
5089                 i = 0;
5090                 for_each_cpu_worker_pool(pool, cpu) {
5091                         BUG_ON(init_worker_pool(pool));
5092                         pool->cpu = cpu;
5093                         cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5094                         pool->attrs->nice = std_nice[i++];
5095                         pool->node = cpu_to_node(cpu);
5096 
5097                         /* alloc pool ID */
5098                         mutex_lock(&wq_pool_mutex);
5099                         BUG_ON(worker_pool_assign_id(pool));
5100                         mutex_unlock(&wq_pool_mutex);
5101                 }
5102         }
5103 
5104         /* create the initial worker */
5105         for_each_online_cpu(cpu) {
5106                 struct worker_pool *pool;
5107 
5108                 for_each_cpu_worker_pool(pool, cpu) {
5109                         pool->flags &= ~POOL_DISASSOCIATED;
5110                         BUG_ON(create_and_start_worker(pool) < 0);
5111                 }
5112         }
5113 
5114         /* create default unbound and ordered wq attrs */
5115         for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5116                 struct workqueue_attrs *attrs;
5117 
5118                 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5119                 attrs->nice = std_nice[i];
5120                 unbound_std_wq_attrs[i] = attrs;
5121 
5122                 /*
5123                  * An ordered wq should have only one pwq as ordering is
5124                  * guaranteed by max_active which is enforced by pwqs.
5125                  * Turn off NUMA so that dfl_pwq is used for all nodes.
5126                  */
5127                 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5128                 attrs->nice = std_nice[i];
5129                 attrs->no_numa = true;
5130                 ordered_wq_attrs[i] = attrs;
5131         }
5132 
5133         system_wq = alloc_workqueue("events", 0, 0);
5134         system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5135         system_long_wq = alloc_workqueue("events_long", 0, 0);
5136         system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5137                                             WQ_UNBOUND_MAX_ACTIVE);
5138         system_freezable_wq = alloc_workqueue("events_freezable",
5139                                               WQ_FREEZABLE, 0);
5140         BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
5141                !system_unbound_wq || !system_freezable_wq);
5142         return 0;
5143 }
5144 early_initcall(init_workqueues);
5145 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp