~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/locking/locktorture.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0+
  2 /*
  3  * Module-based torture test facility for locking
  4  *
  5  * Copyright (C) IBM Corporation, 2014
  6  *
  7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8  *          Davidlohr Bueso <dave@stgolabs.net>
  9  *      Based on kernel/rcu/torture.c.
 10  */
 11 
 12 #define pr_fmt(fmt) fmt
 13 
 14 #include <linux/kernel.h>
 15 #include <linux/module.h>
 16 #include <linux/kthread.h>
 17 #include <linux/sched/rt.h>
 18 #include <linux/spinlock.h>
 19 #include <linux/rwlock.h>
 20 #include <linux/mutex.h>
 21 #include <linux/rwsem.h>
 22 #include <linux/smp.h>
 23 #include <linux/interrupt.h>
 24 #include <linux/sched.h>
 25 #include <uapi/linux/sched/types.h>
 26 #include <linux/rtmutex.h>
 27 #include <linux/atomic.h>
 28 #include <linux/moduleparam.h>
 29 #include <linux/delay.h>
 30 #include <linux/slab.h>
 31 #include <linux/percpu-rwsem.h>
 32 #include <linux/torture.h>
 33 
 34 MODULE_LICENSE("GPL");
 35 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
 36 
 37 torture_param(int, nwriters_stress, -1,
 38              "Number of write-locking stress-test threads");
 39 torture_param(int, nreaders_stress, -1,
 40              "Number of read-locking stress-test threads");
 41 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
 42 torture_param(int, onoff_interval, 0,
 43              "Time between CPU hotplugs (s), 0=disable");
 44 torture_param(int, shuffle_interval, 3,
 45              "Number of jiffies between shuffles, 0=disable");
 46 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
 47 torture_param(int, stat_interval, 60,
 48              "Number of seconds between stats printk()s");
 49 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
 50 torture_param(int, verbose, 1,
 51              "Enable verbose debugging printk()s");
 52 
 53 static char *torture_type = "spin_lock";
 54 module_param(torture_type, charp, 0444);
 55 MODULE_PARM_DESC(torture_type,
 56                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
 57 
 58 static struct task_struct *stats_task;
 59 static struct task_struct **writer_tasks;
 60 static struct task_struct **reader_tasks;
 61 
 62 static bool lock_is_write_held;
 63 static bool lock_is_read_held;
 64 
 65 struct lock_stress_stats {
 66         long n_lock_fail;
 67         long n_lock_acquired;
 68 };
 69 
 70 /* Forward reference. */
 71 static void lock_torture_cleanup(void);
 72 
 73 /*
 74  * Operations vector for selecting different types of tests.
 75  */
 76 struct lock_torture_ops {
 77         void (*init)(void);
 78         int (*writelock)(void);
 79         void (*write_delay)(struct torture_random_state *trsp);
 80         void (*task_boost)(struct torture_random_state *trsp);
 81         void (*writeunlock)(void);
 82         int (*readlock)(void);
 83         void (*read_delay)(struct torture_random_state *trsp);
 84         void (*readunlock)(void);
 85 
 86         unsigned long flags; /* for irq spinlocks */
 87         const char *name;
 88 };
 89 
 90 struct lock_torture_cxt {
 91         int nrealwriters_stress;
 92         int nrealreaders_stress;
 93         bool debug_lock;
 94         atomic_t n_lock_torture_errors;
 95         struct lock_torture_ops *cur_ops;
 96         struct lock_stress_stats *lwsa; /* writer statistics */
 97         struct lock_stress_stats *lrsa; /* reader statistics */
 98 };
 99 static struct lock_torture_cxt cxt = { 0, 0, false,
100                                        ATOMIC_INIT(0),
101                                        NULL, NULL};
102 /*
103  * Definitions for lock torture testing.
104  */
105 
106 static int torture_lock_busted_write_lock(void)
107 {
108         return 0;  /* BUGGY, do not use in real life!!! */
109 }
110 
111 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
112 {
113         const unsigned long longdelay_ms = 100;
114 
115         /* We want a long delay occasionally to force massive contention.  */
116         if (!(torture_random(trsp) %
117               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
118                 mdelay(longdelay_ms);
119         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
120                 torture_preempt_schedule();  /* Allow test to be preempted. */
121 }
122 
123 static void torture_lock_busted_write_unlock(void)
124 {
125           /* BUGGY, do not use in real life!!! */
126 }
127 
128 static void torture_boost_dummy(struct torture_random_state *trsp)
129 {
130         /* Only rtmutexes care about priority */
131 }
132 
133 static struct lock_torture_ops lock_busted_ops = {
134         .writelock      = torture_lock_busted_write_lock,
135         .write_delay    = torture_lock_busted_write_delay,
136         .task_boost     = torture_boost_dummy,
137         .writeunlock    = torture_lock_busted_write_unlock,
138         .readlock       = NULL,
139         .read_delay     = NULL,
140         .readunlock     = NULL,
141         .name           = "lock_busted"
142 };
143 
144 static DEFINE_SPINLOCK(torture_spinlock);
145 
146 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
147 {
148         spin_lock(&torture_spinlock);
149         return 0;
150 }
151 
152 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
153 {
154         const unsigned long shortdelay_us = 2;
155         const unsigned long longdelay_ms = 100;
156 
157         /* We want a short delay mostly to emulate likely code, and
158          * we want a long delay occasionally to force massive contention.
159          */
160         if (!(torture_random(trsp) %
161               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
162                 mdelay(longdelay_ms);
163         if (!(torture_random(trsp) %
164               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
165                 udelay(shortdelay_us);
166         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
167                 torture_preempt_schedule();  /* Allow test to be preempted. */
168 }
169 
170 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
171 {
172         spin_unlock(&torture_spinlock);
173 }
174 
175 static struct lock_torture_ops spin_lock_ops = {
176         .writelock      = torture_spin_lock_write_lock,
177         .write_delay    = torture_spin_lock_write_delay,
178         .task_boost     = torture_boost_dummy,
179         .writeunlock    = torture_spin_lock_write_unlock,
180         .readlock       = NULL,
181         .read_delay     = NULL,
182         .readunlock     = NULL,
183         .name           = "spin_lock"
184 };
185 
186 static int torture_spin_lock_write_lock_irq(void)
187 __acquires(torture_spinlock)
188 {
189         unsigned long flags;
190 
191         spin_lock_irqsave(&torture_spinlock, flags);
192         cxt.cur_ops->flags = flags;
193         return 0;
194 }
195 
196 static void torture_lock_spin_write_unlock_irq(void)
197 __releases(torture_spinlock)
198 {
199         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
200 }
201 
202 static struct lock_torture_ops spin_lock_irq_ops = {
203         .writelock      = torture_spin_lock_write_lock_irq,
204         .write_delay    = torture_spin_lock_write_delay,
205         .task_boost     = torture_boost_dummy,
206         .writeunlock    = torture_lock_spin_write_unlock_irq,
207         .readlock       = NULL,
208         .read_delay     = NULL,
209         .readunlock     = NULL,
210         .name           = "spin_lock_irq"
211 };
212 
213 static DEFINE_RWLOCK(torture_rwlock);
214 
215 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
216 {
217         write_lock(&torture_rwlock);
218         return 0;
219 }
220 
221 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
222 {
223         const unsigned long shortdelay_us = 2;
224         const unsigned long longdelay_ms = 100;
225 
226         /* We want a short delay mostly to emulate likely code, and
227          * we want a long delay occasionally to force massive contention.
228          */
229         if (!(torture_random(trsp) %
230               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
231                 mdelay(longdelay_ms);
232         else
233                 udelay(shortdelay_us);
234 }
235 
236 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
237 {
238         write_unlock(&torture_rwlock);
239 }
240 
241 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
242 {
243         read_lock(&torture_rwlock);
244         return 0;
245 }
246 
247 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
248 {
249         const unsigned long shortdelay_us = 10;
250         const unsigned long longdelay_ms = 100;
251 
252         /* We want a short delay mostly to emulate likely code, and
253          * we want a long delay occasionally to force massive contention.
254          */
255         if (!(torture_random(trsp) %
256               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
257                 mdelay(longdelay_ms);
258         else
259                 udelay(shortdelay_us);
260 }
261 
262 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
263 {
264         read_unlock(&torture_rwlock);
265 }
266 
267 static struct lock_torture_ops rw_lock_ops = {
268         .writelock      = torture_rwlock_write_lock,
269         .write_delay    = torture_rwlock_write_delay,
270         .task_boost     = torture_boost_dummy,
271         .writeunlock    = torture_rwlock_write_unlock,
272         .readlock       = torture_rwlock_read_lock,
273         .read_delay     = torture_rwlock_read_delay,
274         .readunlock     = torture_rwlock_read_unlock,
275         .name           = "rw_lock"
276 };
277 
278 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
279 {
280         unsigned long flags;
281 
282         write_lock_irqsave(&torture_rwlock, flags);
283         cxt.cur_ops->flags = flags;
284         return 0;
285 }
286 
287 static void torture_rwlock_write_unlock_irq(void)
288 __releases(torture_rwlock)
289 {
290         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
291 }
292 
293 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
294 {
295         unsigned long flags;
296 
297         read_lock_irqsave(&torture_rwlock, flags);
298         cxt.cur_ops->flags = flags;
299         return 0;
300 }
301 
302 static void torture_rwlock_read_unlock_irq(void)
303 __releases(torture_rwlock)
304 {
305         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
306 }
307 
308 static struct lock_torture_ops rw_lock_irq_ops = {
309         .writelock      = torture_rwlock_write_lock_irq,
310         .write_delay    = torture_rwlock_write_delay,
311         .task_boost     = torture_boost_dummy,
312         .writeunlock    = torture_rwlock_write_unlock_irq,
313         .readlock       = torture_rwlock_read_lock_irq,
314         .read_delay     = torture_rwlock_read_delay,
315         .readunlock     = torture_rwlock_read_unlock_irq,
316         .name           = "rw_lock_irq"
317 };
318 
319 static DEFINE_MUTEX(torture_mutex);
320 
321 static int torture_mutex_lock(void) __acquires(torture_mutex)
322 {
323         mutex_lock(&torture_mutex);
324         return 0;
325 }
326 
327 static void torture_mutex_delay(struct torture_random_state *trsp)
328 {
329         const unsigned long longdelay_ms = 100;
330 
331         /* We want a long delay occasionally to force massive contention.  */
332         if (!(torture_random(trsp) %
333               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
334                 mdelay(longdelay_ms * 5);
335         else
336                 mdelay(longdelay_ms / 5);
337         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
338                 torture_preempt_schedule();  /* Allow test to be preempted. */
339 }
340 
341 static void torture_mutex_unlock(void) __releases(torture_mutex)
342 {
343         mutex_unlock(&torture_mutex);
344 }
345 
346 static struct lock_torture_ops mutex_lock_ops = {
347         .writelock      = torture_mutex_lock,
348         .write_delay    = torture_mutex_delay,
349         .task_boost     = torture_boost_dummy,
350         .writeunlock    = torture_mutex_unlock,
351         .readlock       = NULL,
352         .read_delay     = NULL,
353         .readunlock     = NULL,
354         .name           = "mutex_lock"
355 };
356 
357 #include <linux/ww_mutex.h>
358 static DEFINE_WD_CLASS(torture_ww_class);
359 static DEFINE_WW_MUTEX(torture_ww_mutex_0, &torture_ww_class);
360 static DEFINE_WW_MUTEX(torture_ww_mutex_1, &torture_ww_class);
361 static DEFINE_WW_MUTEX(torture_ww_mutex_2, &torture_ww_class);
362 
363 static int torture_ww_mutex_lock(void)
364 __acquires(torture_ww_mutex_0)
365 __acquires(torture_ww_mutex_1)
366 __acquires(torture_ww_mutex_2)
367 {
368         LIST_HEAD(list);
369         struct reorder_lock {
370                 struct list_head link;
371                 struct ww_mutex *lock;
372         } locks[3], *ll, *ln;
373         struct ww_acquire_ctx ctx;
374 
375         locks[0].lock = &torture_ww_mutex_0;
376         list_add(&locks[0].link, &list);
377 
378         locks[1].lock = &torture_ww_mutex_1;
379         list_add(&locks[1].link, &list);
380 
381         locks[2].lock = &torture_ww_mutex_2;
382         list_add(&locks[2].link, &list);
383 
384         ww_acquire_init(&ctx, &torture_ww_class);
385 
386         list_for_each_entry(ll, &list, link) {
387                 int err;
388 
389                 err = ww_mutex_lock(ll->lock, &ctx);
390                 if (!err)
391                         continue;
392 
393                 ln = ll;
394                 list_for_each_entry_continue_reverse(ln, &list, link)
395                         ww_mutex_unlock(ln->lock);
396 
397                 if (err != -EDEADLK)
398                         return err;
399 
400                 ww_mutex_lock_slow(ll->lock, &ctx);
401                 list_move(&ll->link, &list);
402         }
403 
404         ww_acquire_fini(&ctx);
405         return 0;
406 }
407 
408 static void torture_ww_mutex_unlock(void)
409 __releases(torture_ww_mutex_0)
410 __releases(torture_ww_mutex_1)
411 __releases(torture_ww_mutex_2)
412 {
413         ww_mutex_unlock(&torture_ww_mutex_0);
414         ww_mutex_unlock(&torture_ww_mutex_1);
415         ww_mutex_unlock(&torture_ww_mutex_2);
416 }
417 
418 static struct lock_torture_ops ww_mutex_lock_ops = {
419         .writelock      = torture_ww_mutex_lock,
420         .write_delay    = torture_mutex_delay,
421         .task_boost     = torture_boost_dummy,
422         .writeunlock    = torture_ww_mutex_unlock,
423         .readlock       = NULL,
424         .read_delay     = NULL,
425         .readunlock     = NULL,
426         .name           = "ww_mutex_lock"
427 };
428 
429 #ifdef CONFIG_RT_MUTEXES
430 static DEFINE_RT_MUTEX(torture_rtmutex);
431 
432 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
433 {
434         rt_mutex_lock(&torture_rtmutex);
435         return 0;
436 }
437 
438 static void torture_rtmutex_boost(struct torture_random_state *trsp)
439 {
440         int policy;
441         struct sched_param param;
442         const unsigned int factor = 50000; /* yes, quite arbitrary */
443 
444         if (!rt_task(current)) {
445                 /*
446                  * Boost priority once every ~50k operations. When the
447                  * task tries to take the lock, the rtmutex it will account
448                  * for the new priority, and do any corresponding pi-dance.
449                  */
450                 if (trsp && !(torture_random(trsp) %
451                               (cxt.nrealwriters_stress * factor))) {
452                         policy = SCHED_FIFO;
453                         param.sched_priority = MAX_RT_PRIO - 1;
454                 } else /* common case, do nothing */
455                         return;
456         } else {
457                 /*
458                  * The task will remain boosted for another ~500k operations,
459                  * then restored back to its original prio, and so forth.
460                  *
461                  * When @trsp is nil, we want to force-reset the task for
462                  * stopping the kthread.
463                  */
464                 if (!trsp || !(torture_random(trsp) %
465                                (cxt.nrealwriters_stress * factor * 2))) {
466                         policy = SCHED_NORMAL;
467                         param.sched_priority = 0;
468                 } else /* common case, do nothing */
469                         return;
470         }
471 
472         sched_setscheduler_nocheck(current, policy, &param);
473 }
474 
475 static void torture_rtmutex_delay(struct torture_random_state *trsp)
476 {
477         const unsigned long shortdelay_us = 2;
478         const unsigned long longdelay_ms = 100;
479 
480         /*
481          * We want a short delay mostly to emulate likely code, and
482          * we want a long delay occasionally to force massive contention.
483          */
484         if (!(torture_random(trsp) %
485               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
486                 mdelay(longdelay_ms);
487         if (!(torture_random(trsp) %
488               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
489                 udelay(shortdelay_us);
490         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
491                 torture_preempt_schedule();  /* Allow test to be preempted. */
492 }
493 
494 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
495 {
496         rt_mutex_unlock(&torture_rtmutex);
497 }
498 
499 static struct lock_torture_ops rtmutex_lock_ops = {
500         .writelock      = torture_rtmutex_lock,
501         .write_delay    = torture_rtmutex_delay,
502         .task_boost     = torture_rtmutex_boost,
503         .writeunlock    = torture_rtmutex_unlock,
504         .readlock       = NULL,
505         .read_delay     = NULL,
506         .readunlock     = NULL,
507         .name           = "rtmutex_lock"
508 };
509 #endif
510 
511 static DECLARE_RWSEM(torture_rwsem);
512 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
513 {
514         down_write(&torture_rwsem);
515         return 0;
516 }
517 
518 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
519 {
520         const unsigned long longdelay_ms = 100;
521 
522         /* We want a long delay occasionally to force massive contention.  */
523         if (!(torture_random(trsp) %
524               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
525                 mdelay(longdelay_ms * 10);
526         else
527                 mdelay(longdelay_ms / 10);
528         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
529                 torture_preempt_schedule();  /* Allow test to be preempted. */
530 }
531 
532 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
533 {
534         up_write(&torture_rwsem);
535 }
536 
537 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
538 {
539         down_read(&torture_rwsem);
540         return 0;
541 }
542 
543 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
544 {
545         const unsigned long longdelay_ms = 100;
546 
547         /* We want a long delay occasionally to force massive contention.  */
548         if (!(torture_random(trsp) %
549               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
550                 mdelay(longdelay_ms * 2);
551         else
552                 mdelay(longdelay_ms / 2);
553         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
554                 torture_preempt_schedule();  /* Allow test to be preempted. */
555 }
556 
557 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
558 {
559         up_read(&torture_rwsem);
560 }
561 
562 static struct lock_torture_ops rwsem_lock_ops = {
563         .writelock      = torture_rwsem_down_write,
564         .write_delay    = torture_rwsem_write_delay,
565         .task_boost     = torture_boost_dummy,
566         .writeunlock    = torture_rwsem_up_write,
567         .readlock       = torture_rwsem_down_read,
568         .read_delay     = torture_rwsem_read_delay,
569         .readunlock     = torture_rwsem_up_read,
570         .name           = "rwsem_lock"
571 };
572 
573 #include <linux/percpu-rwsem.h>
574 static struct percpu_rw_semaphore pcpu_rwsem;
575 
576 void torture_percpu_rwsem_init(void)
577 {
578         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
579 }
580 
581 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
582 {
583         percpu_down_write(&pcpu_rwsem);
584         return 0;
585 }
586 
587 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
588 {
589         percpu_up_write(&pcpu_rwsem);
590 }
591 
592 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
593 {
594         percpu_down_read(&pcpu_rwsem);
595         return 0;
596 }
597 
598 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
599 {
600         percpu_up_read(&pcpu_rwsem);
601 }
602 
603 static struct lock_torture_ops percpu_rwsem_lock_ops = {
604         .init           = torture_percpu_rwsem_init,
605         .writelock      = torture_percpu_rwsem_down_write,
606         .write_delay    = torture_rwsem_write_delay,
607         .task_boost     = torture_boost_dummy,
608         .writeunlock    = torture_percpu_rwsem_up_write,
609         .readlock       = torture_percpu_rwsem_down_read,
610         .read_delay     = torture_rwsem_read_delay,
611         .readunlock     = torture_percpu_rwsem_up_read,
612         .name           = "percpu_rwsem_lock"
613 };
614 
615 /*
616  * Lock torture writer kthread.  Repeatedly acquires and releases
617  * the lock, checking for duplicate acquisitions.
618  */
619 static int lock_torture_writer(void *arg)
620 {
621         struct lock_stress_stats *lwsp = arg;
622         static DEFINE_TORTURE_RANDOM(rand);
623 
624         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
625         set_user_nice(current, MAX_NICE);
626 
627         do {
628                 if ((torture_random(&rand) & 0xfffff) == 0)
629                         schedule_timeout_uninterruptible(1);
630 
631                 cxt.cur_ops->task_boost(&rand);
632                 cxt.cur_ops->writelock();
633                 if (WARN_ON_ONCE(lock_is_write_held))
634                         lwsp->n_lock_fail++;
635                 lock_is_write_held = 1;
636                 if (WARN_ON_ONCE(lock_is_read_held))
637                         lwsp->n_lock_fail++; /* rare, but... */
638 
639                 lwsp->n_lock_acquired++;
640                 cxt.cur_ops->write_delay(&rand);
641                 lock_is_write_held = 0;
642                 cxt.cur_ops->writeunlock();
643 
644                 stutter_wait("lock_torture_writer");
645         } while (!torture_must_stop());
646 
647         cxt.cur_ops->task_boost(NULL); /* reset prio */
648         torture_kthread_stopping("lock_torture_writer");
649         return 0;
650 }
651 
652 /*
653  * Lock torture reader kthread.  Repeatedly acquires and releases
654  * the reader lock.
655  */
656 static int lock_torture_reader(void *arg)
657 {
658         struct lock_stress_stats *lrsp = arg;
659         static DEFINE_TORTURE_RANDOM(rand);
660 
661         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
662         set_user_nice(current, MAX_NICE);
663 
664         do {
665                 if ((torture_random(&rand) & 0xfffff) == 0)
666                         schedule_timeout_uninterruptible(1);
667 
668                 cxt.cur_ops->readlock();
669                 lock_is_read_held = 1;
670                 if (WARN_ON_ONCE(lock_is_write_held))
671                         lrsp->n_lock_fail++; /* rare, but... */
672 
673                 lrsp->n_lock_acquired++;
674                 cxt.cur_ops->read_delay(&rand);
675                 lock_is_read_held = 0;
676                 cxt.cur_ops->readunlock();
677 
678                 stutter_wait("lock_torture_reader");
679         } while (!torture_must_stop());
680         torture_kthread_stopping("lock_torture_reader");
681         return 0;
682 }
683 
684 /*
685  * Create an lock-torture-statistics message in the specified buffer.
686  */
687 static void __torture_print_stats(char *page,
688                                   struct lock_stress_stats *statp, bool write)
689 {
690         bool fail = 0;
691         int i, n_stress;
692         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
693         long long sum = 0;
694 
695         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
696         for (i = 0; i < n_stress; i++) {
697                 if (statp[i].n_lock_fail)
698                         fail = true;
699                 sum += statp[i].n_lock_acquired;
700                 if (max < statp[i].n_lock_fail)
701                         max = statp[i].n_lock_fail;
702                 if (min > statp[i].n_lock_fail)
703                         min = statp[i].n_lock_fail;
704         }
705         page += sprintf(page,
706                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
707                         write ? "Writes" : "Reads ",
708                         sum, max, min, max / 2 > min ? "???" : "",
709                         fail, fail ? "!!!" : "");
710         if (fail)
711                 atomic_inc(&cxt.n_lock_torture_errors);
712 }
713 
714 /*
715  * Print torture statistics.  Caller must ensure that there is only one
716  * call to this function at a given time!!!  This is normally accomplished
717  * by relying on the module system to only have one copy of the module
718  * loaded, and then by giving the lock_torture_stats kthread full control
719  * (or the init/cleanup functions when lock_torture_stats thread is not
720  * running).
721  */
722 static void lock_torture_stats_print(void)
723 {
724         int size = cxt.nrealwriters_stress * 200 + 8192;
725         char *buf;
726 
727         if (cxt.cur_ops->readlock)
728                 size += cxt.nrealreaders_stress * 200 + 8192;
729 
730         buf = kmalloc(size, GFP_KERNEL);
731         if (!buf) {
732                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
733                        size);
734                 return;
735         }
736 
737         __torture_print_stats(buf, cxt.lwsa, true);
738         pr_alert("%s", buf);
739         kfree(buf);
740 
741         if (cxt.cur_ops->readlock) {
742                 buf = kmalloc(size, GFP_KERNEL);
743                 if (!buf) {
744                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
745                                size);
746                         return;
747                 }
748 
749                 __torture_print_stats(buf, cxt.lrsa, false);
750                 pr_alert("%s", buf);
751                 kfree(buf);
752         }
753 }
754 
755 /*
756  * Periodically prints torture statistics, if periodic statistics printing
757  * was specified via the stat_interval module parameter.
758  *
759  * No need to worry about fullstop here, since this one doesn't reference
760  * volatile state or register callbacks.
761  */
762 static int lock_torture_stats(void *arg)
763 {
764         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
765         do {
766                 schedule_timeout_interruptible(stat_interval * HZ);
767                 lock_torture_stats_print();
768                 torture_shutdown_absorb("lock_torture_stats");
769         } while (!torture_must_stop());
770         torture_kthread_stopping("lock_torture_stats");
771         return 0;
772 }
773 
774 static inline void
775 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
776                                 const char *tag)
777 {
778         pr_alert("%s" TORTURE_FLAG
779                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
780                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
781                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
782                  verbose, shuffle_interval, stutter, shutdown_secs,
783                  onoff_interval, onoff_holdoff);
784 }
785 
786 static void lock_torture_cleanup(void)
787 {
788         int i;
789 
790         if (torture_cleanup_begin())
791                 return;
792 
793         /*
794          * Indicates early cleanup, meaning that the test has not run,
795          * such as when passing bogus args when loading the module. As
796          * such, only perform the underlying torture-specific cleanups,
797          * and avoid anything related to locktorture.
798          */
799         if (!cxt.lwsa && !cxt.lrsa)
800                 goto end;
801 
802         if (writer_tasks) {
803                 for (i = 0; i < cxt.nrealwriters_stress; i++)
804                         torture_stop_kthread(lock_torture_writer,
805                                              writer_tasks[i]);
806                 kfree(writer_tasks);
807                 writer_tasks = NULL;
808         }
809 
810         if (reader_tasks) {
811                 for (i = 0; i < cxt.nrealreaders_stress; i++)
812                         torture_stop_kthread(lock_torture_reader,
813                                              reader_tasks[i]);
814                 kfree(reader_tasks);
815                 reader_tasks = NULL;
816         }
817 
818         torture_stop_kthread(lock_torture_stats, stats_task);
819         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
820 
821         if (atomic_read(&cxt.n_lock_torture_errors))
822                 lock_torture_print_module_parms(cxt.cur_ops,
823                                                 "End of test: FAILURE");
824         else if (torture_onoff_failures())
825                 lock_torture_print_module_parms(cxt.cur_ops,
826                                                 "End of test: LOCK_HOTPLUG");
827         else
828                 lock_torture_print_module_parms(cxt.cur_ops,
829                                                 "End of test: SUCCESS");
830 
831         kfree(cxt.lwsa);
832         cxt.lwsa = NULL;
833         kfree(cxt.lrsa);
834         cxt.lrsa = NULL;
835 
836 end:
837         torture_cleanup_end();
838 }
839 
840 static int __init lock_torture_init(void)
841 {
842         int i, j;
843         int firsterr = 0;
844         static struct lock_torture_ops *torture_ops[] = {
845                 &lock_busted_ops,
846                 &spin_lock_ops, &spin_lock_irq_ops,
847                 &rw_lock_ops, &rw_lock_irq_ops,
848                 &mutex_lock_ops,
849                 &ww_mutex_lock_ops,
850 #ifdef CONFIG_RT_MUTEXES
851                 &rtmutex_lock_ops,
852 #endif
853                 &rwsem_lock_ops,
854                 &percpu_rwsem_lock_ops,
855         };
856 
857         if (!torture_init_begin(torture_type, verbose))
858                 return -EBUSY;
859 
860         /* Process args and tell the world that the torturer is on the job. */
861         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
862                 cxt.cur_ops = torture_ops[i];
863                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
864                         break;
865         }
866         if (i == ARRAY_SIZE(torture_ops)) {
867                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
868                          torture_type);
869                 pr_alert("lock-torture types:");
870                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
871                         pr_alert(" %s", torture_ops[i]->name);
872                 pr_alert("\n");
873                 firsterr = -EINVAL;
874                 goto unwind;
875         }
876 
877         if (nwriters_stress == 0 && nreaders_stress == 0) {
878                 pr_alert("lock-torture: must run at least one locking thread\n");
879                 firsterr = -EINVAL;
880                 goto unwind;
881         }
882 
883         if (cxt.cur_ops->init)
884                 cxt.cur_ops->init();
885 
886         if (nwriters_stress >= 0)
887                 cxt.nrealwriters_stress = nwriters_stress;
888         else
889                 cxt.nrealwriters_stress = 2 * num_online_cpus();
890 
891 #ifdef CONFIG_DEBUG_MUTEXES
892         if (strncmp(torture_type, "mutex", 5) == 0)
893                 cxt.debug_lock = true;
894 #endif
895 #ifdef CONFIG_DEBUG_RT_MUTEXES
896         if (strncmp(torture_type, "rtmutex", 7) == 0)
897                 cxt.debug_lock = true;
898 #endif
899 #ifdef CONFIG_DEBUG_SPINLOCK
900         if ((strncmp(torture_type, "spin", 4) == 0) ||
901             (strncmp(torture_type, "rw_lock", 7) == 0))
902                 cxt.debug_lock = true;
903 #endif
904 
905         /* Initialize the statistics so that each run gets its own numbers. */
906         if (nwriters_stress) {
907                 lock_is_write_held = 0;
908                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
909                                          sizeof(*cxt.lwsa),
910                                          GFP_KERNEL);
911                 if (cxt.lwsa == NULL) {
912                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
913                         firsterr = -ENOMEM;
914                         goto unwind;
915                 }
916 
917                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
918                         cxt.lwsa[i].n_lock_fail = 0;
919                         cxt.lwsa[i].n_lock_acquired = 0;
920                 }
921         }
922 
923         if (cxt.cur_ops->readlock) {
924                 if (nreaders_stress >= 0)
925                         cxt.nrealreaders_stress = nreaders_stress;
926                 else {
927                         /*
928                          * By default distribute evenly the number of
929                          * readers and writers. We still run the same number
930                          * of threads as the writer-only locks default.
931                          */
932                         if (nwriters_stress < 0) /* user doesn't care */
933                                 cxt.nrealwriters_stress = num_online_cpus();
934                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
935                 }
936 
937                 if (nreaders_stress) {
938                         lock_is_read_held = 0;
939                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
940                                                  sizeof(*cxt.lrsa),
941                                                  GFP_KERNEL);
942                         if (cxt.lrsa == NULL) {
943                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
944                                 firsterr = -ENOMEM;
945                                 kfree(cxt.lwsa);
946                                 cxt.lwsa = NULL;
947                                 goto unwind;
948                         }
949 
950                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
951                                 cxt.lrsa[i].n_lock_fail = 0;
952                                 cxt.lrsa[i].n_lock_acquired = 0;
953                         }
954                 }
955         }
956 
957         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
958 
959         /* Prepare torture context. */
960         if (onoff_interval > 0) {
961                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
962                                               onoff_interval * HZ, NULL);
963                 if (firsterr)
964                         goto unwind;
965         }
966         if (shuffle_interval > 0) {
967                 firsterr = torture_shuffle_init(shuffle_interval);
968                 if (firsterr)
969                         goto unwind;
970         }
971         if (shutdown_secs > 0) {
972                 firsterr = torture_shutdown_init(shutdown_secs,
973                                                  lock_torture_cleanup);
974                 if (firsterr)
975                         goto unwind;
976         }
977         if (stutter > 0) {
978                 firsterr = torture_stutter_init(stutter, stutter);
979                 if (firsterr)
980                         goto unwind;
981         }
982 
983         if (nwriters_stress) {
984                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
985                                        sizeof(writer_tasks[0]),
986                                        GFP_KERNEL);
987                 if (writer_tasks == NULL) {
988                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
989                         firsterr = -ENOMEM;
990                         goto unwind;
991                 }
992         }
993 
994         if (cxt.cur_ops->readlock) {
995                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
996                                        sizeof(reader_tasks[0]),
997                                        GFP_KERNEL);
998                 if (reader_tasks == NULL) {
999                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
1000                         kfree(writer_tasks);
1001                         writer_tasks = NULL;
1002                         firsterr = -ENOMEM;
1003                         goto unwind;
1004                 }
1005         }
1006 
1007         /*
1008          * Create the kthreads and start torturing (oh, those poor little locks).
1009          *
1010          * TODO: Note that we interleave writers with readers, giving writers a
1011          * slight advantage, by creating its kthread first. This can be modified
1012          * for very specific needs, or even let the user choose the policy, if
1013          * ever wanted.
1014          */
1015         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1016                     j < cxt.nrealreaders_stress; i++, j++) {
1017                 if (i >= cxt.nrealwriters_stress)
1018                         goto create_reader;
1019 
1020                 /* Create writer. */
1021                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1022                                                   writer_tasks[i]);
1023                 if (firsterr)
1024                         goto unwind;
1025 
1026         create_reader:
1027                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1028                         continue;
1029                 /* Create reader. */
1030                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1031                                                   reader_tasks[j]);
1032                 if (firsterr)
1033                         goto unwind;
1034         }
1035         if (stat_interval > 0) {
1036                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1037                                                   stats_task);
1038                 if (firsterr)
1039                         goto unwind;
1040         }
1041         torture_init_end();
1042         return 0;
1043 
1044 unwind:
1045         torture_init_end();
1046         lock_torture_cleanup();
1047         return firsterr;
1048 }
1049 
1050 module_init(lock_torture_init);
1051 module_exit(lock_torture_cleanup);
1052 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp