~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/locking/locktorture.c

Version: ~ [ linux-5.11 ] ~ [ linux-5.10.17 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.99 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.176 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.221 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.257 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.257 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0+
  2 /*
  3  * Module-based torture test facility for locking
  4  *
  5  * Copyright (C) IBM Corporation, 2014
  6  *
  7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
  8  *          Davidlohr Bueso <dave@stgolabs.net>
  9  *      Based on kernel/rcu/torture.c.
 10  */
 11 
 12 #define pr_fmt(fmt) fmt
 13 
 14 #include <linux/kernel.h>
 15 #include <linux/module.h>
 16 #include <linux/kthread.h>
 17 #include <linux/sched/rt.h>
 18 #include <linux/spinlock.h>
 19 #include <linux/mutex.h>
 20 #include <linux/rwsem.h>
 21 #include <linux/smp.h>
 22 #include <linux/interrupt.h>
 23 #include <linux/sched.h>
 24 #include <uapi/linux/sched/types.h>
 25 #include <linux/rtmutex.h>
 26 #include <linux/atomic.h>
 27 #include <linux/moduleparam.h>
 28 #include <linux/delay.h>
 29 #include <linux/slab.h>
 30 #include <linux/percpu-rwsem.h>
 31 #include <linux/torture.h>
 32 
 33 MODULE_LICENSE("GPL");
 34 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
 35 
 36 torture_param(int, nwriters_stress, -1,
 37              "Number of write-locking stress-test threads");
 38 torture_param(int, nreaders_stress, -1,
 39              "Number of read-locking stress-test threads");
 40 torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)");
 41 torture_param(int, onoff_interval, 0,
 42              "Time between CPU hotplugs (s), 0=disable");
 43 torture_param(int, shuffle_interval, 3,
 44              "Number of jiffies between shuffles, 0=disable");
 45 torture_param(int, shutdown_secs, 0, "Shutdown time (j), <= zero to disable.");
 46 torture_param(int, stat_interval, 60,
 47              "Number of seconds between stats printk()s");
 48 torture_param(int, stutter, 5, "Number of jiffies to run/halt test, 0=disable");
 49 torture_param(int, verbose, 1,
 50              "Enable verbose debugging printk()s");
 51 
 52 static char *torture_type = "spin_lock";
 53 module_param(torture_type, charp, 0444);
 54 MODULE_PARM_DESC(torture_type,
 55                  "Type of lock to torture (spin_lock, spin_lock_irq, mutex_lock, ...)");
 56 
 57 static struct task_struct *stats_task;
 58 static struct task_struct **writer_tasks;
 59 static struct task_struct **reader_tasks;
 60 
 61 static bool lock_is_write_held;
 62 static bool lock_is_read_held;
 63 
 64 struct lock_stress_stats {
 65         long n_lock_fail;
 66         long n_lock_acquired;
 67 };
 68 
 69 /* Forward reference. */
 70 static void lock_torture_cleanup(void);
 71 
 72 /*
 73  * Operations vector for selecting different types of tests.
 74  */
 75 struct lock_torture_ops {
 76         void (*init)(void);
 77         int (*writelock)(void);
 78         void (*write_delay)(struct torture_random_state *trsp);
 79         void (*task_boost)(struct torture_random_state *trsp);
 80         void (*writeunlock)(void);
 81         int (*readlock)(void);
 82         void (*read_delay)(struct torture_random_state *trsp);
 83         void (*readunlock)(void);
 84 
 85         unsigned long flags; /* for irq spinlocks */
 86         const char *name;
 87 };
 88 
 89 struct lock_torture_cxt {
 90         int nrealwriters_stress;
 91         int nrealreaders_stress;
 92         bool debug_lock;
 93         atomic_t n_lock_torture_errors;
 94         struct lock_torture_ops *cur_ops;
 95         struct lock_stress_stats *lwsa; /* writer statistics */
 96         struct lock_stress_stats *lrsa; /* reader statistics */
 97 };
 98 static struct lock_torture_cxt cxt = { 0, 0, false,
 99                                        ATOMIC_INIT(0),
100                                        NULL, NULL};
101 /*
102  * Definitions for lock torture testing.
103  */
104 
105 static int torture_lock_busted_write_lock(void)
106 {
107         return 0;  /* BUGGY, do not use in real life!!! */
108 }
109 
110 static void torture_lock_busted_write_delay(struct torture_random_state *trsp)
111 {
112         const unsigned long longdelay_ms = 100;
113 
114         /* We want a long delay occasionally to force massive contention.  */
115         if (!(torture_random(trsp) %
116               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
117                 mdelay(longdelay_ms);
118         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
119                 torture_preempt_schedule();  /* Allow test to be preempted. */
120 }
121 
122 static void torture_lock_busted_write_unlock(void)
123 {
124           /* BUGGY, do not use in real life!!! */
125 }
126 
127 static void torture_boost_dummy(struct torture_random_state *trsp)
128 {
129         /* Only rtmutexes care about priority */
130 }
131 
132 static struct lock_torture_ops lock_busted_ops = {
133         .writelock      = torture_lock_busted_write_lock,
134         .write_delay    = torture_lock_busted_write_delay,
135         .task_boost     = torture_boost_dummy,
136         .writeunlock    = torture_lock_busted_write_unlock,
137         .readlock       = NULL,
138         .read_delay     = NULL,
139         .readunlock     = NULL,
140         .name           = "lock_busted"
141 };
142 
143 static DEFINE_SPINLOCK(torture_spinlock);
144 
145 static int torture_spin_lock_write_lock(void) __acquires(torture_spinlock)
146 {
147         spin_lock(&torture_spinlock);
148         return 0;
149 }
150 
151 static void torture_spin_lock_write_delay(struct torture_random_state *trsp)
152 {
153         const unsigned long shortdelay_us = 2;
154         const unsigned long longdelay_ms = 100;
155 
156         /* We want a short delay mostly to emulate likely code, and
157          * we want a long delay occasionally to force massive contention.
158          */
159         if (!(torture_random(trsp) %
160               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
161                 mdelay(longdelay_ms);
162         if (!(torture_random(trsp) %
163               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
164                 udelay(shortdelay_us);
165         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
166                 torture_preempt_schedule();  /* Allow test to be preempted. */
167 }
168 
169 static void torture_spin_lock_write_unlock(void) __releases(torture_spinlock)
170 {
171         spin_unlock(&torture_spinlock);
172 }
173 
174 static struct lock_torture_ops spin_lock_ops = {
175         .writelock      = torture_spin_lock_write_lock,
176         .write_delay    = torture_spin_lock_write_delay,
177         .task_boost     = torture_boost_dummy,
178         .writeunlock    = torture_spin_lock_write_unlock,
179         .readlock       = NULL,
180         .read_delay     = NULL,
181         .readunlock     = NULL,
182         .name           = "spin_lock"
183 };
184 
185 static int torture_spin_lock_write_lock_irq(void)
186 __acquires(torture_spinlock)
187 {
188         unsigned long flags;
189 
190         spin_lock_irqsave(&torture_spinlock, flags);
191         cxt.cur_ops->flags = flags;
192         return 0;
193 }
194 
195 static void torture_lock_spin_write_unlock_irq(void)
196 __releases(torture_spinlock)
197 {
198         spin_unlock_irqrestore(&torture_spinlock, cxt.cur_ops->flags);
199 }
200 
201 static struct lock_torture_ops spin_lock_irq_ops = {
202         .writelock      = torture_spin_lock_write_lock_irq,
203         .write_delay    = torture_spin_lock_write_delay,
204         .task_boost     = torture_boost_dummy,
205         .writeunlock    = torture_lock_spin_write_unlock_irq,
206         .readlock       = NULL,
207         .read_delay     = NULL,
208         .readunlock     = NULL,
209         .name           = "spin_lock_irq"
210 };
211 
212 static DEFINE_RWLOCK(torture_rwlock);
213 
214 static int torture_rwlock_write_lock(void) __acquires(torture_rwlock)
215 {
216         write_lock(&torture_rwlock);
217         return 0;
218 }
219 
220 static void torture_rwlock_write_delay(struct torture_random_state *trsp)
221 {
222         const unsigned long shortdelay_us = 2;
223         const unsigned long longdelay_ms = 100;
224 
225         /* We want a short delay mostly to emulate likely code, and
226          * we want a long delay occasionally to force massive contention.
227          */
228         if (!(torture_random(trsp) %
229               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
230                 mdelay(longdelay_ms);
231         else
232                 udelay(shortdelay_us);
233 }
234 
235 static void torture_rwlock_write_unlock(void) __releases(torture_rwlock)
236 {
237         write_unlock(&torture_rwlock);
238 }
239 
240 static int torture_rwlock_read_lock(void) __acquires(torture_rwlock)
241 {
242         read_lock(&torture_rwlock);
243         return 0;
244 }
245 
246 static void torture_rwlock_read_delay(struct torture_random_state *trsp)
247 {
248         const unsigned long shortdelay_us = 10;
249         const unsigned long longdelay_ms = 100;
250 
251         /* We want a short delay mostly to emulate likely code, and
252          * we want a long delay occasionally to force massive contention.
253          */
254         if (!(torture_random(trsp) %
255               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
256                 mdelay(longdelay_ms);
257         else
258                 udelay(shortdelay_us);
259 }
260 
261 static void torture_rwlock_read_unlock(void) __releases(torture_rwlock)
262 {
263         read_unlock(&torture_rwlock);
264 }
265 
266 static struct lock_torture_ops rw_lock_ops = {
267         .writelock      = torture_rwlock_write_lock,
268         .write_delay    = torture_rwlock_write_delay,
269         .task_boost     = torture_boost_dummy,
270         .writeunlock    = torture_rwlock_write_unlock,
271         .readlock       = torture_rwlock_read_lock,
272         .read_delay     = torture_rwlock_read_delay,
273         .readunlock     = torture_rwlock_read_unlock,
274         .name           = "rw_lock"
275 };
276 
277 static int torture_rwlock_write_lock_irq(void) __acquires(torture_rwlock)
278 {
279         unsigned long flags;
280 
281         write_lock_irqsave(&torture_rwlock, flags);
282         cxt.cur_ops->flags = flags;
283         return 0;
284 }
285 
286 static void torture_rwlock_write_unlock_irq(void)
287 __releases(torture_rwlock)
288 {
289         write_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
290 }
291 
292 static int torture_rwlock_read_lock_irq(void) __acquires(torture_rwlock)
293 {
294         unsigned long flags;
295 
296         read_lock_irqsave(&torture_rwlock, flags);
297         cxt.cur_ops->flags = flags;
298         return 0;
299 }
300 
301 static void torture_rwlock_read_unlock_irq(void)
302 __releases(torture_rwlock)
303 {
304         read_unlock_irqrestore(&torture_rwlock, cxt.cur_ops->flags);
305 }
306 
307 static struct lock_torture_ops rw_lock_irq_ops = {
308         .writelock      = torture_rwlock_write_lock_irq,
309         .write_delay    = torture_rwlock_write_delay,
310         .task_boost     = torture_boost_dummy,
311         .writeunlock    = torture_rwlock_write_unlock_irq,
312         .readlock       = torture_rwlock_read_lock_irq,
313         .read_delay     = torture_rwlock_read_delay,
314         .readunlock     = torture_rwlock_read_unlock_irq,
315         .name           = "rw_lock_irq"
316 };
317 
318 static DEFINE_MUTEX(torture_mutex);
319 
320 static int torture_mutex_lock(void) __acquires(torture_mutex)
321 {
322         mutex_lock(&torture_mutex);
323         return 0;
324 }
325 
326 static void torture_mutex_delay(struct torture_random_state *trsp)
327 {
328         const unsigned long longdelay_ms = 100;
329 
330         /* We want a long delay occasionally to force massive contention.  */
331         if (!(torture_random(trsp) %
332               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
333                 mdelay(longdelay_ms * 5);
334         else
335                 mdelay(longdelay_ms / 5);
336         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
337                 torture_preempt_schedule();  /* Allow test to be preempted. */
338 }
339 
340 static void torture_mutex_unlock(void) __releases(torture_mutex)
341 {
342         mutex_unlock(&torture_mutex);
343 }
344 
345 static struct lock_torture_ops mutex_lock_ops = {
346         .writelock      = torture_mutex_lock,
347         .write_delay    = torture_mutex_delay,
348         .task_boost     = torture_boost_dummy,
349         .writeunlock    = torture_mutex_unlock,
350         .readlock       = NULL,
351         .read_delay     = NULL,
352         .readunlock     = NULL,
353         .name           = "mutex_lock"
354 };
355 
356 #include <linux/ww_mutex.h>
357 static DEFINE_WD_CLASS(torture_ww_class);
358 static DEFINE_WW_MUTEX(torture_ww_mutex_0, &torture_ww_class);
359 static DEFINE_WW_MUTEX(torture_ww_mutex_1, &torture_ww_class);
360 static DEFINE_WW_MUTEX(torture_ww_mutex_2, &torture_ww_class);
361 
362 static int torture_ww_mutex_lock(void)
363 __acquires(torture_ww_mutex_0)
364 __acquires(torture_ww_mutex_1)
365 __acquires(torture_ww_mutex_2)
366 {
367         LIST_HEAD(list);
368         struct reorder_lock {
369                 struct list_head link;
370                 struct ww_mutex *lock;
371         } locks[3], *ll, *ln;
372         struct ww_acquire_ctx ctx;
373 
374         locks[0].lock = &torture_ww_mutex_0;
375         list_add(&locks[0].link, &list);
376 
377         locks[1].lock = &torture_ww_mutex_1;
378         list_add(&locks[1].link, &list);
379 
380         locks[2].lock = &torture_ww_mutex_2;
381         list_add(&locks[2].link, &list);
382 
383         ww_acquire_init(&ctx, &torture_ww_class);
384 
385         list_for_each_entry(ll, &list, link) {
386                 int err;
387 
388                 err = ww_mutex_lock(ll->lock, &ctx);
389                 if (!err)
390                         continue;
391 
392                 ln = ll;
393                 list_for_each_entry_continue_reverse(ln, &list, link)
394                         ww_mutex_unlock(ln->lock);
395 
396                 if (err != -EDEADLK)
397                         return err;
398 
399                 ww_mutex_lock_slow(ll->lock, &ctx);
400                 list_move(&ll->link, &list);
401         }
402 
403         ww_acquire_fini(&ctx);
404         return 0;
405 }
406 
407 static void torture_ww_mutex_unlock(void)
408 __releases(torture_ww_mutex_0)
409 __releases(torture_ww_mutex_1)
410 __releases(torture_ww_mutex_2)
411 {
412         ww_mutex_unlock(&torture_ww_mutex_0);
413         ww_mutex_unlock(&torture_ww_mutex_1);
414         ww_mutex_unlock(&torture_ww_mutex_2);
415 }
416 
417 static struct lock_torture_ops ww_mutex_lock_ops = {
418         .writelock      = torture_ww_mutex_lock,
419         .write_delay    = torture_mutex_delay,
420         .task_boost     = torture_boost_dummy,
421         .writeunlock    = torture_ww_mutex_unlock,
422         .readlock       = NULL,
423         .read_delay     = NULL,
424         .readunlock     = NULL,
425         .name           = "ww_mutex_lock"
426 };
427 
428 #ifdef CONFIG_RT_MUTEXES
429 static DEFINE_RT_MUTEX(torture_rtmutex);
430 
431 static int torture_rtmutex_lock(void) __acquires(torture_rtmutex)
432 {
433         rt_mutex_lock(&torture_rtmutex);
434         return 0;
435 }
436 
437 static void torture_rtmutex_boost(struct torture_random_state *trsp)
438 {
439         int policy;
440         struct sched_param param;
441         const unsigned int factor = 50000; /* yes, quite arbitrary */
442 
443         if (!rt_task(current)) {
444                 /*
445                  * Boost priority once every ~50k operations. When the
446                  * task tries to take the lock, the rtmutex it will account
447                  * for the new priority, and do any corresponding pi-dance.
448                  */
449                 if (trsp && !(torture_random(trsp) %
450                               (cxt.nrealwriters_stress * factor))) {
451                         policy = SCHED_FIFO;
452                         param.sched_priority = MAX_RT_PRIO - 1;
453                 } else /* common case, do nothing */
454                         return;
455         } else {
456                 /*
457                  * The task will remain boosted for another ~500k operations,
458                  * then restored back to its original prio, and so forth.
459                  *
460                  * When @trsp is nil, we want to force-reset the task for
461                  * stopping the kthread.
462                  */
463                 if (!trsp || !(torture_random(trsp) %
464                                (cxt.nrealwriters_stress * factor * 2))) {
465                         policy = SCHED_NORMAL;
466                         param.sched_priority = 0;
467                 } else /* common case, do nothing */
468                         return;
469         }
470 
471         sched_setscheduler_nocheck(current, policy, &param);
472 }
473 
474 static void torture_rtmutex_delay(struct torture_random_state *trsp)
475 {
476         const unsigned long shortdelay_us = 2;
477         const unsigned long longdelay_ms = 100;
478 
479         /*
480          * We want a short delay mostly to emulate likely code, and
481          * we want a long delay occasionally to force massive contention.
482          */
483         if (!(torture_random(trsp) %
484               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
485                 mdelay(longdelay_ms);
486         if (!(torture_random(trsp) %
487               (cxt.nrealwriters_stress * 2 * shortdelay_us)))
488                 udelay(shortdelay_us);
489         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
490                 torture_preempt_schedule();  /* Allow test to be preempted. */
491 }
492 
493 static void torture_rtmutex_unlock(void) __releases(torture_rtmutex)
494 {
495         rt_mutex_unlock(&torture_rtmutex);
496 }
497 
498 static struct lock_torture_ops rtmutex_lock_ops = {
499         .writelock      = torture_rtmutex_lock,
500         .write_delay    = torture_rtmutex_delay,
501         .task_boost     = torture_rtmutex_boost,
502         .writeunlock    = torture_rtmutex_unlock,
503         .readlock       = NULL,
504         .read_delay     = NULL,
505         .readunlock     = NULL,
506         .name           = "rtmutex_lock"
507 };
508 #endif
509 
510 static DECLARE_RWSEM(torture_rwsem);
511 static int torture_rwsem_down_write(void) __acquires(torture_rwsem)
512 {
513         down_write(&torture_rwsem);
514         return 0;
515 }
516 
517 static void torture_rwsem_write_delay(struct torture_random_state *trsp)
518 {
519         const unsigned long longdelay_ms = 100;
520 
521         /* We want a long delay occasionally to force massive contention.  */
522         if (!(torture_random(trsp) %
523               (cxt.nrealwriters_stress * 2000 * longdelay_ms)))
524                 mdelay(longdelay_ms * 10);
525         else
526                 mdelay(longdelay_ms / 10);
527         if (!(torture_random(trsp) % (cxt.nrealwriters_stress * 20000)))
528                 torture_preempt_schedule();  /* Allow test to be preempted. */
529 }
530 
531 static void torture_rwsem_up_write(void) __releases(torture_rwsem)
532 {
533         up_write(&torture_rwsem);
534 }
535 
536 static int torture_rwsem_down_read(void) __acquires(torture_rwsem)
537 {
538         down_read(&torture_rwsem);
539         return 0;
540 }
541 
542 static void torture_rwsem_read_delay(struct torture_random_state *trsp)
543 {
544         const unsigned long longdelay_ms = 100;
545 
546         /* We want a long delay occasionally to force massive contention.  */
547         if (!(torture_random(trsp) %
548               (cxt.nrealreaders_stress * 2000 * longdelay_ms)))
549                 mdelay(longdelay_ms * 2);
550         else
551                 mdelay(longdelay_ms / 2);
552         if (!(torture_random(trsp) % (cxt.nrealreaders_stress * 20000)))
553                 torture_preempt_schedule();  /* Allow test to be preempted. */
554 }
555 
556 static void torture_rwsem_up_read(void) __releases(torture_rwsem)
557 {
558         up_read(&torture_rwsem);
559 }
560 
561 static struct lock_torture_ops rwsem_lock_ops = {
562         .writelock      = torture_rwsem_down_write,
563         .write_delay    = torture_rwsem_write_delay,
564         .task_boost     = torture_boost_dummy,
565         .writeunlock    = torture_rwsem_up_write,
566         .readlock       = torture_rwsem_down_read,
567         .read_delay     = torture_rwsem_read_delay,
568         .readunlock     = torture_rwsem_up_read,
569         .name           = "rwsem_lock"
570 };
571 
572 #include <linux/percpu-rwsem.h>
573 static struct percpu_rw_semaphore pcpu_rwsem;
574 
575 void torture_percpu_rwsem_init(void)
576 {
577         BUG_ON(percpu_init_rwsem(&pcpu_rwsem));
578 }
579 
580 static int torture_percpu_rwsem_down_write(void) __acquires(pcpu_rwsem)
581 {
582         percpu_down_write(&pcpu_rwsem);
583         return 0;
584 }
585 
586 static void torture_percpu_rwsem_up_write(void) __releases(pcpu_rwsem)
587 {
588         percpu_up_write(&pcpu_rwsem);
589 }
590 
591 static int torture_percpu_rwsem_down_read(void) __acquires(pcpu_rwsem)
592 {
593         percpu_down_read(&pcpu_rwsem);
594         return 0;
595 }
596 
597 static void torture_percpu_rwsem_up_read(void) __releases(pcpu_rwsem)
598 {
599         percpu_up_read(&pcpu_rwsem);
600 }
601 
602 static struct lock_torture_ops percpu_rwsem_lock_ops = {
603         .init           = torture_percpu_rwsem_init,
604         .writelock      = torture_percpu_rwsem_down_write,
605         .write_delay    = torture_rwsem_write_delay,
606         .task_boost     = torture_boost_dummy,
607         .writeunlock    = torture_percpu_rwsem_up_write,
608         .readlock       = torture_percpu_rwsem_down_read,
609         .read_delay     = torture_rwsem_read_delay,
610         .readunlock     = torture_percpu_rwsem_up_read,
611         .name           = "percpu_rwsem_lock"
612 };
613 
614 /*
615  * Lock torture writer kthread.  Repeatedly acquires and releases
616  * the lock, checking for duplicate acquisitions.
617  */
618 static int lock_torture_writer(void *arg)
619 {
620         struct lock_stress_stats *lwsp = arg;
621         static DEFINE_TORTURE_RANDOM(rand);
622 
623         VERBOSE_TOROUT_STRING("lock_torture_writer task started");
624         set_user_nice(current, MAX_NICE);
625 
626         do {
627                 if ((torture_random(&rand) & 0xfffff) == 0)
628                         schedule_timeout_uninterruptible(1);
629 
630                 cxt.cur_ops->task_boost(&rand);
631                 cxt.cur_ops->writelock();
632                 if (WARN_ON_ONCE(lock_is_write_held))
633                         lwsp->n_lock_fail++;
634                 lock_is_write_held = 1;
635                 if (WARN_ON_ONCE(lock_is_read_held))
636                         lwsp->n_lock_fail++; /* rare, but... */
637 
638                 lwsp->n_lock_acquired++;
639                 cxt.cur_ops->write_delay(&rand);
640                 lock_is_write_held = 0;
641                 cxt.cur_ops->writeunlock();
642 
643                 stutter_wait("lock_torture_writer");
644         } while (!torture_must_stop());
645 
646         cxt.cur_ops->task_boost(NULL); /* reset prio */
647         torture_kthread_stopping("lock_torture_writer");
648         return 0;
649 }
650 
651 /*
652  * Lock torture reader kthread.  Repeatedly acquires and releases
653  * the reader lock.
654  */
655 static int lock_torture_reader(void *arg)
656 {
657         struct lock_stress_stats *lrsp = arg;
658         static DEFINE_TORTURE_RANDOM(rand);
659 
660         VERBOSE_TOROUT_STRING("lock_torture_reader task started");
661         set_user_nice(current, MAX_NICE);
662 
663         do {
664                 if ((torture_random(&rand) & 0xfffff) == 0)
665                         schedule_timeout_uninterruptible(1);
666 
667                 cxt.cur_ops->readlock();
668                 lock_is_read_held = 1;
669                 if (WARN_ON_ONCE(lock_is_write_held))
670                         lrsp->n_lock_fail++; /* rare, but... */
671 
672                 lrsp->n_lock_acquired++;
673                 cxt.cur_ops->read_delay(&rand);
674                 lock_is_read_held = 0;
675                 cxt.cur_ops->readunlock();
676 
677                 stutter_wait("lock_torture_reader");
678         } while (!torture_must_stop());
679         torture_kthread_stopping("lock_torture_reader");
680         return 0;
681 }
682 
683 /*
684  * Create an lock-torture-statistics message in the specified buffer.
685  */
686 static void __torture_print_stats(char *page,
687                                   struct lock_stress_stats *statp, bool write)
688 {
689         bool fail = 0;
690         int i, n_stress;
691         long max = 0, min = statp ? statp[0].n_lock_acquired : 0;
692         long long sum = 0;
693 
694         n_stress = write ? cxt.nrealwriters_stress : cxt.nrealreaders_stress;
695         for (i = 0; i < n_stress; i++) {
696                 if (statp[i].n_lock_fail)
697                         fail = true;
698                 sum += statp[i].n_lock_acquired;
699                 if (max < statp[i].n_lock_fail)
700                         max = statp[i].n_lock_fail;
701                 if (min > statp[i].n_lock_fail)
702                         min = statp[i].n_lock_fail;
703         }
704         page += sprintf(page,
705                         "%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
706                         write ? "Writes" : "Reads ",
707                         sum, max, min, max / 2 > min ? "???" : "",
708                         fail, fail ? "!!!" : "");
709         if (fail)
710                 atomic_inc(&cxt.n_lock_torture_errors);
711 }
712 
713 /*
714  * Print torture statistics.  Caller must ensure that there is only one
715  * call to this function at a given time!!!  This is normally accomplished
716  * by relying on the module system to only have one copy of the module
717  * loaded, and then by giving the lock_torture_stats kthread full control
718  * (or the init/cleanup functions when lock_torture_stats thread is not
719  * running).
720  */
721 static void lock_torture_stats_print(void)
722 {
723         int size = cxt.nrealwriters_stress * 200 + 8192;
724         char *buf;
725 
726         if (cxt.cur_ops->readlock)
727                 size += cxt.nrealreaders_stress * 200 + 8192;
728 
729         buf = kmalloc(size, GFP_KERNEL);
730         if (!buf) {
731                 pr_err("lock_torture_stats_print: Out of memory, need: %d",
732                        size);
733                 return;
734         }
735 
736         __torture_print_stats(buf, cxt.lwsa, true);
737         pr_alert("%s", buf);
738         kfree(buf);
739 
740         if (cxt.cur_ops->readlock) {
741                 buf = kmalloc(size, GFP_KERNEL);
742                 if (!buf) {
743                         pr_err("lock_torture_stats_print: Out of memory, need: %d",
744                                size);
745                         return;
746                 }
747 
748                 __torture_print_stats(buf, cxt.lrsa, false);
749                 pr_alert("%s", buf);
750                 kfree(buf);
751         }
752 }
753 
754 /*
755  * Periodically prints torture statistics, if periodic statistics printing
756  * was specified via the stat_interval module parameter.
757  *
758  * No need to worry about fullstop here, since this one doesn't reference
759  * volatile state or register callbacks.
760  */
761 static int lock_torture_stats(void *arg)
762 {
763         VERBOSE_TOROUT_STRING("lock_torture_stats task started");
764         do {
765                 schedule_timeout_interruptible(stat_interval * HZ);
766                 lock_torture_stats_print();
767                 torture_shutdown_absorb("lock_torture_stats");
768         } while (!torture_must_stop());
769         torture_kthread_stopping("lock_torture_stats");
770         return 0;
771 }
772 
773 static inline void
774 lock_torture_print_module_parms(struct lock_torture_ops *cur_ops,
775                                 const char *tag)
776 {
777         pr_alert("%s" TORTURE_FLAG
778                  "--- %s%s: nwriters_stress=%d nreaders_stress=%d stat_interval=%d verbose=%d shuffle_interval=%d stutter=%d shutdown_secs=%d onoff_interval=%d onoff_holdoff=%d\n",
779                  torture_type, tag, cxt.debug_lock ? " [debug]": "",
780                  cxt.nrealwriters_stress, cxt.nrealreaders_stress, stat_interval,
781                  verbose, shuffle_interval, stutter, shutdown_secs,
782                  onoff_interval, onoff_holdoff);
783 }
784 
785 static void lock_torture_cleanup(void)
786 {
787         int i;
788 
789         if (torture_cleanup_begin())
790                 return;
791 
792         /*
793          * Indicates early cleanup, meaning that the test has not run,
794          * such as when passing bogus args when loading the module. As
795          * such, only perform the underlying torture-specific cleanups,
796          * and avoid anything related to locktorture.
797          */
798         if (!cxt.lwsa && !cxt.lrsa)
799                 goto end;
800 
801         if (writer_tasks) {
802                 for (i = 0; i < cxt.nrealwriters_stress; i++)
803                         torture_stop_kthread(lock_torture_writer,
804                                              writer_tasks[i]);
805                 kfree(writer_tasks);
806                 writer_tasks = NULL;
807         }
808 
809         if (reader_tasks) {
810                 for (i = 0; i < cxt.nrealreaders_stress; i++)
811                         torture_stop_kthread(lock_torture_reader,
812                                              reader_tasks[i]);
813                 kfree(reader_tasks);
814                 reader_tasks = NULL;
815         }
816 
817         torture_stop_kthread(lock_torture_stats, stats_task);
818         lock_torture_stats_print();  /* -After- the stats thread is stopped! */
819 
820         if (atomic_read(&cxt.n_lock_torture_errors))
821                 lock_torture_print_module_parms(cxt.cur_ops,
822                                                 "End of test: FAILURE");
823         else if (torture_onoff_failures())
824                 lock_torture_print_module_parms(cxt.cur_ops,
825                                                 "End of test: LOCK_HOTPLUG");
826         else
827                 lock_torture_print_module_parms(cxt.cur_ops,
828                                                 "End of test: SUCCESS");
829 
830         kfree(cxt.lwsa);
831         cxt.lwsa = NULL;
832         kfree(cxt.lrsa);
833         cxt.lrsa = NULL;
834 
835 end:
836         torture_cleanup_end();
837 }
838 
839 static int __init lock_torture_init(void)
840 {
841         int i, j;
842         int firsterr = 0;
843         static struct lock_torture_ops *torture_ops[] = {
844                 &lock_busted_ops,
845                 &spin_lock_ops, &spin_lock_irq_ops,
846                 &rw_lock_ops, &rw_lock_irq_ops,
847                 &mutex_lock_ops,
848                 &ww_mutex_lock_ops,
849 #ifdef CONFIG_RT_MUTEXES
850                 &rtmutex_lock_ops,
851 #endif
852                 &rwsem_lock_ops,
853                 &percpu_rwsem_lock_ops,
854         };
855 
856         if (!torture_init_begin(torture_type, verbose))
857                 return -EBUSY;
858 
859         /* Process args and tell the world that the torturer is on the job. */
860         for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
861                 cxt.cur_ops = torture_ops[i];
862                 if (strcmp(torture_type, cxt.cur_ops->name) == 0)
863                         break;
864         }
865         if (i == ARRAY_SIZE(torture_ops)) {
866                 pr_alert("lock-torture: invalid torture type: \"%s\"\n",
867                          torture_type);
868                 pr_alert("lock-torture types:");
869                 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
870                         pr_alert(" %s", torture_ops[i]->name);
871                 pr_alert("\n");
872                 firsterr = -EINVAL;
873                 goto unwind;
874         }
875 
876         if (nwriters_stress == 0 && nreaders_stress == 0) {
877                 pr_alert("lock-torture: must run at least one locking thread\n");
878                 firsterr = -EINVAL;
879                 goto unwind;
880         }
881 
882         if (cxt.cur_ops->init)
883                 cxt.cur_ops->init();
884 
885         if (nwriters_stress >= 0)
886                 cxt.nrealwriters_stress = nwriters_stress;
887         else
888                 cxt.nrealwriters_stress = 2 * num_online_cpus();
889 
890 #ifdef CONFIG_DEBUG_MUTEXES
891         if (str_has_prefix(torture_type, "mutex"))
892                 cxt.debug_lock = true;
893 #endif
894 #ifdef CONFIG_DEBUG_RT_MUTEXES
895         if (str_has_prefix(torture_type, "rtmutex"))
896                 cxt.debug_lock = true;
897 #endif
898 #ifdef CONFIG_DEBUG_SPINLOCK
899         if ((str_has_prefix(torture_type, "spin")) ||
900             (str_has_prefix(torture_type, "rw_lock")))
901                 cxt.debug_lock = true;
902 #endif
903 
904         /* Initialize the statistics so that each run gets its own numbers. */
905         if (nwriters_stress) {
906                 lock_is_write_held = 0;
907                 cxt.lwsa = kmalloc_array(cxt.nrealwriters_stress,
908                                          sizeof(*cxt.lwsa),
909                                          GFP_KERNEL);
910                 if (cxt.lwsa == NULL) {
911                         VERBOSE_TOROUT_STRING("cxt.lwsa: Out of memory");
912                         firsterr = -ENOMEM;
913                         goto unwind;
914                 }
915 
916                 for (i = 0; i < cxt.nrealwriters_stress; i++) {
917                         cxt.lwsa[i].n_lock_fail = 0;
918                         cxt.lwsa[i].n_lock_acquired = 0;
919                 }
920         }
921 
922         if (cxt.cur_ops->readlock) {
923                 if (nreaders_stress >= 0)
924                         cxt.nrealreaders_stress = nreaders_stress;
925                 else {
926                         /*
927                          * By default distribute evenly the number of
928                          * readers and writers. We still run the same number
929                          * of threads as the writer-only locks default.
930                          */
931                         if (nwriters_stress < 0) /* user doesn't care */
932                                 cxt.nrealwriters_stress = num_online_cpus();
933                         cxt.nrealreaders_stress = cxt.nrealwriters_stress;
934                 }
935 
936                 if (nreaders_stress) {
937                         lock_is_read_held = 0;
938                         cxt.lrsa = kmalloc_array(cxt.nrealreaders_stress,
939                                                  sizeof(*cxt.lrsa),
940                                                  GFP_KERNEL);
941                         if (cxt.lrsa == NULL) {
942                                 VERBOSE_TOROUT_STRING("cxt.lrsa: Out of memory");
943                                 firsterr = -ENOMEM;
944                                 kfree(cxt.lwsa);
945                                 cxt.lwsa = NULL;
946                                 goto unwind;
947                         }
948 
949                         for (i = 0; i < cxt.nrealreaders_stress; i++) {
950                                 cxt.lrsa[i].n_lock_fail = 0;
951                                 cxt.lrsa[i].n_lock_acquired = 0;
952                         }
953                 }
954         }
955 
956         lock_torture_print_module_parms(cxt.cur_ops, "Start of test");
957 
958         /* Prepare torture context. */
959         if (onoff_interval > 0) {
960                 firsterr = torture_onoff_init(onoff_holdoff * HZ,
961                                               onoff_interval * HZ, NULL);
962                 if (firsterr)
963                         goto unwind;
964         }
965         if (shuffle_interval > 0) {
966                 firsterr = torture_shuffle_init(shuffle_interval);
967                 if (firsterr)
968                         goto unwind;
969         }
970         if (shutdown_secs > 0) {
971                 firsterr = torture_shutdown_init(shutdown_secs,
972                                                  lock_torture_cleanup);
973                 if (firsterr)
974                         goto unwind;
975         }
976         if (stutter > 0) {
977                 firsterr = torture_stutter_init(stutter, stutter);
978                 if (firsterr)
979                         goto unwind;
980         }
981 
982         if (nwriters_stress) {
983                 writer_tasks = kcalloc(cxt.nrealwriters_stress,
984                                        sizeof(writer_tasks[0]),
985                                        GFP_KERNEL);
986                 if (writer_tasks == NULL) {
987                         VERBOSE_TOROUT_ERRSTRING("writer_tasks: Out of memory");
988                         firsterr = -ENOMEM;
989                         goto unwind;
990                 }
991         }
992 
993         if (cxt.cur_ops->readlock) {
994                 reader_tasks = kcalloc(cxt.nrealreaders_stress,
995                                        sizeof(reader_tasks[0]),
996                                        GFP_KERNEL);
997                 if (reader_tasks == NULL) {
998                         VERBOSE_TOROUT_ERRSTRING("reader_tasks: Out of memory");
999                         kfree(writer_tasks);
1000                         writer_tasks = NULL;
1001                         firsterr = -ENOMEM;
1002                         goto unwind;
1003                 }
1004         }
1005 
1006         /*
1007          * Create the kthreads and start torturing (oh, those poor little locks).
1008          *
1009          * TODO: Note that we interleave writers with readers, giving writers a
1010          * slight advantage, by creating its kthread first. This can be modified
1011          * for very specific needs, or even let the user choose the policy, if
1012          * ever wanted.
1013          */
1014         for (i = 0, j = 0; i < cxt.nrealwriters_stress ||
1015                     j < cxt.nrealreaders_stress; i++, j++) {
1016                 if (i >= cxt.nrealwriters_stress)
1017                         goto create_reader;
1018 
1019                 /* Create writer. */
1020                 firsterr = torture_create_kthread(lock_torture_writer, &cxt.lwsa[i],
1021                                                   writer_tasks[i]);
1022                 if (firsterr)
1023                         goto unwind;
1024 
1025         create_reader:
1026                 if (cxt.cur_ops->readlock == NULL || (j >= cxt.nrealreaders_stress))
1027                         continue;
1028                 /* Create reader. */
1029                 firsterr = torture_create_kthread(lock_torture_reader, &cxt.lrsa[j],
1030                                                   reader_tasks[j]);
1031                 if (firsterr)
1032                         goto unwind;
1033         }
1034         if (stat_interval > 0) {
1035                 firsterr = torture_create_kthread(lock_torture_stats, NULL,
1036                                                   stats_task);
1037                 if (firsterr)
1038                         goto unwind;
1039         }
1040         torture_init_end();
1041         return 0;
1042 
1043 unwind:
1044         torture_init_end();
1045         lock_torture_cleanup();
1046         return firsterr;
1047 }
1048 
1049 module_init(lock_torture_init);
1050 module_exit(lock_torture_cleanup);
1051 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp