1 /*
2 * Copyright 2005, Red Hat, Inc., Ingo Molnar
3 * Released under the General Public License (GPL).
4 *
5 * This file contains the spinlock/rwlock implementations for
6 * DEBUG_SPINLOCK.
7 */
8
9 #include <linux/spinlock.h>
10 #include <linux/nmi.h>
11 #include <linux/interrupt.h>
12 #include <linux/debug_locks.h>
13 #include <linux/delay.h>
14 #include <linux/export.h>
15
16 void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
17 struct lock_class_key *key, short inner)
18 {
19 #ifdef CONFIG_DEBUG_LOCK_ALLOC
20 /*
21 * Make sure we are not reinitializing a held lock:
22 */
23 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
24 lockdep_init_map_wait(&lock->dep_map, name, key, 0, inner);
25 #endif
26 lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
27 lock->magic = SPINLOCK_MAGIC;
28 lock->owner = SPINLOCK_OWNER_INIT;
29 lock->owner_cpu = -1;
30 }
31
32 EXPORT_SYMBOL(__raw_spin_lock_init);
33
34 #ifndef CONFIG_PREEMPT_RT
35 void __rwlock_init(rwlock_t *lock, const char *name,
36 struct lock_class_key *key)
37 {
38 #ifdef CONFIG_DEBUG_LOCK_ALLOC
39 /*
40 * Make sure we are not reinitializing a held lock:
41 */
42 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
43 lockdep_init_map_wait(&lock->dep_map, name, key, 0, LD_WAIT_CONFIG);
44 #endif
45 lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
46 lock->magic = RWLOCK_MAGIC;
47 lock->owner = SPINLOCK_OWNER_INIT;
48 lock->owner_cpu = -1;
49 }
50
51 EXPORT_SYMBOL(__rwlock_init);
52 #endif
53
54 static void spin_dump(raw_spinlock_t *lock, const char *msg)
55 {
56 struct task_struct *owner = READ_ONCE(lock->owner);
57
58 if (owner == SPINLOCK_OWNER_INIT)
59 owner = NULL;
60 printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
61 msg, raw_smp_processor_id(),
62 current->comm, task_pid_nr(current));
63 printk(KERN_EMERG " lock: %pS, .magic: %08x, .owner: %s/%d, "
64 ".owner_cpu: %d\n",
65 lock, READ_ONCE(lock->magic),
66 owner ? owner->comm : "<none>",
67 owner ? task_pid_nr(owner) : -1,
68 READ_ONCE(lock->owner_cpu));
69 dump_stack();
70 }
71
72 static void spin_bug(raw_spinlock_t *lock, const char *msg)
73 {
74 if (!debug_locks_off())
75 return;
76
77 spin_dump(lock, msg);
78 }
79
80 #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
81
82 static inline void
83 debug_spin_lock_before(raw_spinlock_t *lock)
84 {
85 SPIN_BUG_ON(READ_ONCE(lock->magic) != SPINLOCK_MAGIC, lock, "bad magic");
86 SPIN_BUG_ON(READ_ONCE(lock->owner) == current, lock, "recursion");
87 SPIN_BUG_ON(READ_ONCE(lock->owner_cpu) == raw_smp_processor_id(),
88 lock, "cpu recursion");
89 }
90
91 static inline void debug_spin_lock_after(raw_spinlock_t *lock)
92 {
93 WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
94 WRITE_ONCE(lock->owner, current);
95 }
96
97 static inline void debug_spin_unlock(raw_spinlock_t *lock)
98 {
99 SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
100 SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
101 SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
102 SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
103 lock, "wrong CPU");
104 WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
105 WRITE_ONCE(lock->owner_cpu, -1);
106 }
107
108 /*
109 * We are now relying on the NMI watchdog to detect lockup instead of doing
110 * the detection here with an unfair lock which can cause problem of its own.
111 */
112 void do_raw_spin_lock(raw_spinlock_t *lock)
113 {
114 debug_spin_lock_before(lock);
115 arch_spin_lock(&lock->raw_lock);
116 mmiowb_spin_lock();
117 debug_spin_lock_after(lock);
118 }
119
120 int do_raw_spin_trylock(raw_spinlock_t *lock)
121 {
122 int ret = arch_spin_trylock(&lock->raw_lock);
123
124 if (ret) {
125 mmiowb_spin_lock();
126 debug_spin_lock_after(lock);
127 }
128 #ifndef CONFIG_SMP
129 /*
130 * Must not happen on UP:
131 */
132 SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
133 #endif
134 return ret;
135 }
136
137 void do_raw_spin_unlock(raw_spinlock_t *lock)
138 {
139 mmiowb_spin_unlock();
140 debug_spin_unlock(lock);
141 arch_spin_unlock(&lock->raw_lock);
142 }
143
144 #ifndef CONFIG_PREEMPT_RT
145 static void rwlock_bug(rwlock_t *lock, const char *msg)
146 {
147 if (!debug_locks_off())
148 return;
149
150 printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
151 msg, raw_smp_processor_id(), current->comm,
152 task_pid_nr(current), lock);
153 dump_stack();
154 }
155
156 #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
157
158 void do_raw_read_lock(rwlock_t *lock)
159 {
160 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
161 arch_read_lock(&lock->raw_lock);
162 }
163
164 int do_raw_read_trylock(rwlock_t *lock)
165 {
166 int ret = arch_read_trylock(&lock->raw_lock);
167
168 #ifndef CONFIG_SMP
169 /*
170 * Must not happen on UP:
171 */
172 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
173 #endif
174 return ret;
175 }
176
177 void do_raw_read_unlock(rwlock_t *lock)
178 {
179 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
180 arch_read_unlock(&lock->raw_lock);
181 }
182
183 static inline void debug_write_lock_before(rwlock_t *lock)
184 {
185 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
186 RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
187 RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
188 lock, "cpu recursion");
189 }
190
191 static inline void debug_write_lock_after(rwlock_t *lock)
192 {
193 WRITE_ONCE(lock->owner_cpu, raw_smp_processor_id());
194 WRITE_ONCE(lock->owner, current);
195 }
196
197 static inline void debug_write_unlock(rwlock_t *lock)
198 {
199 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
200 RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
201 RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
202 lock, "wrong CPU");
203 WRITE_ONCE(lock->owner, SPINLOCK_OWNER_INIT);
204 WRITE_ONCE(lock->owner_cpu, -1);
205 }
206
207 void do_raw_write_lock(rwlock_t *lock)
208 {
209 debug_write_lock_before(lock);
210 arch_write_lock(&lock->raw_lock);
211 debug_write_lock_after(lock);
212 }
213
214 int do_raw_write_trylock(rwlock_t *lock)
215 {
216 int ret = arch_write_trylock(&lock->raw_lock);
217
218 if (ret)
219 debug_write_lock_after(lock);
220 #ifndef CONFIG_SMP
221 /*
222 * Must not happen on UP:
223 */
224 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
225 #endif
226 return ret;
227 }
228
229 void do_raw_write_unlock(rwlock_t *lock)
230 {
231 debug_write_unlock(lock);
232 arch_write_unlock(&lock->raw_lock);
233 }
234
235 #endif /* !CONFIG_PREEMPT_RT */
236
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.