~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arc/include/asm/spinlock.h

Version: ~ [ linux-5.13-rc5 ] ~ [ linux-5.12.9 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.42 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.124 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.193 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.235 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.271 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.271 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License version 2 as
  6  * published by the Free Software Foundation.
  7  */
  8 
  9 #ifndef __ASM_SPINLOCK_H
 10 #define __ASM_SPINLOCK_H
 11 
 12 #include <asm/spinlock_types.h>
 13 #include <asm/processor.h>
 14 #include <asm/barrier.h>
 15 
 16 #define arch_spin_is_locked(x)  ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
 17 
 18 #ifdef CONFIG_ARC_HAS_LLSC
 19 
 20 static inline void arch_spin_lock(arch_spinlock_t *lock)
 21 {
 22         unsigned int val;
 23 
 24         smp_mb();
 25 
 26         __asm__ __volatile__(
 27         "1:     llock   %[val], [%[slock]]      \n"
 28         "       breq    %[val], %[LOCKED], 1b   \n"     /* spin while LOCKED */
 29         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
 30         "       bnz     1b                      \n"
 31         "                                       \n"
 32         : [val]         "=&r"   (val)
 33         : [slock]       "r"     (&(lock->slock)),
 34           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
 35         : "memory", "cc");
 36 
 37         smp_mb();
 38 }
 39 
 40 /* 1 - lock taken successfully */
 41 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 42 {
 43         unsigned int val, got_it = 0;
 44 
 45         smp_mb();
 46 
 47         __asm__ __volatile__(
 48         "1:     llock   %[val], [%[slock]]      \n"
 49         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
 50         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
 51         "       bnz     1b                      \n"
 52         "       mov     %[got_it], 1            \n"
 53         "4:                                     \n"
 54         "                                       \n"
 55         : [val]         "=&r"   (val),
 56           [got_it]      "+&r"   (got_it)
 57         : [slock]       "r"     (&(lock->slock)),
 58           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
 59         : "memory", "cc");
 60 
 61         smp_mb();
 62 
 63         return got_it;
 64 }
 65 
 66 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 67 {
 68         smp_mb();
 69 
 70         lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__;
 71 
 72         smp_mb();
 73 }
 74 
 75 /*
 76  * Read-write spinlocks, allowing multiple readers but only one writer.
 77  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 78  */
 79 
 80 static inline void arch_read_lock(arch_rwlock_t *rw)
 81 {
 82         unsigned int val;
 83 
 84         smp_mb();
 85 
 86         /*
 87          * zero means writer holds the lock exclusively, deny Reader.
 88          * Otherwise grant lock to first/subseq reader
 89          *
 90          *      if (rw->counter > 0) {
 91          *              rw->counter--;
 92          *              ret = 1;
 93          *      }
 94          */
 95 
 96         __asm__ __volatile__(
 97         "1:     llock   %[val], [%[rwlock]]     \n"
 98         "       brls    %[val], %[WR_LOCKED], 1b\n"     /* <= 0: spin while write locked */
 99         "       sub     %[val], %[val], 1       \n"     /* reader lock */
100         "       scond   %[val], [%[rwlock]]     \n"
101         "       bnz     1b                      \n"
102         "                                       \n"
103         : [val]         "=&r"   (val)
104         : [rwlock]      "r"     (&(rw->counter)),
105           [WR_LOCKED]   "ir"    (0)
106         : "memory", "cc");
107 
108         smp_mb();
109 }
110 
111 /* 1 - lock taken successfully */
112 static inline int arch_read_trylock(arch_rwlock_t *rw)
113 {
114         unsigned int val, got_it = 0;
115 
116         smp_mb();
117 
118         __asm__ __volatile__(
119         "1:     llock   %[val], [%[rwlock]]     \n"
120         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
121         "       sub     %[val], %[val], 1       \n"     /* counter-- */
122         "       scond   %[val], [%[rwlock]]     \n"
123         "       bnz     1b                      \n"     /* retry if collided with someone */
124         "       mov     %[got_it], 1            \n"
125         "                                       \n"
126         "4: ; --- done ---                      \n"
127 
128         : [val]         "=&r"   (val),
129           [got_it]      "+&r"   (got_it)
130         : [rwlock]      "r"     (&(rw->counter)),
131           [WR_LOCKED]   "ir"    (0)
132         : "memory", "cc");
133 
134         smp_mb();
135 
136         return got_it;
137 }
138 
139 static inline void arch_write_lock(arch_rwlock_t *rw)
140 {
141         unsigned int val;
142 
143         smp_mb();
144 
145         /*
146          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
147          * deny writer. Otherwise if unlocked grant to writer
148          * Hence the claim that Linux rwlocks are unfair to writers.
149          * (can be starved for an indefinite time by readers).
150          *
151          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
152          *              rw->counter = 0;
153          *              ret = 1;
154          *      }
155          */
156 
157         __asm__ __volatile__(
158         "1:     llock   %[val], [%[rwlock]]     \n"
159         "       brne    %[val], %[UNLOCKED], 1b \n"     /* while !UNLOCKED spin */
160         "       mov     %[val], %[WR_LOCKED]    \n"
161         "       scond   %[val], [%[rwlock]]     \n"
162         "       bnz     1b                      \n"
163         "                                       \n"
164         : [val]         "=&r"   (val)
165         : [rwlock]      "r"     (&(rw->counter)),
166           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
167           [WR_LOCKED]   "ir"    (0)
168         : "memory", "cc");
169 
170         smp_mb();
171 }
172 
173 /* 1 - lock taken successfully */
174 static inline int arch_write_trylock(arch_rwlock_t *rw)
175 {
176         unsigned int val, got_it = 0;
177 
178         smp_mb();
179 
180         __asm__ __volatile__(
181         "1:     llock   %[val], [%[rwlock]]     \n"
182         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
183         "       mov     %[val], %[WR_LOCKED]    \n"
184         "       scond   %[val], [%[rwlock]]     \n"
185         "       bnz     1b                      \n"     /* retry if collided with someone */
186         "       mov     %[got_it], 1            \n"
187         "                                       \n"
188         "4: ; --- done ---                      \n"
189 
190         : [val]         "=&r"   (val),
191           [got_it]      "+&r"   (got_it)
192         : [rwlock]      "r"     (&(rw->counter)),
193           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
194           [WR_LOCKED]   "ir"    (0)
195         : "memory", "cc");
196 
197         smp_mb();
198 
199         return got_it;
200 }
201 
202 static inline void arch_read_unlock(arch_rwlock_t *rw)
203 {
204         unsigned int val;
205 
206         smp_mb();
207 
208         /*
209          * rw->counter++;
210          */
211         __asm__ __volatile__(
212         "1:     llock   %[val], [%[rwlock]]     \n"
213         "       add     %[val], %[val], 1       \n"
214         "       scond   %[val], [%[rwlock]]     \n"
215         "       bnz     1b                      \n"
216         "                                       \n"
217         : [val]         "=&r"   (val)
218         : [rwlock]      "r"     (&(rw->counter))
219         : "memory", "cc");
220 
221         smp_mb();
222 }
223 
224 static inline void arch_write_unlock(arch_rwlock_t *rw)
225 {
226         smp_mb();
227 
228         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
229 
230         smp_mb();
231 }
232 
233 #else   /* !CONFIG_ARC_HAS_LLSC */
234 
235 static inline void arch_spin_lock(arch_spinlock_t *lock)
236 {
237         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
238 
239         /*
240          * This smp_mb() is technically superfluous, we only need the one
241          * after the lock for providing the ACQUIRE semantics.
242          * However doing the "right" thing was regressing hackbench
243          * so keeping this, pending further investigation
244          */
245         smp_mb();
246 
247         __asm__ __volatile__(
248         "1:     ex  %0, [%1]            \n"
249 #ifdef CONFIG_EZNPS_MTM_EXT
250         "       .word %3                \n"
251 #endif
252         "       breq  %0, %2, 1b        \n"
253         : "+&r" (val)
254         : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
255 #ifdef CONFIG_EZNPS_MTM_EXT
256         , "i"(CTOP_INST_SCHD_RW)
257 #endif
258         : "memory");
259 
260         /*
261          * ACQUIRE barrier to ensure load/store after taking the lock
262          * don't "bleed-up" out of the critical section (leak-in is allowed)
263          * http://www.spinics.net/lists/kernel/msg2010409.html
264          *
265          * ARCv2 only has load-load, store-store and all-all barrier
266          * thus need the full all-all barrier
267          */
268         smp_mb();
269 }
270 
271 /* 1 - lock taken successfully */
272 static inline int arch_spin_trylock(arch_spinlock_t *lock)
273 {
274         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
275 
276         smp_mb();
277 
278         __asm__ __volatile__(
279         "1:     ex  %0, [%1]            \n"
280         : "+r" (val)
281         : "r"(&(lock->slock))
282         : "memory");
283 
284         smp_mb();
285 
286         return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
287 }
288 
289 static inline void arch_spin_unlock(arch_spinlock_t *lock)
290 {
291         unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
292 
293         /*
294          * RELEASE barrier: given the instructions avail on ARCv2, full barrier
295          * is the only option
296          */
297         smp_mb();
298 
299         /*
300          * EX is not really required here, a simple STore of 0 suffices.
301          * However this causes tasklist livelocks in SystemC based SMP virtual
302          * platforms where the systemc core scheduler uses EX as a cue for
303          * moving to next core. Do a git log of this file for details
304          */
305         __asm__ __volatile__(
306         "       ex  %0, [%1]            \n"
307         : "+r" (val)
308         : "r"(&(lock->slock))
309         : "memory");
310 
311         /*
312          * superfluous, but keeping for now - see pairing version in
313          * arch_spin_lock above
314          */
315         smp_mb();
316 }
317 
318 /*
319  * Read-write spinlocks, allowing multiple readers but only one writer.
320  * Unfair locking as Writers could be starved indefinitely by Reader(s)
321  *
322  * The spinlock itself is contained in @counter and access to it is
323  * serialized with @lock_mutex.
324  */
325 
326 /* 1 - lock taken successfully */
327 static inline int arch_read_trylock(arch_rwlock_t *rw)
328 {
329         int ret = 0;
330         unsigned long flags;
331 
332         local_irq_save(flags);
333         arch_spin_lock(&(rw->lock_mutex));
334 
335         /*
336          * zero means writer holds the lock exclusively, deny Reader.
337          * Otherwise grant lock to first/subseq reader
338          */
339         if (rw->counter > 0) {
340                 rw->counter--;
341                 ret = 1;
342         }
343 
344         arch_spin_unlock(&(rw->lock_mutex));
345         local_irq_restore(flags);
346 
347         smp_mb();
348         return ret;
349 }
350 
351 /* 1 - lock taken successfully */
352 static inline int arch_write_trylock(arch_rwlock_t *rw)
353 {
354         int ret = 0;
355         unsigned long flags;
356 
357         local_irq_save(flags);
358         arch_spin_lock(&(rw->lock_mutex));
359 
360         /*
361          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
362          * deny writer. Otherwise if unlocked grant to writer
363          * Hence the claim that Linux rwlocks are unfair to writers.
364          * (can be starved for an indefinite time by readers).
365          */
366         if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
367                 rw->counter = 0;
368                 ret = 1;
369         }
370         arch_spin_unlock(&(rw->lock_mutex));
371         local_irq_restore(flags);
372 
373         return ret;
374 }
375 
376 static inline void arch_read_lock(arch_rwlock_t *rw)
377 {
378         while (!arch_read_trylock(rw))
379                 cpu_relax();
380 }
381 
382 static inline void arch_write_lock(arch_rwlock_t *rw)
383 {
384         while (!arch_write_trylock(rw))
385                 cpu_relax();
386 }
387 
388 static inline void arch_read_unlock(arch_rwlock_t *rw)
389 {
390         unsigned long flags;
391 
392         local_irq_save(flags);
393         arch_spin_lock(&(rw->lock_mutex));
394         rw->counter++;
395         arch_spin_unlock(&(rw->lock_mutex));
396         local_irq_restore(flags);
397 }
398 
399 static inline void arch_write_unlock(arch_rwlock_t *rw)
400 {
401         unsigned long flags;
402 
403         local_irq_save(flags);
404         arch_spin_lock(&(rw->lock_mutex));
405         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
406         arch_spin_unlock(&(rw->lock_mutex));
407         local_irq_restore(flags);
408 }
409 
410 #endif
411 
412 #endif /* __ASM_SPINLOCK_H */
413 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp