~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/locking/qrwlock.c

Version: ~ [ linux-5.3 ] ~ [ linux-5.2.15 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.73 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.144 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.193 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.193 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.73 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Queued read/write locks
  3  *
  4  * This program is free software; you can redistribute it and/or modify
  5  * it under the terms of the GNU General Public License as published by
  6  * the Free Software Foundation; either version 2 of the License, or
  7  * (at your option) any later version.
  8  *
  9  * This program is distributed in the hope that it will be useful,
 10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 12  * GNU General Public License for more details.
 13  *
 14  * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
 15  *
 16  * Authors: Waiman Long <waiman.long@hp.com>
 17  */
 18 #include <linux/smp.h>
 19 #include <linux/bug.h>
 20 #include <linux/cpumask.h>
 21 #include <linux/percpu.h>
 22 #include <linux/hardirq.h>
 23 #include <asm/qrwlock.h>
 24 
 25 /*
 26  * This internal data structure is used for optimizing access to some of
 27  * the subfields within the atomic_t cnts.
 28  */
 29 struct __qrwlock {
 30         union {
 31                 atomic_t cnts;
 32                 struct {
 33 #ifdef __LITTLE_ENDIAN
 34                         u8 wmode;       /* Writer mode   */
 35                         u8 rcnts[3];    /* Reader counts */
 36 #else
 37                         u8 rcnts[3];    /* Reader counts */
 38                         u8 wmode;       /* Writer mode   */
 39 #endif
 40                 };
 41         };
 42         arch_spinlock_t lock;
 43 };
 44 
 45 /**
 46  * rspin_until_writer_unlock - inc reader count & spin until writer is gone
 47  * @lock  : Pointer to queue rwlock structure
 48  * @writer: Current queue rwlock writer status byte
 49  *
 50  * In interrupt context or at the head of the queue, the reader will just
 51  * increment the reader count & wait until the writer releases the lock.
 52  */
 53 static __always_inline void
 54 rspin_until_writer_unlock(struct qrwlock *lock, u32 cnts)
 55 {
 56         while ((cnts & _QW_WMASK) == _QW_LOCKED) {
 57                 cpu_relax();
 58                 cnts = atomic_read_acquire(&lock->cnts);
 59         }
 60 }
 61 
 62 /**
 63  * queued_read_lock_slowpath - acquire read lock of a queue rwlock
 64  * @lock: Pointer to queue rwlock structure
 65  * @cnts: Current qrwlock lock value
 66  */
 67 void queued_read_lock_slowpath(struct qrwlock *lock, u32 cnts)
 68 {
 69         /*
 70          * Readers come here when they cannot get the lock without waiting
 71          */
 72         if (unlikely(in_interrupt())) {
 73                 /*
 74                  * Readers in interrupt context will get the lock immediately
 75                  * if the writer is just waiting (not holding the lock yet).
 76                  * The rspin_until_writer_unlock() function returns immediately
 77                  * in this case. Otherwise, they will spin (with ACQUIRE
 78                  * semantics) until the lock is available without waiting in
 79                  * the queue.
 80                  */
 81                 rspin_until_writer_unlock(lock, cnts);
 82                 return;
 83         }
 84         atomic_sub(_QR_BIAS, &lock->cnts);
 85 
 86         /*
 87          * Put the reader into the wait queue
 88          */
 89         arch_spin_lock(&lock->wait_lock);
 90 
 91         /*
 92          * The ACQUIRE semantics of the following spinning code ensure
 93          * that accesses can't leak upwards out of our subsequent critical
 94          * section in the case that the lock is currently held for write.
 95          */
 96         cnts = atomic_fetch_add_acquire(_QR_BIAS, &lock->cnts);
 97         rspin_until_writer_unlock(lock, cnts);
 98 
 99         /*
100          * Signal the next one in queue to become queue head
101          */
102         arch_spin_unlock(&lock->wait_lock);
103 }
104 EXPORT_SYMBOL(queued_read_lock_slowpath);
105 
106 /**
107  * queued_write_lock_slowpath - acquire write lock of a queue rwlock
108  * @lock : Pointer to queue rwlock structure
109  */
110 void queued_write_lock_slowpath(struct qrwlock *lock)
111 {
112         u32 cnts;
113 
114         /* Put the writer into the wait queue */
115         arch_spin_lock(&lock->wait_lock);
116 
117         /* Try to acquire the lock directly if no reader is present */
118         if (!atomic_read(&lock->cnts) &&
119             (atomic_cmpxchg_acquire(&lock->cnts, 0, _QW_LOCKED) == 0))
120                 goto unlock;
121 
122         /*
123          * Set the waiting flag to notify readers that a writer is pending,
124          * or wait for a previous writer to go away.
125          */
126         for (;;) {
127                 struct __qrwlock *l = (struct __qrwlock *)lock;
128 
129                 if (!READ_ONCE(l->wmode) &&
130                    (cmpxchg_relaxed(&l->wmode, 0, _QW_WAITING) == 0))
131                         break;
132 
133                 cpu_relax();
134         }
135 
136         /* When no more readers, set the locked flag */
137         for (;;) {
138                 cnts = atomic_read(&lock->cnts);
139                 if ((cnts == _QW_WAITING) &&
140                     (atomic_cmpxchg_acquire(&lock->cnts, _QW_WAITING,
141                                             _QW_LOCKED) == _QW_WAITING))
142                         break;
143 
144                 cpu_relax();
145         }
146 unlock:
147         arch_spin_unlock(&lock->wait_lock);
148 }
149 EXPORT_SYMBOL(queued_write_lock_slowpath);
150 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp