~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/rcu/srcutiny.c

Version: ~ [ linux-4.14 ] ~ [ linux-4.13.12 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.61 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.97 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.46 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.80 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.50 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.95 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Sleepable Read-Copy Update mechanism for mutual exclusion,
  3  *      tiny version for non-preemptible single-CPU use.
  4  *
  5  * This program is free software; you can redistribute it and/or modify
  6  * it under the terms of the GNU General Public License as published by
  7  * the Free Software Foundation; either version 2 of the License, or
  8  * (at your option) any later version.
  9  *
 10  * This program is distributed in the hope that it will be useful,
 11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  *
 15  * You should have received a copy of the GNU General Public License
 16  * along with this program; if not, you can access it online at
 17  * http://www.gnu.org/licenses/gpl-2.0.html.
 18  *
 19  * Copyright (C) IBM Corporation, 2017
 20  *
 21  * Author: Paul McKenney <paulmck@us.ibm.com>
 22  */
 23 
 24 #include <linux/export.h>
 25 #include <linux/mutex.h>
 26 #include <linux/preempt.h>
 27 #include <linux/rcupdate_wait.h>
 28 #include <linux/sched.h>
 29 #include <linux/delay.h>
 30 #include <linux/srcu.h>
 31 
 32 #include <linux/rcu_node_tree.h>
 33 #include "rcu_segcblist.h"
 34 #include "rcu.h"
 35 
 36 int rcu_scheduler_active __read_mostly;
 37 
 38 static int init_srcu_struct_fields(struct srcu_struct *sp)
 39 {
 40         sp->srcu_lock_nesting[0] = 0;
 41         sp->srcu_lock_nesting[1] = 0;
 42         init_swait_queue_head(&sp->srcu_wq);
 43         sp->srcu_cb_head = NULL;
 44         sp->srcu_cb_tail = &sp->srcu_cb_head;
 45         sp->srcu_gp_running = false;
 46         sp->srcu_gp_waiting = false;
 47         sp->srcu_idx = 0;
 48         INIT_WORK(&sp->srcu_work, srcu_drive_gp);
 49         return 0;
 50 }
 51 
 52 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 53 
 54 int __init_srcu_struct(struct srcu_struct *sp, const char *name,
 55                        struct lock_class_key *key)
 56 {
 57         /* Don't re-initialize a lock while it is held. */
 58         debug_check_no_locks_freed((void *)sp, sizeof(*sp));
 59         lockdep_init_map(&sp->dep_map, name, key, 0);
 60         return init_srcu_struct_fields(sp);
 61 }
 62 EXPORT_SYMBOL_GPL(__init_srcu_struct);
 63 
 64 #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 65 
 66 /*
 67  * init_srcu_struct - initialize a sleep-RCU structure
 68  * @sp: structure to initialize.
 69  *
 70  * Must invoke this on a given srcu_struct before passing that srcu_struct
 71  * to any other function.  Each srcu_struct represents a separate domain
 72  * of SRCU protection.
 73  */
 74 int init_srcu_struct(struct srcu_struct *sp)
 75 {
 76         return init_srcu_struct_fields(sp);
 77 }
 78 EXPORT_SYMBOL_GPL(init_srcu_struct);
 79 
 80 #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
 81 
 82 /*
 83  * cleanup_srcu_struct - deconstruct a sleep-RCU structure
 84  * @sp: structure to clean up.
 85  *
 86  * Must invoke this after you are finished using a given srcu_struct that
 87  * was initialized via init_srcu_struct(), else you leak memory.
 88  */
 89 void cleanup_srcu_struct(struct srcu_struct *sp)
 90 {
 91         WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]);
 92         flush_work(&sp->srcu_work);
 93         WARN_ON(sp->srcu_gp_running);
 94         WARN_ON(sp->srcu_gp_waiting);
 95         WARN_ON(sp->srcu_cb_head);
 96         WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail);
 97 }
 98 EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
 99 
100 /*
101  * Removes the count for the old reader from the appropriate element of
102  * the srcu_struct.
103  */
104 void __srcu_read_unlock(struct srcu_struct *sp, int idx)
105 {
106         int newval = sp->srcu_lock_nesting[idx] - 1;
107 
108         WRITE_ONCE(sp->srcu_lock_nesting[idx], newval);
109         if (!newval && READ_ONCE(sp->srcu_gp_waiting))
110                 swake_up(&sp->srcu_wq);
111 }
112 EXPORT_SYMBOL_GPL(__srcu_read_unlock);
113 
114 /*
115  * Workqueue handler to drive one grace period and invoke any callbacks
116  * that become ready as a result.  Single-CPU and !PREEMPT operation
117  * means that we get away with murder on synchronization.  ;-)
118  */
119 void srcu_drive_gp(struct work_struct *wp)
120 {
121         int idx;
122         struct rcu_head *lh;
123         struct rcu_head *rhp;
124         struct srcu_struct *sp;
125 
126         sp = container_of(wp, struct srcu_struct, srcu_work);
127         if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head))
128                 return; /* Already running or nothing to do. */
129 
130         /* Remove recently arrived callbacks and wait for readers. */
131         WRITE_ONCE(sp->srcu_gp_running, true);
132         local_irq_disable();
133         lh = sp->srcu_cb_head;
134         sp->srcu_cb_head = NULL;
135         sp->srcu_cb_tail = &sp->srcu_cb_head;
136         local_irq_enable();
137         idx = sp->srcu_idx;
138         WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx);
139         WRITE_ONCE(sp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */
140         swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx]));
141         WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
142 
143         /* Invoke the callbacks we removed above. */
144         while (lh) {
145                 rhp = lh;
146                 lh = lh->next;
147                 local_bh_disable();
148                 rhp->func(rhp);
149                 local_bh_enable();
150         }
151 
152         /*
153          * Enable rescheduling, and if there are more callbacks,
154          * reschedule ourselves.  This can race with a call_srcu()
155          * at interrupt level, but the ->srcu_gp_running checks will
156          * straighten that out.
157          */
158         WRITE_ONCE(sp->srcu_gp_running, false);
159         if (READ_ONCE(sp->srcu_cb_head))
160                 schedule_work(&sp->srcu_work);
161 }
162 EXPORT_SYMBOL_GPL(srcu_drive_gp);
163 
164 /*
165  * Enqueue an SRCU callback on the specified srcu_struct structure,
166  * initiating grace-period processing if it is not already running.
167  */
168 void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp,
169                rcu_callback_t func)
170 {
171         unsigned long flags;
172 
173         rhp->func = func;
174         rhp->next = NULL;
175         local_irq_save(flags);
176         *sp->srcu_cb_tail = rhp;
177         sp->srcu_cb_tail = &rhp->next;
178         local_irq_restore(flags);
179         if (!READ_ONCE(sp->srcu_gp_running))
180                 schedule_work(&sp->srcu_work);
181 }
182 EXPORT_SYMBOL_GPL(call_srcu);
183 
184 /*
185  * synchronize_srcu - wait for prior SRCU read-side critical-section completion
186  */
187 void synchronize_srcu(struct srcu_struct *sp)
188 {
189         struct rcu_synchronize rs;
190 
191         init_rcu_head_on_stack(&rs.head);
192         init_completion(&rs.completion);
193         call_srcu(sp, &rs.head, wakeme_after_rcu);
194         wait_for_completion(&rs.completion);
195         destroy_rcu_head_on_stack(&rs.head);
196 }
197 EXPORT_SYMBOL_GPL(synchronize_srcu);
198 
199 /* Lockdep diagnostics.  */
200 void __init rcu_scheduler_starting(void)
201 {
202         rcu_scheduler_active = RCU_SCHEDULER_RUNNING;
203 }
204 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp