~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/lib/percpu_counter.c

Version: ~ [ linux-5.2-rc1 ] ~ [ linux-5.1.2 ] ~ [ linux-5.0.16 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.43 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.119 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.176 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.179 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.139 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.67 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Fast batching percpu counters.
  3  */
  4 
  5 #include <linux/percpu_counter.h>
  6 #include <linux/notifier.h>
  7 #include <linux/mutex.h>
  8 #include <linux/init.h>
  9 #include <linux/cpu.h>
 10 #include <linux/module.h>
 11 #include <linux/debugobjects.h>
 12 
 13 #ifdef CONFIG_HOTPLUG_CPU
 14 static LIST_HEAD(percpu_counters);
 15 static DEFINE_SPINLOCK(percpu_counters_lock);
 16 #endif
 17 
 18 #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
 19 
 20 static struct debug_obj_descr percpu_counter_debug_descr;
 21 
 22 static int percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
 23 {
 24         struct percpu_counter *fbc = addr;
 25 
 26         switch (state) {
 27         case ODEBUG_STATE_ACTIVE:
 28                 percpu_counter_destroy(fbc);
 29                 debug_object_free(fbc, &percpu_counter_debug_descr);
 30                 return 1;
 31         default:
 32                 return 0;
 33         }
 34 }
 35 
 36 static struct debug_obj_descr percpu_counter_debug_descr = {
 37         .name           = "percpu_counter",
 38         .fixup_free     = percpu_counter_fixup_free,
 39 };
 40 
 41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 42 {
 43         debug_object_init(fbc, &percpu_counter_debug_descr);
 44         debug_object_activate(fbc, &percpu_counter_debug_descr);
 45 }
 46 
 47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 48 {
 49         debug_object_deactivate(fbc, &percpu_counter_debug_descr);
 50         debug_object_free(fbc, &percpu_counter_debug_descr);
 51 }
 52 
 53 #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 55 { }
 56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 57 { }
 58 #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 59 
 60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
 61 {
 62         int cpu;
 63         unsigned long flags;
 64 
 65         raw_spin_lock_irqsave(&fbc->lock, flags);
 66         for_each_possible_cpu(cpu) {
 67                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
 68                 *pcount = 0;
 69         }
 70         fbc->count = amount;
 71         raw_spin_unlock_irqrestore(&fbc->lock, flags);
 72 }
 73 EXPORT_SYMBOL(percpu_counter_set);
 74 
 75 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
 76 {
 77         s64 count;
 78 
 79         preempt_disable();
 80         count = __this_cpu_read(*fbc->counters) + amount;
 81         if (count >= batch || count <= -batch) {
 82                 unsigned long flags;
 83                 raw_spin_lock_irqsave(&fbc->lock, flags);
 84                 fbc->count += count;
 85                 __this_cpu_sub(*fbc->counters, count - amount);
 86                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
 87         } else {
 88                 this_cpu_add(*fbc->counters, amount);
 89         }
 90         preempt_enable();
 91 }
 92 EXPORT_SYMBOL(__percpu_counter_add);
 93 
 94 /*
 95  * Add up all the per-cpu counts, return the result.  This is a more accurate
 96  * but much slower version of percpu_counter_read_positive()
 97  */
 98 s64 __percpu_counter_sum(struct percpu_counter *fbc)
 99 {
100         s64 ret;
101         int cpu;
102         unsigned long flags;
103 
104         raw_spin_lock_irqsave(&fbc->lock, flags);
105         ret = fbc->count;
106         for_each_online_cpu(cpu) {
107                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
108                 ret += *pcount;
109         }
110         raw_spin_unlock_irqrestore(&fbc->lock, flags);
111         return ret;
112 }
113 EXPORT_SYMBOL(__percpu_counter_sum);
114 
115 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
116                           struct lock_class_key *key)
117 {
118         unsigned long flags __maybe_unused;
119 
120         raw_spin_lock_init(&fbc->lock);
121         lockdep_set_class(&fbc->lock, key);
122         fbc->count = amount;
123         fbc->counters = alloc_percpu_gfp(s32, gfp);
124         if (!fbc->counters)
125                 return -ENOMEM;
126 
127         debug_percpu_counter_activate(fbc);
128 
129 #ifdef CONFIG_HOTPLUG_CPU
130         INIT_LIST_HEAD(&fbc->list);
131         spin_lock_irqsave(&percpu_counters_lock, flags);
132         list_add(&fbc->list, &percpu_counters);
133         spin_unlock_irqrestore(&percpu_counters_lock, flags);
134 #endif
135         return 0;
136 }
137 EXPORT_SYMBOL(__percpu_counter_init);
138 
139 void percpu_counter_destroy(struct percpu_counter *fbc)
140 {
141         unsigned long flags __maybe_unused;
142 
143         if (!fbc->counters)
144                 return;
145 
146         debug_percpu_counter_deactivate(fbc);
147 
148 #ifdef CONFIG_HOTPLUG_CPU
149         spin_lock_irqsave(&percpu_counters_lock, flags);
150         list_del(&fbc->list);
151         spin_unlock_irqrestore(&percpu_counters_lock, flags);
152 #endif
153         free_percpu(fbc->counters);
154         fbc->counters = NULL;
155 }
156 EXPORT_SYMBOL(percpu_counter_destroy);
157 
158 int percpu_counter_batch __read_mostly = 32;
159 EXPORT_SYMBOL(percpu_counter_batch);
160 
161 static void compute_batch_value(void)
162 {
163         int nr = num_online_cpus();
164 
165         percpu_counter_batch = max(32, nr*2);
166 }
167 
168 static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
169                                         unsigned long action, void *hcpu)
170 {
171 #ifdef CONFIG_HOTPLUG_CPU
172         unsigned int cpu;
173         struct percpu_counter *fbc;
174 
175         compute_batch_value();
176         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
177                 return NOTIFY_OK;
178 
179         cpu = (unsigned long)hcpu;
180         spin_lock_irq(&percpu_counters_lock);
181         list_for_each_entry(fbc, &percpu_counters, list) {
182                 s32 *pcount;
183                 unsigned long flags;
184 
185                 raw_spin_lock_irqsave(&fbc->lock, flags);
186                 pcount = per_cpu_ptr(fbc->counters, cpu);
187                 fbc->count += *pcount;
188                 *pcount = 0;
189                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
190         }
191         spin_unlock_irq(&percpu_counters_lock);
192 #endif
193         return NOTIFY_OK;
194 }
195 
196 /*
197  * Compare counter against given value.
198  * Return 1 if greater, 0 if equal and -1 if less
199  */
200 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
201 {
202         s64     count;
203 
204         count = percpu_counter_read(fbc);
205         /* Check to see if rough count will be sufficient for comparison */
206         if (abs(count - rhs) > (batch * num_online_cpus())) {
207                 if (count > rhs)
208                         return 1;
209                 else
210                         return -1;
211         }
212         /* Need to use precise count */
213         count = percpu_counter_sum(fbc);
214         if (count > rhs)
215                 return 1;
216         else if (count < rhs)
217                 return -1;
218         else
219                 return 0;
220 }
221 EXPORT_SYMBOL(__percpu_counter_compare);
222 
223 static int __init percpu_counter_startup(void)
224 {
225         compute_batch_value();
226         hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
227         return 0;
228 }
229 module_init(percpu_counter_startup);
230 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp