~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/lib/percpu_counter.c

Version: ~ [ linux-5.6-rc1 ] ~ [ linux-5.5.2 ] ~ [ linux-5.4.17 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.102 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.170 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.213 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.213 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Fast batching percpu counters.
  3  */
  4 
  5 #include <linux/percpu_counter.h>
  6 #include <linux/notifier.h>
  7 #include <linux/mutex.h>
  8 #include <linux/init.h>
  9 #include <linux/cpu.h>
 10 #include <linux/module.h>
 11 #include <linux/debugobjects.h>
 12 
 13 #ifdef CONFIG_HOTPLUG_CPU
 14 static LIST_HEAD(percpu_counters);
 15 static DEFINE_SPINLOCK(percpu_counters_lock);
 16 #endif
 17 
 18 #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
 19 
 20 static struct debug_obj_descr percpu_counter_debug_descr;
 21 
 22 static int percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
 23 {
 24         struct percpu_counter *fbc = addr;
 25 
 26         switch (state) {
 27         case ODEBUG_STATE_ACTIVE:
 28                 percpu_counter_destroy(fbc);
 29                 debug_object_free(fbc, &percpu_counter_debug_descr);
 30                 return 1;
 31         default:
 32                 return 0;
 33         }
 34 }
 35 
 36 static struct debug_obj_descr percpu_counter_debug_descr = {
 37         .name           = "percpu_counter",
 38         .fixup_free     = percpu_counter_fixup_free,
 39 };
 40 
 41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 42 {
 43         debug_object_init(fbc, &percpu_counter_debug_descr);
 44         debug_object_activate(fbc, &percpu_counter_debug_descr);
 45 }
 46 
 47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 48 {
 49         debug_object_deactivate(fbc, &percpu_counter_debug_descr);
 50         debug_object_free(fbc, &percpu_counter_debug_descr);
 51 }
 52 
 53 #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 55 { }
 56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 57 { }
 58 #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 59 
 60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
 61 {
 62         int cpu;
 63         unsigned long flags;
 64 
 65         raw_spin_lock_irqsave(&fbc->lock, flags);
 66         for_each_possible_cpu(cpu) {
 67                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
 68                 *pcount = 0;
 69         }
 70         fbc->count = amount;
 71         raw_spin_unlock_irqrestore(&fbc->lock, flags);
 72 }
 73 EXPORT_SYMBOL(percpu_counter_set);
 74 
 75 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
 76 {
 77         s64 count;
 78 
 79         preempt_disable();
 80         count = __this_cpu_read(*fbc->counters) + amount;
 81         if (count >= batch || count <= -batch) {
 82                 unsigned long flags;
 83                 raw_spin_lock_irqsave(&fbc->lock, flags);
 84                 fbc->count += count;
 85                 __this_cpu_sub(*fbc->counters, count - amount);
 86                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
 87         } else {
 88                 this_cpu_add(*fbc->counters, amount);
 89         }
 90         preempt_enable();
 91 }
 92 EXPORT_SYMBOL(__percpu_counter_add);
 93 
 94 /*
 95  * Add up all the per-cpu counts, return the result.  This is a more accurate
 96  * but much slower version of percpu_counter_read_positive()
 97  */
 98 s64 __percpu_counter_sum(struct percpu_counter *fbc)
 99 {
100         s64 ret;
101         int cpu;
102         unsigned long flags;
103 
104         raw_spin_lock_irqsave(&fbc->lock, flags);
105         ret = fbc->count;
106         for_each_online_cpu(cpu) {
107                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
108                 ret += *pcount;
109         }
110         raw_spin_unlock_irqrestore(&fbc->lock, flags);
111         return ret;
112 }
113 EXPORT_SYMBOL(__percpu_counter_sum);
114 
115 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
116                           struct lock_class_key *key)
117 {
118         raw_spin_lock_init(&fbc->lock);
119         lockdep_set_class(&fbc->lock, key);
120         fbc->count = amount;
121         fbc->counters = alloc_percpu(s32);
122         if (!fbc->counters)
123                 return -ENOMEM;
124 
125         debug_percpu_counter_activate(fbc);
126 
127 #ifdef CONFIG_HOTPLUG_CPU
128         INIT_LIST_HEAD(&fbc->list);
129         spin_lock(&percpu_counters_lock);
130         list_add(&fbc->list, &percpu_counters);
131         spin_unlock(&percpu_counters_lock);
132 #endif
133         return 0;
134 }
135 EXPORT_SYMBOL(__percpu_counter_init);
136 
137 void percpu_counter_destroy(struct percpu_counter *fbc)
138 {
139         if (!fbc->counters)
140                 return;
141 
142         debug_percpu_counter_deactivate(fbc);
143 
144 #ifdef CONFIG_HOTPLUG_CPU
145         spin_lock(&percpu_counters_lock);
146         list_del(&fbc->list);
147         spin_unlock(&percpu_counters_lock);
148 #endif
149         free_percpu(fbc->counters);
150         fbc->counters = NULL;
151 }
152 EXPORT_SYMBOL(percpu_counter_destroy);
153 
154 int percpu_counter_batch __read_mostly = 32;
155 EXPORT_SYMBOL(percpu_counter_batch);
156 
157 static void compute_batch_value(void)
158 {
159         int nr = num_online_cpus();
160 
161         percpu_counter_batch = max(32, nr*2);
162 }
163 
164 static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
165                                         unsigned long action, void *hcpu)
166 {
167 #ifdef CONFIG_HOTPLUG_CPU
168         unsigned int cpu;
169         struct percpu_counter *fbc;
170 
171         compute_batch_value();
172         if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
173                 return NOTIFY_OK;
174 
175         cpu = (unsigned long)hcpu;
176         spin_lock(&percpu_counters_lock);
177         list_for_each_entry(fbc, &percpu_counters, list) {
178                 s32 *pcount;
179                 unsigned long flags;
180 
181                 raw_spin_lock_irqsave(&fbc->lock, flags);
182                 pcount = per_cpu_ptr(fbc->counters, cpu);
183                 fbc->count += *pcount;
184                 *pcount = 0;
185                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
186         }
187         spin_unlock(&percpu_counters_lock);
188 #endif
189         return NOTIFY_OK;
190 }
191 
192 /*
193  * Compare counter against given value.
194  * Return 1 if greater, 0 if equal and -1 if less
195  */
196 int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
197 {
198         s64     count;
199 
200         count = percpu_counter_read(fbc);
201         /* Check to see if rough count will be sufficient for comparison */
202         if (abs(count - rhs) > (percpu_counter_batch*num_online_cpus())) {
203                 if (count > rhs)
204                         return 1;
205                 else
206                         return -1;
207         }
208         /* Need to use precise count */
209         count = percpu_counter_sum(fbc);
210         if (count > rhs)
211                 return 1;
212         else if (count < rhs)
213                 return -1;
214         else
215                 return 0;
216 }
217 EXPORT_SYMBOL(percpu_counter_compare);
218 
219 static int __init percpu_counter_startup(void)
220 {
221         compute_batch_value();
222         hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
223         return 0;
224 }
225 module_init(percpu_counter_startup);
226 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp