~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/lib/percpu_counter.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Fast batching percpu counters.
  4  */
  5 
  6 #include <linux/percpu_counter.h>
  7 #include <linux/mutex.h>
  8 #include <linux/init.h>
  9 #include <linux/cpu.h>
 10 #include <linux/module.h>
 11 #include <linux/debugobjects.h>
 12 
 13 #ifdef CONFIG_HOTPLUG_CPU
 14 static LIST_HEAD(percpu_counters);
 15 static DEFINE_SPINLOCK(percpu_counters_lock);
 16 #endif
 17 
 18 #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
 19 
 20 static struct debug_obj_descr percpu_counter_debug_descr;
 21 
 22 static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
 23 {
 24         struct percpu_counter *fbc = addr;
 25 
 26         switch (state) {
 27         case ODEBUG_STATE_ACTIVE:
 28                 percpu_counter_destroy(fbc);
 29                 debug_object_free(fbc, &percpu_counter_debug_descr);
 30                 return true;
 31         default:
 32                 return false;
 33         }
 34 }
 35 
 36 static struct debug_obj_descr percpu_counter_debug_descr = {
 37         .name           = "percpu_counter",
 38         .fixup_free     = percpu_counter_fixup_free,
 39 };
 40 
 41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 42 {
 43         debug_object_init(fbc, &percpu_counter_debug_descr);
 44         debug_object_activate(fbc, &percpu_counter_debug_descr);
 45 }
 46 
 47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 48 {
 49         debug_object_deactivate(fbc, &percpu_counter_debug_descr);
 50         debug_object_free(fbc, &percpu_counter_debug_descr);
 51 }
 52 
 53 #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
 55 { }
 56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
 57 { }
 58 #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
 59 
 60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
 61 {
 62         int cpu;
 63         unsigned long flags;
 64 
 65         raw_spin_lock_irqsave(&fbc->lock, flags);
 66         for_each_possible_cpu(cpu) {
 67                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
 68                 *pcount = 0;
 69         }
 70         fbc->count = amount;
 71         raw_spin_unlock_irqrestore(&fbc->lock, flags);
 72 }
 73 EXPORT_SYMBOL(percpu_counter_set);
 74 
 75 /**
 76  * This function is both preempt and irq safe. The former is due to explicit
 77  * preemption disable. The latter is guaranteed by the fact that the slow path
 78  * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
 79  * this_cpu_add which is irq-safe by definition. Hence there is no need muck
 80  * with irq state before calling this one
 81  */
 82 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
 83 {
 84         s64 count;
 85 
 86         preempt_disable();
 87         count = __this_cpu_read(*fbc->counters) + amount;
 88         if (count >= batch || count <= -batch) {
 89                 unsigned long flags;
 90                 raw_spin_lock_irqsave(&fbc->lock, flags);
 91                 fbc->count += count;
 92                 __this_cpu_sub(*fbc->counters, count - amount);
 93                 raw_spin_unlock_irqrestore(&fbc->lock, flags);
 94         } else {
 95                 this_cpu_add(*fbc->counters, amount);
 96         }
 97         preempt_enable();
 98 }
 99 EXPORT_SYMBOL(percpu_counter_add_batch);
100 
101 /*
102  * Add up all the per-cpu counts, return the result.  This is a more accurate
103  * but much slower version of percpu_counter_read_positive()
104  */
105 s64 __percpu_counter_sum(struct percpu_counter *fbc)
106 {
107         s64 ret;
108         int cpu;
109         unsigned long flags;
110 
111         raw_spin_lock_irqsave(&fbc->lock, flags);
112         ret = fbc->count;
113         for_each_online_cpu(cpu) {
114                 s32 *pcount = per_cpu_ptr(fbc->counters, cpu);
115                 ret += *pcount;
116         }
117         raw_spin_unlock_irqrestore(&fbc->lock, flags);
118         return ret;
119 }
120 EXPORT_SYMBOL(__percpu_counter_sum);
121 
122 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,
123                           struct lock_class_key *key)
124 {
125         unsigned long flags __maybe_unused;
126 
127         raw_spin_lock_init(&fbc->lock);
128         lockdep_set_class(&fbc->lock, key);
129         fbc->count = amount;
130         fbc->counters = alloc_percpu_gfp(s32, gfp);
131         if (!fbc->counters)
132                 return -ENOMEM;
133 
134         debug_percpu_counter_activate(fbc);
135 
136 #ifdef CONFIG_HOTPLUG_CPU
137         INIT_LIST_HEAD(&fbc->list);
138         spin_lock_irqsave(&percpu_counters_lock, flags);
139         list_add(&fbc->list, &percpu_counters);
140         spin_unlock_irqrestore(&percpu_counters_lock, flags);
141 #endif
142         return 0;
143 }
144 EXPORT_SYMBOL(__percpu_counter_init);
145 
146 void percpu_counter_destroy(struct percpu_counter *fbc)
147 {
148         unsigned long flags __maybe_unused;
149 
150         if (!fbc->counters)
151                 return;
152 
153         debug_percpu_counter_deactivate(fbc);
154 
155 #ifdef CONFIG_HOTPLUG_CPU
156         spin_lock_irqsave(&percpu_counters_lock, flags);
157         list_del(&fbc->list);
158         spin_unlock_irqrestore(&percpu_counters_lock, flags);
159 #endif
160         free_percpu(fbc->counters);
161         fbc->counters = NULL;
162 }
163 EXPORT_SYMBOL(percpu_counter_destroy);
164 
165 int percpu_counter_batch __read_mostly = 32;
166 EXPORT_SYMBOL(percpu_counter_batch);
167 
168 static int compute_batch_value(unsigned int cpu)
169 {
170         int nr = num_online_cpus();
171 
172         percpu_counter_batch = max(32, nr*2);
173         return 0;
174 }
175 
176 static int percpu_counter_cpu_dead(unsigned int cpu)
177 {
178 #ifdef CONFIG_HOTPLUG_CPU
179         struct percpu_counter *fbc;
180 
181         compute_batch_value(cpu);
182 
183         spin_lock_irq(&percpu_counters_lock);
184         list_for_each_entry(fbc, &percpu_counters, list) {
185                 s32 *pcount;
186 
187                 raw_spin_lock(&fbc->lock);
188                 pcount = per_cpu_ptr(fbc->counters, cpu);
189                 fbc->count += *pcount;
190                 *pcount = 0;
191                 raw_spin_unlock(&fbc->lock);
192         }
193         spin_unlock_irq(&percpu_counters_lock);
194 #endif
195         return 0;
196 }
197 
198 /*
199  * Compare counter against given value.
200  * Return 1 if greater, 0 if equal and -1 if less
201  */
202 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
203 {
204         s64     count;
205 
206         count = percpu_counter_read(fbc);
207         /* Check to see if rough count will be sufficient for comparison */
208         if (abs(count - rhs) > (batch * num_online_cpus())) {
209                 if (count > rhs)
210                         return 1;
211                 else
212                         return -1;
213         }
214         /* Need to use precise count */
215         count = percpu_counter_sum(fbc);
216         if (count > rhs)
217                 return 1;
218         else if (count < rhs)
219                 return -1;
220         else
221                 return 0;
222 }
223 EXPORT_SYMBOL(__percpu_counter_compare);
224 
225 static int __init percpu_counter_startup(void)
226 {
227         int ret;
228 
229         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
230                                 compute_batch_value, NULL);
231         WARN_ON(ret < 0);
232         ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
233                                         "lib/percpu_cnt:dead", NULL,
234                                         percpu_counter_cpu_dead);
235         WARN_ON(ret < 0);
236         return 0;
237 }
238 module_init(percpu_counter_startup);
239 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp