~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/vmstat.h

Version: ~ [ linux-5.2 ] ~ [ linux-5.1.16 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.57 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.132 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.184 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.184 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.69 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 #ifndef _LINUX_VMSTAT_H
  3 #define _LINUX_VMSTAT_H
  4 
  5 #include <linux/types.h>
  6 #include <linux/percpu.h>
  7 #include <linux/mmzone.h>
  8 #include <linux/vm_event_item.h>
  9 #include <linux/atomic.h>
 10 #include <linux/static_key.h>
 11 
 12 extern int sysctl_stat_interval;
 13 
 14 #ifdef CONFIG_NUMA
 15 #define ENABLE_NUMA_STAT   1
 16 #define DISABLE_NUMA_STAT   0
 17 extern int sysctl_vm_numa_stat;
 18 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
 19 extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
 20                 int write, void __user *buffer, size_t *length, loff_t *ppos);
 21 #endif
 22 
 23 #ifdef CONFIG_VM_EVENT_COUNTERS
 24 /*
 25  * Light weight per cpu counter implementation.
 26  *
 27  * Counters should only be incremented and no critical kernel component
 28  * should rely on the counter values.
 29  *
 30  * Counters are handled completely inline. On many platforms the code
 31  * generated will simply be the increment of a global address.
 32  */
 33 
 34 struct vm_event_state {
 35         unsigned long event[NR_VM_EVENT_ITEMS];
 36 };
 37 
 38 DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
 39 
 40 /*
 41  * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 42  * local_irq_disable overhead.
 43  */
 44 static inline void __count_vm_event(enum vm_event_item item)
 45 {
 46         raw_cpu_inc(vm_event_states.event[item]);
 47 }
 48 
 49 static inline void count_vm_event(enum vm_event_item item)
 50 {
 51         this_cpu_inc(vm_event_states.event[item]);
 52 }
 53 
 54 static inline void __count_vm_events(enum vm_event_item item, long delta)
 55 {
 56         raw_cpu_add(vm_event_states.event[item], delta);
 57 }
 58 
 59 static inline void count_vm_events(enum vm_event_item item, long delta)
 60 {
 61         this_cpu_add(vm_event_states.event[item], delta);
 62 }
 63 
 64 extern void all_vm_events(unsigned long *);
 65 
 66 extern void vm_events_fold_cpu(int cpu);
 67 
 68 #else
 69 
 70 /* Disable counters */
 71 static inline void count_vm_event(enum vm_event_item item)
 72 {
 73 }
 74 static inline void count_vm_events(enum vm_event_item item, long delta)
 75 {
 76 }
 77 static inline void __count_vm_event(enum vm_event_item item)
 78 {
 79 }
 80 static inline void __count_vm_events(enum vm_event_item item, long delta)
 81 {
 82 }
 83 static inline void all_vm_events(unsigned long *ret)
 84 {
 85 }
 86 static inline void vm_events_fold_cpu(int cpu)
 87 {
 88 }
 89 
 90 #endif /* CONFIG_VM_EVENT_COUNTERS */
 91 
 92 #ifdef CONFIG_NUMA_BALANCING
 93 #define count_vm_numa_event(x)     count_vm_event(x)
 94 #define count_vm_numa_events(x, y) count_vm_events(x, y)
 95 #else
 96 #define count_vm_numa_event(x) do {} while (0)
 97 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
 98 #endif /* CONFIG_NUMA_BALANCING */
 99 
100 #ifdef CONFIG_DEBUG_TLBFLUSH
101 #define count_vm_tlb_event(x)      count_vm_event(x)
102 #define count_vm_tlb_events(x, y)  count_vm_events(x, y)
103 #else
104 #define count_vm_tlb_event(x)     do {} while (0)
105 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
106 #endif
107 
108 #ifdef CONFIG_DEBUG_VM_VMACACHE
109 #define count_vm_vmacache_event(x) count_vm_event(x)
110 #else
111 #define count_vm_vmacache_event(x) do {} while (0)
112 #endif
113 
114 #define __count_zid_vm_events(item, zid, delta) \
115         __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
116 
117 /*
118  * Zone and node-based page accounting with per cpu differentials.
119  */
120 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
121 extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
122 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
123 
124 #ifdef CONFIG_NUMA
125 static inline void zone_numa_state_add(long x, struct zone *zone,
126                                  enum numa_stat_item item)
127 {
128         atomic_long_add(x, &zone->vm_numa_stat[item]);
129         atomic_long_add(x, &vm_numa_stat[item]);
130 }
131 
132 static inline unsigned long global_numa_state(enum numa_stat_item item)
133 {
134         long x = atomic_long_read(&vm_numa_stat[item]);
135 
136         return x;
137 }
138 
139 static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
140                                         enum numa_stat_item item)
141 {
142         long x = atomic_long_read(&zone->vm_numa_stat[item]);
143         int cpu;
144 
145         for_each_online_cpu(cpu)
146                 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
147 
148         return x;
149 }
150 #endif /* CONFIG_NUMA */
151 
152 static inline void zone_page_state_add(long x, struct zone *zone,
153                                  enum zone_stat_item item)
154 {
155         atomic_long_add(x, &zone->vm_stat[item]);
156         atomic_long_add(x, &vm_zone_stat[item]);
157 }
158 
159 static inline void node_page_state_add(long x, struct pglist_data *pgdat,
160                                  enum node_stat_item item)
161 {
162         atomic_long_add(x, &pgdat->vm_stat[item]);
163         atomic_long_add(x, &vm_node_stat[item]);
164 }
165 
166 static inline unsigned long global_zone_page_state(enum zone_stat_item item)
167 {
168         long x = atomic_long_read(&vm_zone_stat[item]);
169 #ifdef CONFIG_SMP
170         if (x < 0)
171                 x = 0;
172 #endif
173         return x;
174 }
175 
176 static inline unsigned long global_node_page_state(enum node_stat_item item)
177 {
178         long x = atomic_long_read(&vm_node_stat[item]);
179 #ifdef CONFIG_SMP
180         if (x < 0)
181                 x = 0;
182 #endif
183         return x;
184 }
185 
186 static inline unsigned long zone_page_state(struct zone *zone,
187                                         enum zone_stat_item item)
188 {
189         long x = atomic_long_read(&zone->vm_stat[item]);
190 #ifdef CONFIG_SMP
191         if (x < 0)
192                 x = 0;
193 #endif
194         return x;
195 }
196 
197 /*
198  * More accurate version that also considers the currently pending
199  * deltas. For that we need to loop over all cpus to find the current
200  * deltas. There is no synchronization so the result cannot be
201  * exactly accurate either.
202  */
203 static inline unsigned long zone_page_state_snapshot(struct zone *zone,
204                                         enum zone_stat_item item)
205 {
206         long x = atomic_long_read(&zone->vm_stat[item]);
207 
208 #ifdef CONFIG_SMP
209         int cpu;
210         for_each_online_cpu(cpu)
211                 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
212 
213         if (x < 0)
214                 x = 0;
215 #endif
216         return x;
217 }
218 
219 #ifdef CONFIG_NUMA
220 extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
221 extern unsigned long sum_zone_node_page_state(int node,
222                                               enum zone_stat_item item);
223 extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
224 extern unsigned long node_page_state(struct pglist_data *pgdat,
225                                                 enum node_stat_item item);
226 #else
227 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
228 #define node_page_state(node, item) global_node_page_state(item)
229 #endif /* CONFIG_NUMA */
230 
231 #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
232 #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
233 #define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
234 #define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))
235 
236 #ifdef CONFIG_SMP
237 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
238 void __inc_zone_page_state(struct page *, enum zone_stat_item);
239 void __dec_zone_page_state(struct page *, enum zone_stat_item);
240 
241 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
242 void __inc_node_page_state(struct page *, enum node_stat_item);
243 void __dec_node_page_state(struct page *, enum node_stat_item);
244 
245 void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
246 void inc_zone_page_state(struct page *, enum zone_stat_item);
247 void dec_zone_page_state(struct page *, enum zone_stat_item);
248 
249 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
250 void inc_node_page_state(struct page *, enum node_stat_item);
251 void dec_node_page_state(struct page *, enum node_stat_item);
252 
253 extern void inc_node_state(struct pglist_data *, enum node_stat_item);
254 extern void __inc_zone_state(struct zone *, enum zone_stat_item);
255 extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
256 extern void dec_zone_state(struct zone *, enum zone_stat_item);
257 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
258 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
259 
260 void quiet_vmstat(void);
261 void cpu_vm_stats_fold(int cpu);
262 void refresh_zone_stat_thresholds(void);
263 
264 struct ctl_table;
265 int vmstat_refresh(struct ctl_table *, int write,
266                    void __user *buffer, size_t *lenp, loff_t *ppos);
267 
268 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
269 
270 int calculate_pressure_threshold(struct zone *zone);
271 int calculate_normal_threshold(struct zone *zone);
272 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
273                                 int (*calculate_pressure)(struct zone *));
274 #else /* CONFIG_SMP */
275 
276 /*
277  * We do not maintain differentials in a single processor configuration.
278  * The functions directly modify the zone and global counters.
279  */
280 static inline void __mod_zone_page_state(struct zone *zone,
281                         enum zone_stat_item item, long delta)
282 {
283         zone_page_state_add(delta, zone, item);
284 }
285 
286 static inline void __mod_node_page_state(struct pglist_data *pgdat,
287                         enum node_stat_item item, int delta)
288 {
289         node_page_state_add(delta, pgdat, item);
290 }
291 
292 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
293 {
294         atomic_long_inc(&zone->vm_stat[item]);
295         atomic_long_inc(&vm_zone_stat[item]);
296 }
297 
298 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
299 {
300         atomic_long_inc(&pgdat->vm_stat[item]);
301         atomic_long_inc(&vm_node_stat[item]);
302 }
303 
304 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
305 {
306         atomic_long_dec(&zone->vm_stat[item]);
307         atomic_long_dec(&vm_zone_stat[item]);
308 }
309 
310 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
311 {
312         atomic_long_dec(&pgdat->vm_stat[item]);
313         atomic_long_dec(&vm_node_stat[item]);
314 }
315 
316 static inline void __inc_zone_page_state(struct page *page,
317                         enum zone_stat_item item)
318 {
319         __inc_zone_state(page_zone(page), item);
320 }
321 
322 static inline void __inc_node_page_state(struct page *page,
323                         enum node_stat_item item)
324 {
325         __inc_node_state(page_pgdat(page), item);
326 }
327 
328 
329 static inline void __dec_zone_page_state(struct page *page,
330                         enum zone_stat_item item)
331 {
332         __dec_zone_state(page_zone(page), item);
333 }
334 
335 static inline void __dec_node_page_state(struct page *page,
336                         enum node_stat_item item)
337 {
338         __dec_node_state(page_pgdat(page), item);
339 }
340 
341 
342 /*
343  * We only use atomic operations to update counters. So there is no need to
344  * disable interrupts.
345  */
346 #define inc_zone_page_state __inc_zone_page_state
347 #define dec_zone_page_state __dec_zone_page_state
348 #define mod_zone_page_state __mod_zone_page_state
349 
350 #define inc_node_page_state __inc_node_page_state
351 #define dec_node_page_state __dec_node_page_state
352 #define mod_node_page_state __mod_node_page_state
353 
354 #define inc_zone_state __inc_zone_state
355 #define inc_node_state __inc_node_state
356 #define dec_zone_state __dec_zone_state
357 
358 #define set_pgdat_percpu_threshold(pgdat, callback) { }
359 
360 static inline void refresh_zone_stat_thresholds(void) { }
361 static inline void cpu_vm_stats_fold(int cpu) { }
362 static inline void quiet_vmstat(void) { }
363 
364 static inline void drain_zonestat(struct zone *zone,
365                         struct per_cpu_pageset *pset) { }
366 #endif          /* CONFIG_SMP */
367 
368 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
369                                              int migratetype)
370 {
371         __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
372         if (is_migrate_cma(migratetype))
373                 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
374 }
375 
376 extern const char * const vmstat_text[];
377 
378 #endif /* _LINUX_VMSTAT_H */
379 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp