~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/vmstat.h

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 #ifndef _LINUX_VMSTAT_H
  3 #define _LINUX_VMSTAT_H
  4 
  5 #include <linux/types.h>
  6 #include <linux/percpu.h>
  7 #include <linux/mmzone.h>
  8 #include <linux/vm_event_item.h>
  9 #include <linux/atomic.h>
 10 #include <linux/static_key.h>
 11 #include <linux/mmdebug.h>
 12 
 13 extern int sysctl_stat_interval;
 14 
 15 #ifdef CONFIG_NUMA
 16 #define ENABLE_NUMA_STAT   1
 17 #define DISABLE_NUMA_STAT   0
 18 extern int sysctl_vm_numa_stat;
 19 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
 20 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
 21                 void *buffer, size_t *length, loff_t *ppos);
 22 #endif
 23 
 24 struct reclaim_stat {
 25         unsigned nr_dirty;
 26         unsigned nr_unqueued_dirty;
 27         unsigned nr_congested;
 28         unsigned nr_writeback;
 29         unsigned nr_immediate;
 30         unsigned nr_pageout;
 31         unsigned nr_activate[ANON_AND_FILE];
 32         unsigned nr_ref_keep;
 33         unsigned nr_unmap_fail;
 34         unsigned nr_lazyfree_fail;
 35 };
 36 
 37 enum writeback_stat_item {
 38         NR_DIRTY_THRESHOLD,
 39         NR_DIRTY_BG_THRESHOLD,
 40         NR_VM_WRITEBACK_STAT_ITEMS,
 41 };
 42 
 43 #ifdef CONFIG_VM_EVENT_COUNTERS
 44 /*
 45  * Light weight per cpu counter implementation.
 46  *
 47  * Counters should only be incremented and no critical kernel component
 48  * should rely on the counter values.
 49  *
 50  * Counters are handled completely inline. On many platforms the code
 51  * generated will simply be the increment of a global address.
 52  */
 53 
 54 struct vm_event_state {
 55         unsigned long event[NR_VM_EVENT_ITEMS];
 56 };
 57 
 58 DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
 59 
 60 /*
 61  * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
 62  * local_irq_disable overhead.
 63  */
 64 static inline void __count_vm_event(enum vm_event_item item)
 65 {
 66         raw_cpu_inc(vm_event_states.event[item]);
 67 }
 68 
 69 static inline void count_vm_event(enum vm_event_item item)
 70 {
 71         this_cpu_inc(vm_event_states.event[item]);
 72 }
 73 
 74 static inline void __count_vm_events(enum vm_event_item item, long delta)
 75 {
 76         raw_cpu_add(vm_event_states.event[item], delta);
 77 }
 78 
 79 static inline void count_vm_events(enum vm_event_item item, long delta)
 80 {
 81         this_cpu_add(vm_event_states.event[item], delta);
 82 }
 83 
 84 extern void all_vm_events(unsigned long *);
 85 
 86 extern void vm_events_fold_cpu(int cpu);
 87 
 88 #else
 89 
 90 /* Disable counters */
 91 static inline void count_vm_event(enum vm_event_item item)
 92 {
 93 }
 94 static inline void count_vm_events(enum vm_event_item item, long delta)
 95 {
 96 }
 97 static inline void __count_vm_event(enum vm_event_item item)
 98 {
 99 }
100 static inline void __count_vm_events(enum vm_event_item item, long delta)
101 {
102 }
103 static inline void all_vm_events(unsigned long *ret)
104 {
105 }
106 static inline void vm_events_fold_cpu(int cpu)
107 {
108 }
109 
110 #endif /* CONFIG_VM_EVENT_COUNTERS */
111 
112 #ifdef CONFIG_NUMA_BALANCING
113 #define count_vm_numa_event(x)     count_vm_event(x)
114 #define count_vm_numa_events(x, y) count_vm_events(x, y)
115 #else
116 #define count_vm_numa_event(x) do {} while (0)
117 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
118 #endif /* CONFIG_NUMA_BALANCING */
119 
120 #ifdef CONFIG_DEBUG_TLBFLUSH
121 #define count_vm_tlb_event(x)      count_vm_event(x)
122 #define count_vm_tlb_events(x, y)  count_vm_events(x, y)
123 #else
124 #define count_vm_tlb_event(x)     do {} while (0)
125 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
126 #endif
127 
128 #ifdef CONFIG_DEBUG_VM_VMACACHE
129 #define count_vm_vmacache_event(x) count_vm_event(x)
130 #else
131 #define count_vm_vmacache_event(x) do {} while (0)
132 #endif
133 
134 #define __count_zid_vm_events(item, zid, delta) \
135         __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
136 
137 /*
138  * Zone and node-based page accounting with per cpu differentials.
139  */
140 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
141 extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
142 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
143 
144 #ifdef CONFIG_NUMA
145 static inline void zone_numa_state_add(long x, struct zone *zone,
146                                  enum numa_stat_item item)
147 {
148         atomic_long_add(x, &zone->vm_numa_stat[item]);
149         atomic_long_add(x, &vm_numa_stat[item]);
150 }
151 
152 static inline unsigned long global_numa_state(enum numa_stat_item item)
153 {
154         long x = atomic_long_read(&vm_numa_stat[item]);
155 
156         return x;
157 }
158 
159 static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
160                                         enum numa_stat_item item)
161 {
162         long x = atomic_long_read(&zone->vm_numa_stat[item]);
163         int cpu;
164 
165         for_each_online_cpu(cpu)
166                 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
167 
168         return x;
169 }
170 #endif /* CONFIG_NUMA */
171 
172 static inline void zone_page_state_add(long x, struct zone *zone,
173                                  enum zone_stat_item item)
174 {
175         atomic_long_add(x, &zone->vm_stat[item]);
176         atomic_long_add(x, &vm_zone_stat[item]);
177 }
178 
179 static inline void node_page_state_add(long x, struct pglist_data *pgdat,
180                                  enum node_stat_item item)
181 {
182         atomic_long_add(x, &pgdat->vm_stat[item]);
183         atomic_long_add(x, &vm_node_stat[item]);
184 }
185 
186 static inline unsigned long global_zone_page_state(enum zone_stat_item item)
187 {
188         long x = atomic_long_read(&vm_zone_stat[item]);
189 #ifdef CONFIG_SMP
190         if (x < 0)
191                 x = 0;
192 #endif
193         return x;
194 }
195 
196 static inline
197 unsigned long global_node_page_state_pages(enum node_stat_item item)
198 {
199         long x = atomic_long_read(&vm_node_stat[item]);
200 #ifdef CONFIG_SMP
201         if (x < 0)
202                 x = 0;
203 #endif
204         return x;
205 }
206 
207 static inline unsigned long global_node_page_state(enum node_stat_item item)
208 {
209         VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
210 
211         return global_node_page_state_pages(item);
212 }
213 
214 static inline unsigned long zone_page_state(struct zone *zone,
215                                         enum zone_stat_item item)
216 {
217         long x = atomic_long_read(&zone->vm_stat[item]);
218 #ifdef CONFIG_SMP
219         if (x < 0)
220                 x = 0;
221 #endif
222         return x;
223 }
224 
225 /*
226  * More accurate version that also considers the currently pending
227  * deltas. For that we need to loop over all cpus to find the current
228  * deltas. There is no synchronization so the result cannot be
229  * exactly accurate either.
230  */
231 static inline unsigned long zone_page_state_snapshot(struct zone *zone,
232                                         enum zone_stat_item item)
233 {
234         long x = atomic_long_read(&zone->vm_stat[item]);
235 
236 #ifdef CONFIG_SMP
237         int cpu;
238         for_each_online_cpu(cpu)
239                 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
240 
241         if (x < 0)
242                 x = 0;
243 #endif
244         return x;
245 }
246 
247 #ifdef CONFIG_NUMA
248 extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
249 extern unsigned long sum_zone_node_page_state(int node,
250                                               enum zone_stat_item item);
251 extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
252 extern unsigned long node_page_state(struct pglist_data *pgdat,
253                                                 enum node_stat_item item);
254 extern unsigned long node_page_state_pages(struct pglist_data *pgdat,
255                                            enum node_stat_item item);
256 #else
257 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
258 #define node_page_state(node, item) global_node_page_state(item)
259 #define node_page_state_pages(node, item) global_node_page_state_pages(item)
260 #endif /* CONFIG_NUMA */
261 
262 #ifdef CONFIG_SMP
263 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
264 void __inc_zone_page_state(struct page *, enum zone_stat_item);
265 void __dec_zone_page_state(struct page *, enum zone_stat_item);
266 
267 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
268 void __inc_node_page_state(struct page *, enum node_stat_item);
269 void __dec_node_page_state(struct page *, enum node_stat_item);
270 
271 void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
272 void inc_zone_page_state(struct page *, enum zone_stat_item);
273 void dec_zone_page_state(struct page *, enum zone_stat_item);
274 
275 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
276 void inc_node_page_state(struct page *, enum node_stat_item);
277 void dec_node_page_state(struct page *, enum node_stat_item);
278 
279 extern void inc_node_state(struct pglist_data *, enum node_stat_item);
280 extern void __inc_zone_state(struct zone *, enum zone_stat_item);
281 extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
282 extern void dec_zone_state(struct zone *, enum zone_stat_item);
283 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
284 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
285 
286 void quiet_vmstat(void);
287 void cpu_vm_stats_fold(int cpu);
288 void refresh_zone_stat_thresholds(void);
289 
290 struct ctl_table;
291 int vmstat_refresh(struct ctl_table *, int write, void *buffer, size_t *lenp,
292                 loff_t *ppos);
293 
294 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
295 
296 int calculate_pressure_threshold(struct zone *zone);
297 int calculate_normal_threshold(struct zone *zone);
298 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
299                                 int (*calculate_pressure)(struct zone *));
300 #else /* CONFIG_SMP */
301 
302 /*
303  * We do not maintain differentials in a single processor configuration.
304  * The functions directly modify the zone and global counters.
305  */
306 static inline void __mod_zone_page_state(struct zone *zone,
307                         enum zone_stat_item item, long delta)
308 {
309         zone_page_state_add(delta, zone, item);
310 }
311 
312 static inline void __mod_node_page_state(struct pglist_data *pgdat,
313                         enum node_stat_item item, int delta)
314 {
315         if (vmstat_item_in_bytes(item)) {
316                 /*
317                  * Only cgroups use subpage accounting right now; at
318                  * the global level, these items still change in
319                  * multiples of whole pages. Store them as pages
320                  * internally to keep the per-cpu counters compact.
321                  */
322                 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
323                 delta >>= PAGE_SHIFT;
324         }
325 
326         node_page_state_add(delta, pgdat, item);
327 }
328 
329 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
330 {
331         atomic_long_inc(&zone->vm_stat[item]);
332         atomic_long_inc(&vm_zone_stat[item]);
333 }
334 
335 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
336 {
337         atomic_long_inc(&pgdat->vm_stat[item]);
338         atomic_long_inc(&vm_node_stat[item]);
339 }
340 
341 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
342 {
343         atomic_long_dec(&zone->vm_stat[item]);
344         atomic_long_dec(&vm_zone_stat[item]);
345 }
346 
347 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
348 {
349         atomic_long_dec(&pgdat->vm_stat[item]);
350         atomic_long_dec(&vm_node_stat[item]);
351 }
352 
353 static inline void __inc_zone_page_state(struct page *page,
354                         enum zone_stat_item item)
355 {
356         __inc_zone_state(page_zone(page), item);
357 }
358 
359 static inline void __inc_node_page_state(struct page *page,
360                         enum node_stat_item item)
361 {
362         __inc_node_state(page_pgdat(page), item);
363 }
364 
365 
366 static inline void __dec_zone_page_state(struct page *page,
367                         enum zone_stat_item item)
368 {
369         __dec_zone_state(page_zone(page), item);
370 }
371 
372 static inline void __dec_node_page_state(struct page *page,
373                         enum node_stat_item item)
374 {
375         __dec_node_state(page_pgdat(page), item);
376 }
377 
378 
379 /*
380  * We only use atomic operations to update counters. So there is no need to
381  * disable interrupts.
382  */
383 #define inc_zone_page_state __inc_zone_page_state
384 #define dec_zone_page_state __dec_zone_page_state
385 #define mod_zone_page_state __mod_zone_page_state
386 
387 #define inc_node_page_state __inc_node_page_state
388 #define dec_node_page_state __dec_node_page_state
389 #define mod_node_page_state __mod_node_page_state
390 
391 #define inc_zone_state __inc_zone_state
392 #define inc_node_state __inc_node_state
393 #define dec_zone_state __dec_zone_state
394 
395 #define set_pgdat_percpu_threshold(pgdat, callback) { }
396 
397 static inline void refresh_zone_stat_thresholds(void) { }
398 static inline void cpu_vm_stats_fold(int cpu) { }
399 static inline void quiet_vmstat(void) { }
400 
401 static inline void drain_zonestat(struct zone *zone,
402                         struct per_cpu_pageset *pset) { }
403 #endif          /* CONFIG_SMP */
404 
405 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
406                                              int migratetype)
407 {
408         __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
409         if (is_migrate_cma(migratetype))
410                 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
411 }
412 
413 extern const char * const vmstat_text[];
414 
415 static inline const char *zone_stat_name(enum zone_stat_item item)
416 {
417         return vmstat_text[item];
418 }
419 
420 #ifdef CONFIG_NUMA
421 static inline const char *numa_stat_name(enum numa_stat_item item)
422 {
423         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
424                            item];
425 }
426 #endif /* CONFIG_NUMA */
427 
428 static inline const char *node_stat_name(enum node_stat_item item)
429 {
430         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
431                            NR_VM_NUMA_STAT_ITEMS +
432                            item];
433 }
434 
435 static inline const char *lru_list_name(enum lru_list lru)
436 {
437         return node_stat_name(NR_LRU_BASE + lru) + 3; // skip "nr_"
438 }
439 
440 static inline const char *writeback_stat_name(enum writeback_stat_item item)
441 {
442         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
443                            NR_VM_NUMA_STAT_ITEMS +
444                            NR_VM_NODE_STAT_ITEMS +
445                            item];
446 }
447 
448 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
449 static inline const char *vm_event_name(enum vm_event_item item)
450 {
451         return vmstat_text[NR_VM_ZONE_STAT_ITEMS +
452                            NR_VM_NUMA_STAT_ITEMS +
453                            NR_VM_NODE_STAT_ITEMS +
454                            NR_VM_WRITEBACK_STAT_ITEMS +
455                            item];
456 }
457 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
458 
459 #ifdef CONFIG_MEMCG
460 
461 void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
462                         int val);
463 
464 static inline void mod_lruvec_state(struct lruvec *lruvec,
465                                     enum node_stat_item idx, int val)
466 {
467         unsigned long flags;
468 
469         local_irq_save(flags);
470         __mod_lruvec_state(lruvec, idx, val);
471         local_irq_restore(flags);
472 }
473 
474 void __mod_lruvec_page_state(struct page *page,
475                              enum node_stat_item idx, int val);
476 
477 static inline void mod_lruvec_page_state(struct page *page,
478                                          enum node_stat_item idx, int val)
479 {
480         unsigned long flags;
481 
482         local_irq_save(flags);
483         __mod_lruvec_page_state(page, idx, val);
484         local_irq_restore(flags);
485 }
486 
487 #else
488 
489 static inline void __mod_lruvec_state(struct lruvec *lruvec,
490                                       enum node_stat_item idx, int val)
491 {
492         __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
493 }
494 
495 static inline void mod_lruvec_state(struct lruvec *lruvec,
496                                     enum node_stat_item idx, int val)
497 {
498         mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
499 }
500 
501 static inline void __mod_lruvec_page_state(struct page *page,
502                                            enum node_stat_item idx, int val)
503 {
504         __mod_node_page_state(page_pgdat(page), idx, val);
505 }
506 
507 static inline void mod_lruvec_page_state(struct page *page,
508                                          enum node_stat_item idx, int val)
509 {
510         mod_node_page_state(page_pgdat(page), idx, val);
511 }
512 
513 #endif /* CONFIG_MEMCG */
514 
515 static inline void __inc_lruvec_state(struct lruvec *lruvec,
516                                       enum node_stat_item idx)
517 {
518         __mod_lruvec_state(lruvec, idx, 1);
519 }
520 
521 static inline void __dec_lruvec_state(struct lruvec *lruvec,
522                                       enum node_stat_item idx)
523 {
524         __mod_lruvec_state(lruvec, idx, -1);
525 }
526 
527 static inline void __inc_lruvec_page_state(struct page *page,
528                                            enum node_stat_item idx)
529 {
530         __mod_lruvec_page_state(page, idx, 1);
531 }
532 
533 static inline void __dec_lruvec_page_state(struct page *page,
534                                            enum node_stat_item idx)
535 {
536         __mod_lruvec_page_state(page, idx, -1);
537 }
538 
539 static inline void inc_lruvec_state(struct lruvec *lruvec,
540                                     enum node_stat_item idx)
541 {
542         mod_lruvec_state(lruvec, idx, 1);
543 }
544 
545 static inline void dec_lruvec_state(struct lruvec *lruvec,
546                                     enum node_stat_item idx)
547 {
548         mod_lruvec_state(lruvec, idx, -1);
549 }
550 
551 static inline void inc_lruvec_page_state(struct page *page,
552                                          enum node_stat_item idx)
553 {
554         mod_lruvec_page_state(page, idx, 1);
555 }
556 
557 static inline void dec_lruvec_page_state(struct page *page,
558                                          enum node_stat_item idx)
559 {
560         mod_lruvec_page_state(page, idx, -1);
561 }
562 
563 #endif /* _LINUX_VMSTAT_H */
564 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp