~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/kernel/cgroup/rstat.c

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 #include "cgroup-internal.h"
  3 
  4 #include <linux/sched/cputime.h>
  5 
  6 static DEFINE_SPINLOCK(cgroup_rstat_lock);
  7 static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock);
  8 
  9 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu);
 10 
 11 static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)
 12 {
 13         return per_cpu_ptr(cgrp->rstat_cpu, cpu);
 14 }
 15 
 16 /**
 17  * cgroup_rstat_updated - keep track of updated rstat_cpu
 18  * @cgrp: target cgroup
 19  * @cpu: cpu on which rstat_cpu was updated
 20  *
 21  * @cgrp's rstat_cpu on @cpu was updated.  Put it on the parent's matching
 22  * rstat_cpu->updated_children list.  See the comment on top of
 23  * cgroup_rstat_cpu definition for details.
 24  */
 25 void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
 26 {
 27         raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
 28         struct cgroup *parent;
 29         unsigned long flags;
 30 
 31         /* nothing to do for root */
 32         if (!cgroup_parent(cgrp))
 33                 return;
 34 
 35         /*
 36          * Paired with the one in cgroup_rstat_cpu_pop_upated().  Either we
 37          * see NULL updated_next or they see our updated stat.
 38          */
 39         smp_mb();
 40 
 41         /*
 42          * Because @parent's updated_children is terminated with @parent
 43          * instead of NULL, we can tell whether @cgrp is on the list by
 44          * testing the next pointer for NULL.
 45          */
 46         if (cgroup_rstat_cpu(cgrp, cpu)->updated_next)
 47                 return;
 48 
 49         raw_spin_lock_irqsave(cpu_lock, flags);
 50 
 51         /* put @cgrp and all ancestors on the corresponding updated lists */
 52         for (parent = cgroup_parent(cgrp); parent;
 53              cgrp = parent, parent = cgroup_parent(cgrp)) {
 54                 struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
 55                 struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
 56 
 57                 /*
 58                  * Both additions and removals are bottom-up.  If a cgroup
 59                  * is already in the tree, all ancestors are.
 60                  */
 61                 if (rstatc->updated_next)
 62                         break;
 63 
 64                 rstatc->updated_next = prstatc->updated_children;
 65                 prstatc->updated_children = cgrp;
 66         }
 67 
 68         raw_spin_unlock_irqrestore(cpu_lock, flags);
 69 }
 70 EXPORT_SYMBOL_GPL(cgroup_rstat_updated);
 71 
 72 /**
 73  * cgroup_rstat_cpu_pop_updated - iterate and dismantle rstat_cpu updated tree
 74  * @pos: current position
 75  * @root: root of the tree to traversal
 76  * @cpu: target cpu
 77  *
 78  * Walks the udpated rstat_cpu tree on @cpu from @root.  %NULL @pos starts
 79  * the traversal and %NULL return indicates the end.  During traversal,
 80  * each returned cgroup is unlinked from the tree.  Must be called with the
 81  * matching cgroup_rstat_cpu_lock held.
 82  *
 83  * The only ordering guarantee is that, for a parent and a child pair
 84  * covered by a given traversal, if a child is visited, its parent is
 85  * guaranteed to be visited afterwards.
 86  */
 87 static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
 88                                                    struct cgroup *root, int cpu)
 89 {
 90         struct cgroup_rstat_cpu *rstatc;
 91 
 92         if (pos == root)
 93                 return NULL;
 94 
 95         /*
 96          * We're gonna walk down to the first leaf and visit/remove it.  We
 97          * can pick whatever unvisited node as the starting point.
 98          */
 99         if (!pos)
100                 pos = root;
101         else
102                 pos = cgroup_parent(pos);
103 
104         /* walk down to the first leaf */
105         while (true) {
106                 rstatc = cgroup_rstat_cpu(pos, cpu);
107                 if (rstatc->updated_children == pos)
108                         break;
109                 pos = rstatc->updated_children;
110         }
111 
112         /*
113          * Unlink @pos from the tree.  As the updated_children list is
114          * singly linked, we have to walk it to find the removal point.
115          * However, due to the way we traverse, @pos will be the first
116          * child in most cases. The only exception is @root.
117          */
118         if (rstatc->updated_next) {
119                 struct cgroup *parent = cgroup_parent(pos);
120                 struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
121                 struct cgroup_rstat_cpu *nrstatc;
122                 struct cgroup **nextp;
123 
124                 nextp = &prstatc->updated_children;
125                 while (true) {
126                         nrstatc = cgroup_rstat_cpu(*nextp, cpu);
127                         if (*nextp == pos)
128                                 break;
129 
130                         WARN_ON_ONCE(*nextp == parent);
131                         nextp = &nrstatc->updated_next;
132                 }
133 
134                 *nextp = rstatc->updated_next;
135                 rstatc->updated_next = NULL;
136 
137                 /*
138                  * Paired with the one in cgroup_rstat_cpu_updated().
139                  * Either they see NULL updated_next or we see their
140                  * updated stat.
141                  */
142                 smp_mb();
143 
144                 return pos;
145         }
146 
147         /* only happens for @root */
148         return NULL;
149 }
150 
151 /* see cgroup_rstat_flush() */
152 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)
153         __releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock)
154 {
155         int cpu;
156 
157         lockdep_assert_held(&cgroup_rstat_lock);
158 
159         for_each_possible_cpu(cpu) {
160                 raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock,
161                                                        cpu);
162                 struct cgroup *pos = NULL;
163 
164                 raw_spin_lock(cpu_lock);
165                 while ((pos = cgroup_rstat_cpu_pop_updated(pos, cgrp, cpu))) {
166                         struct cgroup_subsys_state *css;
167 
168                         cgroup_base_stat_flush(pos, cpu);
169 
170                         rcu_read_lock();
171                         list_for_each_entry_rcu(css, &pos->rstat_css_list,
172                                                 rstat_css_node)
173                                 css->ss->css_rstat_flush(css, cpu);
174                         rcu_read_unlock();
175                 }
176                 raw_spin_unlock(cpu_lock);
177 
178                 /* if @may_sleep, play nice and yield if necessary */
179                 if (may_sleep && (need_resched() ||
180                                   spin_needbreak(&cgroup_rstat_lock))) {
181                         spin_unlock_irq(&cgroup_rstat_lock);
182                         if (!cond_resched())
183                                 cpu_relax();
184                         spin_lock_irq(&cgroup_rstat_lock);
185                 }
186         }
187 }
188 
189 /**
190  * cgroup_rstat_flush - flush stats in @cgrp's subtree
191  * @cgrp: target cgroup
192  *
193  * Collect all per-cpu stats in @cgrp's subtree into the global counters
194  * and propagate them upwards.  After this function returns, all cgroups in
195  * the subtree have up-to-date ->stat.
196  *
197  * This also gets all cgroups in the subtree including @cgrp off the
198  * ->updated_children lists.
199  *
200  * This function may block.
201  */
202 void cgroup_rstat_flush(struct cgroup *cgrp)
203 {
204         might_sleep();
205 
206         spin_lock_irq(&cgroup_rstat_lock);
207         cgroup_rstat_flush_locked(cgrp, true);
208         spin_unlock_irq(&cgroup_rstat_lock);
209 }
210 
211 /**
212  * cgroup_rstat_flush_irqsafe - irqsafe version of cgroup_rstat_flush()
213  * @cgrp: target cgroup
214  *
215  * This function can be called from any context.
216  */
217 void cgroup_rstat_flush_irqsafe(struct cgroup *cgrp)
218 {
219         unsigned long flags;
220 
221         spin_lock_irqsave(&cgroup_rstat_lock, flags);
222         cgroup_rstat_flush_locked(cgrp, false);
223         spin_unlock_irqrestore(&cgroup_rstat_lock, flags);
224 }
225 
226 /**
227  * cgroup_rstat_flush_begin - flush stats in @cgrp's subtree and hold
228  * @cgrp: target cgroup
229  *
230  * Flush stats in @cgrp's subtree and prevent further flushes.  Must be
231  * paired with cgroup_rstat_flush_release().
232  *
233  * This function may block.
234  */
235 void cgroup_rstat_flush_hold(struct cgroup *cgrp)
236         __acquires(&cgroup_rstat_lock)
237 {
238         might_sleep();
239         spin_lock_irq(&cgroup_rstat_lock);
240         cgroup_rstat_flush_locked(cgrp, true);
241 }
242 
243 /**
244  * cgroup_rstat_flush_release - release cgroup_rstat_flush_hold()
245  */
246 void cgroup_rstat_flush_release(void)
247         __releases(&cgroup_rstat_lock)
248 {
249         spin_unlock_irq(&cgroup_rstat_lock);
250 }
251 
252 int cgroup_rstat_init(struct cgroup *cgrp)
253 {
254         int cpu;
255 
256         /* the root cgrp has rstat_cpu preallocated */
257         if (!cgrp->rstat_cpu) {
258                 cgrp->rstat_cpu = alloc_percpu(struct cgroup_rstat_cpu);
259                 if (!cgrp->rstat_cpu)
260                         return -ENOMEM;
261         }
262 
263         /* ->updated_children list is self terminated */
264         for_each_possible_cpu(cpu) {
265                 struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
266 
267                 rstatc->updated_children = cgrp;
268                 u64_stats_init(&rstatc->bsync);
269         }
270 
271         return 0;
272 }
273 
274 void cgroup_rstat_exit(struct cgroup *cgrp)
275 {
276         int cpu;
277 
278         cgroup_rstat_flush(cgrp);
279 
280         /* sanity check */
281         for_each_possible_cpu(cpu) {
282                 struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
283 
284                 if (WARN_ON_ONCE(rstatc->updated_children != cgrp) ||
285                     WARN_ON_ONCE(rstatc->updated_next))
286                         return;
287         }
288 
289         free_percpu(cgrp->rstat_cpu);
290         cgrp->rstat_cpu = NULL;
291 }
292 
293 void __init cgroup_rstat_boot(void)
294 {
295         int cpu;
296 
297         for_each_possible_cpu(cpu)
298                 raw_spin_lock_init(per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu));
299 
300         BUG_ON(cgroup_rstat_init(&cgrp_dfl_root.cgrp));
301 }
302 
303 /*
304  * Functions for cgroup basic resource statistics implemented on top of
305  * rstat.
306  */
307 static void cgroup_base_stat_add(struct cgroup_base_stat *dst_bstat,
308                                  struct cgroup_base_stat *src_bstat)
309 {
310         dst_bstat->cputime.utime += src_bstat->cputime.utime;
311         dst_bstat->cputime.stime += src_bstat->cputime.stime;
312         dst_bstat->cputime.sum_exec_runtime += src_bstat->cputime.sum_exec_runtime;
313 }
314 
315 static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
316                                  struct cgroup_base_stat *src_bstat)
317 {
318         dst_bstat->cputime.utime -= src_bstat->cputime.utime;
319         dst_bstat->cputime.stime -= src_bstat->cputime.stime;
320         dst_bstat->cputime.sum_exec_runtime -= src_bstat->cputime.sum_exec_runtime;
321 }
322 
323 static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
324 {
325         struct cgroup *parent = cgroup_parent(cgrp);
326         struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
327         struct cgroup_base_stat cur, delta;
328         unsigned seq;
329 
330         /* fetch the current per-cpu values */
331         do {
332                 seq = __u64_stats_fetch_begin(&rstatc->bsync);
333                 cur.cputime = rstatc->bstat.cputime;
334         } while (__u64_stats_fetch_retry(&rstatc->bsync, seq));
335 
336         /* propagate percpu delta to global */
337         delta = cur;
338         cgroup_base_stat_sub(&delta, &rstatc->last_bstat);
339         cgroup_base_stat_add(&cgrp->bstat, &delta);
340         cgroup_base_stat_add(&rstatc->last_bstat, &delta);
341 
342         /* propagate global delta to parent */
343         if (parent) {
344                 delta = cgrp->bstat;
345                 cgroup_base_stat_sub(&delta, &cgrp->last_bstat);
346                 cgroup_base_stat_add(&parent->bstat, &delta);
347                 cgroup_base_stat_add(&cgrp->last_bstat, &delta);
348         }
349 }
350 
351 static struct cgroup_rstat_cpu *
352 cgroup_base_stat_cputime_account_begin(struct cgroup *cgrp)
353 {
354         struct cgroup_rstat_cpu *rstatc;
355 
356         rstatc = get_cpu_ptr(cgrp->rstat_cpu);
357         u64_stats_update_begin(&rstatc->bsync);
358         return rstatc;
359 }
360 
361 static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
362                                                  struct cgroup_rstat_cpu *rstatc)
363 {
364         u64_stats_update_end(&rstatc->bsync);
365         cgroup_rstat_updated(cgrp, smp_processor_id());
366         put_cpu_ptr(rstatc);
367 }
368 
369 void __cgroup_account_cputime(struct cgroup *cgrp, u64 delta_exec)
370 {
371         struct cgroup_rstat_cpu *rstatc;
372 
373         rstatc = cgroup_base_stat_cputime_account_begin(cgrp);
374         rstatc->bstat.cputime.sum_exec_runtime += delta_exec;
375         cgroup_base_stat_cputime_account_end(cgrp, rstatc);
376 }
377 
378 void __cgroup_account_cputime_field(struct cgroup *cgrp,
379                                     enum cpu_usage_stat index, u64 delta_exec)
380 {
381         struct cgroup_rstat_cpu *rstatc;
382 
383         rstatc = cgroup_base_stat_cputime_account_begin(cgrp);
384 
385         switch (index) {
386         case CPUTIME_USER:
387         case CPUTIME_NICE:
388                 rstatc->bstat.cputime.utime += delta_exec;
389                 break;
390         case CPUTIME_SYSTEM:
391         case CPUTIME_IRQ:
392         case CPUTIME_SOFTIRQ:
393                 rstatc->bstat.cputime.stime += delta_exec;
394                 break;
395         default:
396                 break;
397         }
398 
399         cgroup_base_stat_cputime_account_end(cgrp, rstatc);
400 }
401 
402 void cgroup_base_stat_cputime_show(struct seq_file *seq)
403 {
404         struct cgroup *cgrp = seq_css(seq)->cgroup;
405         u64 usage, utime, stime;
406 
407         if (!cgroup_parent(cgrp))
408                 return;
409 
410         cgroup_rstat_flush_hold(cgrp);
411         usage = cgrp->bstat.cputime.sum_exec_runtime;
412         cputime_adjust(&cgrp->bstat.cputime, &cgrp->prev_cputime, &utime, &stime);
413         cgroup_rstat_flush_release();
414 
415         do_div(usage, NSEC_PER_USEC);
416         do_div(utime, NSEC_PER_USEC);
417         do_div(stime, NSEC_PER_USEC);
418 
419         seq_printf(seq, "usage_usec %llu\n"
420                    "user_usec %llu\n"
421                    "system_usec %llu\n",
422                    usage, utime, stime);
423 }
424 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp