~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/kernel/vtime.c

Version: ~ [ linux-5.2-rc4 ] ~ [ linux-5.1.9 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.50 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.125 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.181 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.181 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.68 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *    Virtual cpu timer based timer functions.
  3  *
  4  *    Copyright IBM Corp. 2004, 2012
  5  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
  6  */
  7 
  8 #include <linux/kernel_stat.h>
  9 #include <linux/notifier.h>
 10 #include <linux/kprobes.h>
 11 #include <linux/export.h>
 12 #include <linux/kernel.h>
 13 #include <linux/timex.h>
 14 #include <linux/types.h>
 15 #include <linux/time.h>
 16 #include <linux/cpu.h>
 17 #include <linux/smp.h>
 18 
 19 #include <asm/irq_regs.h>
 20 #include <asm/cputime.h>
 21 #include <asm/vtimer.h>
 22 #include <asm/irq.h>
 23 #include "entry.h"
 24 
 25 static void virt_timer_expire(void);
 26 
 27 DEFINE_PER_CPU(struct s390_idle_data, s390_idle);
 28 
 29 static LIST_HEAD(virt_timer_list);
 30 static DEFINE_SPINLOCK(virt_timer_lock);
 31 static atomic64_t virt_timer_current;
 32 static atomic64_t virt_timer_elapsed;
 33 
 34 static inline u64 get_vtimer(void)
 35 {
 36         u64 timer;
 37 
 38         asm volatile("stpt %0" : "=m" (timer));
 39         return timer;
 40 }
 41 
 42 static inline void set_vtimer(u64 expires)
 43 {
 44         u64 timer;
 45 
 46         asm volatile(
 47                 "       stpt    %0\n"   /* Store current cpu timer value */
 48                 "       spt     %1"     /* Set new value imm. afterwards */
 49                 : "=m" (timer) : "m" (expires));
 50         S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
 51         S390_lowcore.last_update_timer = expires;
 52 }
 53 
 54 static inline int virt_timer_forward(u64 elapsed)
 55 {
 56         BUG_ON(!irqs_disabled());
 57 
 58         if (list_empty(&virt_timer_list))
 59                 return 0;
 60         elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
 61         return elapsed >= atomic64_read(&virt_timer_current);
 62 }
 63 
 64 /*
 65  * Update process times based on virtual cpu times stored by entry.S
 66  * to the lowcore fields user_timer, system_timer & steal_clock.
 67  */
 68 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset)
 69 {
 70         struct thread_info *ti = task_thread_info(tsk);
 71         u64 timer, clock, user, system, steal;
 72 
 73         timer = S390_lowcore.last_update_timer;
 74         clock = S390_lowcore.last_update_clock;
 75         asm volatile(
 76                 "       stpt    %0\n"   /* Store current cpu timer value */
 77                 "       stck    %1"     /* Store current tod clock value */
 78                 : "=m" (S390_lowcore.last_update_timer),
 79                   "=m" (S390_lowcore.last_update_clock));
 80         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
 81         S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock;
 82 
 83         user = S390_lowcore.user_timer - ti->user_timer;
 84         S390_lowcore.steal_timer -= user;
 85         ti->user_timer = S390_lowcore.user_timer;
 86         account_user_time(tsk, user, user);
 87 
 88         system = S390_lowcore.system_timer - ti->system_timer;
 89         S390_lowcore.steal_timer -= system;
 90         ti->system_timer = S390_lowcore.system_timer;
 91         account_system_time(tsk, hardirq_offset, system, system);
 92 
 93         steal = S390_lowcore.steal_timer;
 94         if ((s64) steal > 0) {
 95                 S390_lowcore.steal_timer = 0;
 96                 account_steal_time(steal);
 97         }
 98 
 99         return virt_timer_forward(user + system);
100 }
101 
102 void vtime_task_switch(struct task_struct *prev)
103 {
104         struct thread_info *ti;
105 
106         do_account_vtime(prev, 0);
107         ti = task_thread_info(prev);
108         ti->user_timer = S390_lowcore.user_timer;
109         ti->system_timer = S390_lowcore.system_timer;
110         ti = task_thread_info(current);
111         S390_lowcore.user_timer = ti->user_timer;
112         S390_lowcore.system_timer = ti->system_timer;
113 }
114 
115 /*
116  * In s390, accounting pending user time also implies
117  * accounting system time in order to correctly compute
118  * the stolen time accounting.
119  */
120 void vtime_account_user(struct task_struct *tsk)
121 {
122         if (do_account_vtime(tsk, HARDIRQ_OFFSET))
123                 virt_timer_expire();
124 }
125 
126 /*
127  * Update process times based on virtual cpu times stored by entry.S
128  * to the lowcore fields user_timer, system_timer & steal_clock.
129  */
130 void vtime_account_irq_enter(struct task_struct *tsk)
131 {
132         struct thread_info *ti = task_thread_info(tsk);
133         u64 timer, system;
134 
135         WARN_ON_ONCE(!irqs_disabled());
136 
137         timer = S390_lowcore.last_update_timer;
138         S390_lowcore.last_update_timer = get_vtimer();
139         S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer;
140 
141         system = S390_lowcore.system_timer - ti->system_timer;
142         S390_lowcore.steal_timer -= system;
143         ti->system_timer = S390_lowcore.system_timer;
144         account_system_time(tsk, 0, system, system);
145 
146         virt_timer_forward(system);
147 }
148 EXPORT_SYMBOL_GPL(vtime_account_irq_enter);
149 
150 void vtime_account_system(struct task_struct *tsk)
151 __attribute__((alias("vtime_account_irq_enter")));
152 EXPORT_SYMBOL_GPL(vtime_account_system);
153 
154 void __kprobes vtime_stop_cpu(void)
155 {
156         struct s390_idle_data *idle = &__get_cpu_var(s390_idle);
157         unsigned long long idle_time;
158         unsigned long psw_mask;
159 
160         trace_hardirqs_on();
161 
162         /* Wait for external, I/O or machine check interrupt. */
163         psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT |
164                 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK;
165         idle->nohz_delay = 0;
166 
167         /* Call the assembler magic in entry.S */
168         psw_idle(idle, psw_mask);
169 
170         /* Account time spent with enabled wait psw loaded as idle time. */
171         idle->sequence++;
172         smp_wmb();
173         idle_time = idle->clock_idle_exit - idle->clock_idle_enter;
174         idle->clock_idle_enter = idle->clock_idle_exit = 0ULL;
175         idle->idle_time += idle_time;
176         idle->idle_count++;
177         account_idle_time(idle_time);
178         smp_wmb();
179         idle->sequence++;
180 }
181 
182 cputime64_t s390_get_idle_time(int cpu)
183 {
184         struct s390_idle_data *idle = &per_cpu(s390_idle, cpu);
185         unsigned long long now, idle_enter, idle_exit;
186         unsigned int sequence;
187 
188         do {
189                 now = get_tod_clock();
190                 sequence = ACCESS_ONCE(idle->sequence);
191                 idle_enter = ACCESS_ONCE(idle->clock_idle_enter);
192                 idle_exit = ACCESS_ONCE(idle->clock_idle_exit);
193         } while ((sequence & 1) || (idle->sequence != sequence));
194         return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0;
195 }
196 
197 /*
198  * Sorted add to a list. List is linear searched until first bigger
199  * element is found.
200  */
201 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
202 {
203         struct vtimer_list *tmp;
204 
205         list_for_each_entry(tmp, head, entry) {
206                 if (tmp->expires > timer->expires) {
207                         list_add_tail(&timer->entry, &tmp->entry);
208                         return;
209                 }
210         }
211         list_add_tail(&timer->entry, head);
212 }
213 
214 /*
215  * Handler for expired virtual CPU timer.
216  */
217 static void virt_timer_expire(void)
218 {
219         struct vtimer_list *timer, *tmp;
220         unsigned long elapsed;
221         LIST_HEAD(cb_list);
222 
223         /* walk timer list, fire all expired timers */
224         spin_lock(&virt_timer_lock);
225         elapsed = atomic64_read(&virt_timer_elapsed);
226         list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
227                 if (timer->expires < elapsed)
228                         /* move expired timer to the callback queue */
229                         list_move_tail(&timer->entry, &cb_list);
230                 else
231                         timer->expires -= elapsed;
232         }
233         if (!list_empty(&virt_timer_list)) {
234                 timer = list_first_entry(&virt_timer_list,
235                                          struct vtimer_list, entry);
236                 atomic64_set(&virt_timer_current, timer->expires);
237         }
238         atomic64_sub(elapsed, &virt_timer_elapsed);
239         spin_unlock(&virt_timer_lock);
240 
241         /* Do callbacks and recharge periodic timers */
242         list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
243                 list_del_init(&timer->entry);
244                 timer->function(timer->data);
245                 if (timer->interval) {
246                         /* Recharge interval timer */
247                         timer->expires = timer->interval +
248                                 atomic64_read(&virt_timer_elapsed);
249                         spin_lock(&virt_timer_lock);
250                         list_add_sorted(timer, &virt_timer_list);
251                         spin_unlock(&virt_timer_lock);
252                 }
253         }
254 }
255 
256 void init_virt_timer(struct vtimer_list *timer)
257 {
258         timer->function = NULL;
259         INIT_LIST_HEAD(&timer->entry);
260 }
261 EXPORT_SYMBOL(init_virt_timer);
262 
263 static inline int vtimer_pending(struct vtimer_list *timer)
264 {
265         return !list_empty(&timer->entry);
266 }
267 
268 static void internal_add_vtimer(struct vtimer_list *timer)
269 {
270         if (list_empty(&virt_timer_list)) {
271                 /* First timer, just program it. */
272                 atomic64_set(&virt_timer_current, timer->expires);
273                 atomic64_set(&virt_timer_elapsed, 0);
274                 list_add(&timer->entry, &virt_timer_list);
275         } else {
276                 /* Update timer against current base. */
277                 timer->expires += atomic64_read(&virt_timer_elapsed);
278                 if (likely((s64) timer->expires <
279                            (s64) atomic64_read(&virt_timer_current)))
280                         /* The new timer expires before the current timer. */
281                         atomic64_set(&virt_timer_current, timer->expires);
282                 /* Insert new timer into the list. */
283                 list_add_sorted(timer, &virt_timer_list);
284         }
285 }
286 
287 static void __add_vtimer(struct vtimer_list *timer, int periodic)
288 {
289         unsigned long flags;
290 
291         timer->interval = periodic ? timer->expires : 0;
292         spin_lock_irqsave(&virt_timer_lock, flags);
293         internal_add_vtimer(timer);
294         spin_unlock_irqrestore(&virt_timer_lock, flags);
295 }
296 
297 /*
298  * add_virt_timer - add an oneshot virtual CPU timer
299  */
300 void add_virt_timer(struct vtimer_list *timer)
301 {
302         __add_vtimer(timer, 0);
303 }
304 EXPORT_SYMBOL(add_virt_timer);
305 
306 /*
307  * add_virt_timer_int - add an interval virtual CPU timer
308  */
309 void add_virt_timer_periodic(struct vtimer_list *timer)
310 {
311         __add_vtimer(timer, 1);
312 }
313 EXPORT_SYMBOL(add_virt_timer_periodic);
314 
315 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
316 {
317         unsigned long flags;
318         int rc;
319 
320         BUG_ON(!timer->function);
321 
322         if (timer->expires == expires && vtimer_pending(timer))
323                 return 1;
324         spin_lock_irqsave(&virt_timer_lock, flags);
325         rc = vtimer_pending(timer);
326         if (rc)
327                 list_del_init(&timer->entry);
328         timer->interval = periodic ? expires : 0;
329         timer->expires = expires;
330         internal_add_vtimer(timer);
331         spin_unlock_irqrestore(&virt_timer_lock, flags);
332         return rc;
333 }
334 
335 /*
336  * returns whether it has modified a pending timer (1) or not (0)
337  */
338 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
339 {
340         return __mod_vtimer(timer, expires, 0);
341 }
342 EXPORT_SYMBOL(mod_virt_timer);
343 
344 /*
345  * returns whether it has modified a pending timer (1) or not (0)
346  */
347 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
348 {
349         return __mod_vtimer(timer, expires, 1);
350 }
351 EXPORT_SYMBOL(mod_virt_timer_periodic);
352 
353 /*
354  * Delete a virtual timer.
355  *
356  * returns whether the deleted timer was pending (1) or not (0)
357  */
358 int del_virt_timer(struct vtimer_list *timer)
359 {
360         unsigned long flags;
361 
362         if (!vtimer_pending(timer))
363                 return 0;
364         spin_lock_irqsave(&virt_timer_lock, flags);
365         list_del_init(&timer->entry);
366         spin_unlock_irqrestore(&virt_timer_lock, flags);
367         return 1;
368 }
369 EXPORT_SYMBOL(del_virt_timer);
370 
371 /*
372  * Start the virtual CPU timer on the current CPU.
373  */
374 void init_cpu_vtimer(void)
375 {
376         /* set initial cpu timer */
377         set_vtimer(VTIMER_MAX_SLICE);
378 }
379 
380 static int s390_nohz_notify(struct notifier_block *self, unsigned long action,
381                             void *hcpu)
382 {
383         struct s390_idle_data *idle;
384         long cpu = (long) hcpu;
385 
386         idle = &per_cpu(s390_idle, cpu);
387         switch (action & ~CPU_TASKS_FROZEN) {
388         case CPU_DYING:
389                 idle->nohz_delay = 0;
390         default:
391                 break;
392         }
393         return NOTIFY_OK;
394 }
395 
396 void __init vtime_init(void)
397 {
398         /* Enable cpu timer interrupts on the boot cpu. */
399         init_cpu_vtimer();
400         cpu_notifier(s390_nohz_notify, 0);
401 }
402 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp