~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/xen/spinlock.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Split spinlock implementation out into its own file, so it can be
  3  * compiled in a FTRACE-compatible way.
  4  */
  5 #include <linux/kernel_stat.h>
  6 #include <linux/spinlock.h>
  7 #include <linux/debugfs.h>
  8 #include <linux/log2.h>
  9 #include <linux/gfp.h>
 10 
 11 #include <asm/paravirt.h>
 12 
 13 #include <xen/interface/xen.h>
 14 #include <xen/events.h>
 15 
 16 #include "xen-ops.h"
 17 #include "debugfs.h"
 18 
 19 #ifdef CONFIG_XEN_DEBUG_FS
 20 static struct xen_spinlock_stats
 21 {
 22         u64 taken;
 23         u32 taken_slow;
 24         u32 taken_slow_nested;
 25         u32 taken_slow_pickup;
 26         u32 taken_slow_spurious;
 27         u32 taken_slow_irqenable;
 28 
 29         u64 released;
 30         u32 released_slow;
 31         u32 released_slow_kicked;
 32 
 33 #define HISTO_BUCKETS   30
 34         u32 histo_spin_total[HISTO_BUCKETS+1];
 35         u32 histo_spin_spinning[HISTO_BUCKETS+1];
 36         u32 histo_spin_blocked[HISTO_BUCKETS+1];
 37 
 38         u64 time_total;
 39         u64 time_spinning;
 40         u64 time_blocked;
 41 } spinlock_stats;
 42 
 43 static u8 zero_stats;
 44 
 45 static unsigned lock_timeout = 1 << 10;
 46 #define TIMEOUT lock_timeout
 47 
 48 static inline void check_zero(void)
 49 {
 50         if (unlikely(zero_stats)) {
 51                 memset(&spinlock_stats, 0, sizeof(spinlock_stats));
 52                 zero_stats = 0;
 53         }
 54 }
 55 
 56 #define ADD_STATS(elem, val)                    \
 57         do { check_zero(); spinlock_stats.elem += (val); } while(0)
 58 
 59 static inline u64 spin_time_start(void)
 60 {
 61         return xen_clocksource_read();
 62 }
 63 
 64 static void __spin_time_accum(u64 delta, u32 *array)
 65 {
 66         unsigned index = ilog2(delta);
 67 
 68         check_zero();
 69 
 70         if (index < HISTO_BUCKETS)
 71                 array[index]++;
 72         else
 73                 array[HISTO_BUCKETS]++;
 74 }
 75 
 76 static inline void spin_time_accum_spinning(u64 start)
 77 {
 78         u32 delta = xen_clocksource_read() - start;
 79 
 80         __spin_time_accum(delta, spinlock_stats.histo_spin_spinning);
 81         spinlock_stats.time_spinning += delta;
 82 }
 83 
 84 static inline void spin_time_accum_total(u64 start)
 85 {
 86         u32 delta = xen_clocksource_read() - start;
 87 
 88         __spin_time_accum(delta, spinlock_stats.histo_spin_total);
 89         spinlock_stats.time_total += delta;
 90 }
 91 
 92 static inline void spin_time_accum_blocked(u64 start)
 93 {
 94         u32 delta = xen_clocksource_read() - start;
 95 
 96         __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
 97         spinlock_stats.time_blocked += delta;
 98 }
 99 #else  /* !CONFIG_XEN_DEBUG_FS */
100 #define TIMEOUT                 (1 << 10)
101 #define ADD_STATS(elem, val)    do { (void)(val); } while(0)
102 
103 static inline u64 spin_time_start(void)
104 {
105         return 0;
106 }
107 
108 static inline void spin_time_accum_total(u64 start)
109 {
110 }
111 static inline void spin_time_accum_spinning(u64 start)
112 {
113 }
114 static inline void spin_time_accum_blocked(u64 start)
115 {
116 }
117 #endif  /* CONFIG_XEN_DEBUG_FS */
118 
119 /*
120  * Size struct xen_spinlock so it's the same as arch_spinlock_t.
121  */
122 #if NR_CPUS < 256
123 typedef u8 xen_spinners_t;
124 # define inc_spinners(xl) \
125         asm(LOCK_PREFIX " incb %0" : "+m" ((xl)->spinners) : : "memory");
126 # define dec_spinners(xl) \
127         asm(LOCK_PREFIX " decb %0" : "+m" ((xl)->spinners) : : "memory");
128 #else
129 typedef u16 xen_spinners_t;
130 # define inc_spinners(xl) \
131         asm(LOCK_PREFIX " incw %0" : "+m" ((xl)->spinners) : : "memory");
132 # define dec_spinners(xl) \
133         asm(LOCK_PREFIX " decw %0" : "+m" ((xl)->spinners) : : "memory");
134 #endif
135 
136 struct xen_spinlock {
137         unsigned char lock;             /* 0 -> free; 1 -> locked */
138         xen_spinners_t spinners;        /* count of waiting cpus */
139 };
140 
141 static int xen_spin_is_locked(struct arch_spinlock *lock)
142 {
143         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
144 
145         return xl->lock != 0;
146 }
147 
148 static int xen_spin_is_contended(struct arch_spinlock *lock)
149 {
150         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
151 
152         /* Not strictly true; this is only the count of contended
153            lock-takers entering the slow path. */
154         return xl->spinners != 0;
155 }
156 
157 static int xen_spin_trylock(struct arch_spinlock *lock)
158 {
159         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
160         u8 old = 1;
161 
162         asm("xchgb %b0,%1"
163             : "+q" (old), "+m" (xl->lock) : : "memory");
164 
165         return old == 0;
166 }
167 
168 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
169 static DEFINE_PER_CPU(struct xen_spinlock *, lock_spinners);
170 
171 /*
172  * Mark a cpu as interested in a lock.  Returns the CPU's previous
173  * lock of interest, in case we got preempted by an interrupt.
174  */
175 static inline struct xen_spinlock *spinning_lock(struct xen_spinlock *xl)
176 {
177         struct xen_spinlock *prev;
178 
179         prev = __this_cpu_read(lock_spinners);
180         __this_cpu_write(lock_spinners, xl);
181 
182         wmb();                  /* set lock of interest before count */
183 
184         inc_spinners(xl);
185 
186         return prev;
187 }
188 
189 /*
190  * Mark a cpu as no longer interested in a lock.  Restores previous
191  * lock of interest (NULL for none).
192  */
193 static inline void unspinning_lock(struct xen_spinlock *xl, struct xen_spinlock *prev)
194 {
195         dec_spinners(xl);
196         wmb();                  /* decrement count before restoring lock */
197         __this_cpu_write(lock_spinners, prev);
198 }
199 
200 static noinline int xen_spin_lock_slow(struct arch_spinlock *lock, bool irq_enable)
201 {
202         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
203         struct xen_spinlock *prev;
204         int irq = __this_cpu_read(lock_kicker_irq);
205         int ret;
206         u64 start;
207 
208         /* If kicker interrupts not initialized yet, just spin */
209         if (irq == -1)
210                 return 0;
211 
212         start = spin_time_start();
213 
214         /* announce we're spinning */
215         prev = spinning_lock(xl);
216 
217         ADD_STATS(taken_slow, 1);
218         ADD_STATS(taken_slow_nested, prev != NULL);
219 
220         do {
221                 unsigned long flags;
222 
223                 /* clear pending */
224                 xen_clear_irq_pending(irq);
225 
226                 /* check again make sure it didn't become free while
227                    we weren't looking  */
228                 ret = xen_spin_trylock(lock);
229                 if (ret) {
230                         ADD_STATS(taken_slow_pickup, 1);
231 
232                         /*
233                          * If we interrupted another spinlock while it
234                          * was blocking, make sure it doesn't block
235                          * without rechecking the lock.
236                          */
237                         if (prev != NULL)
238                                 xen_set_irq_pending(irq);
239                         goto out;
240                 }
241 
242                 flags = arch_local_save_flags();
243                 if (irq_enable) {
244                         ADD_STATS(taken_slow_irqenable, 1);
245                         raw_local_irq_enable();
246                 }
247 
248                 /*
249                  * Block until irq becomes pending.  If we're
250                  * interrupted at this point (after the trylock but
251                  * before entering the block), then the nested lock
252                  * handler guarantees that the irq will be left
253                  * pending if there's any chance the lock became free;
254                  * xen_poll_irq() returns immediately if the irq is
255                  * pending.
256                  */
257                 xen_poll_irq(irq);
258 
259                 raw_local_irq_restore(flags);
260 
261                 ADD_STATS(taken_slow_spurious, !xen_test_irq_pending(irq));
262         } while (!xen_test_irq_pending(irq)); /* check for spurious wakeups */
263 
264         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
265 
266 out:
267         unspinning_lock(xl, prev);
268         spin_time_accum_blocked(start);
269 
270         return ret;
271 }
272 
273 static inline void __xen_spin_lock(struct arch_spinlock *lock, bool irq_enable)
274 {
275         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
276         unsigned timeout;
277         u8 oldval;
278         u64 start_spin;
279 
280         ADD_STATS(taken, 1);
281 
282         start_spin = spin_time_start();
283 
284         do {
285                 u64 start_spin_fast = spin_time_start();
286 
287                 timeout = TIMEOUT;
288 
289                 asm("1: xchgb %1,%0\n"
290                     "   testb %1,%1\n"
291                     "   jz 3f\n"
292                     "2: rep;nop\n"
293                     "   cmpb $0,%0\n"
294                     "   je 1b\n"
295                     "   dec %2\n"
296                     "   jnz 2b\n"
297                     "3:\n"
298                     : "+m" (xl->lock), "=q" (oldval), "+r" (timeout)
299                     : "1" (1)
300                     : "memory");
301 
302                 spin_time_accum_spinning(start_spin_fast);
303 
304         } while (unlikely(oldval != 0 &&
305                           (TIMEOUT == ~0 || !xen_spin_lock_slow(lock, irq_enable))));
306 
307         spin_time_accum_total(start_spin);
308 }
309 
310 static void xen_spin_lock(struct arch_spinlock *lock)
311 {
312         __xen_spin_lock(lock, false);
313 }
314 
315 static void xen_spin_lock_flags(struct arch_spinlock *lock, unsigned long flags)
316 {
317         __xen_spin_lock(lock, !raw_irqs_disabled_flags(flags));
318 }
319 
320 static noinline void xen_spin_unlock_slow(struct xen_spinlock *xl)
321 {
322         int cpu;
323 
324         ADD_STATS(released_slow, 1);
325 
326         for_each_online_cpu(cpu) {
327                 /* XXX should mix up next cpu selection */
328                 if (per_cpu(lock_spinners, cpu) == xl) {
329                         ADD_STATS(released_slow_kicked, 1);
330                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
331                 }
332         }
333 }
334 
335 static void xen_spin_unlock(struct arch_spinlock *lock)
336 {
337         struct xen_spinlock *xl = (struct xen_spinlock *)lock;
338 
339         ADD_STATS(released, 1);
340 
341         smp_wmb();              /* make sure no writes get moved after unlock */
342         xl->lock = 0;           /* release lock */
343 
344         /*
345          * Make sure unlock happens before checking for waiting
346          * spinners.  We need a strong barrier to enforce the
347          * write-read ordering to different memory locations, as the
348          * CPU makes no implied guarantees about their ordering.
349          */
350         mb();
351 
352         if (unlikely(xl->spinners))
353                 xen_spin_unlock_slow(xl);
354 }
355 
356 static irqreturn_t dummy_handler(int irq, void *dev_id)
357 {
358         BUG();
359         return IRQ_HANDLED;
360 }
361 
362 void __cpuinit xen_init_lock_cpu(int cpu)
363 {
364         int irq;
365         const char *name;
366 
367         WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n",
368              cpu, per_cpu(lock_kicker_irq, cpu));
369 
370         /*
371          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
372          * (xen: disable PV spinlocks on HVM)
373          */
374         if (xen_hvm_domain())
375                 return;
376 
377         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
378         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
379                                      cpu,
380                                      dummy_handler,
381                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
382                                      name,
383                                      NULL);
384 
385         if (irq >= 0) {
386                 disable_irq(irq); /* make sure it's never delivered */
387                 per_cpu(lock_kicker_irq, cpu) = irq;
388         }
389 
390         printk("cpu %d spinlock event irq %d\n", cpu, irq);
391 }
392 
393 void xen_uninit_lock_cpu(int cpu)
394 {
395         /*
396          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
397          * (xen: disable PV spinlocks on HVM)
398          */
399         if (xen_hvm_domain())
400                 return;
401 
402         unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
403         per_cpu(lock_kicker_irq, cpu) = -1;
404 }
405 
406 void __init xen_init_spinlocks(void)
407 {
408         /*
409          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
410          * (xen: disable PV spinlocks on HVM)
411          */
412         if (xen_hvm_domain())
413                 return;
414 
415         BUILD_BUG_ON(sizeof(struct xen_spinlock) > sizeof(arch_spinlock_t));
416 
417         pv_lock_ops.spin_is_locked = xen_spin_is_locked;
418         pv_lock_ops.spin_is_contended = xen_spin_is_contended;
419         pv_lock_ops.spin_lock = xen_spin_lock;
420         pv_lock_ops.spin_lock_flags = xen_spin_lock_flags;
421         pv_lock_ops.spin_trylock = xen_spin_trylock;
422         pv_lock_ops.spin_unlock = xen_spin_unlock;
423 }
424 
425 #ifdef CONFIG_XEN_DEBUG_FS
426 
427 static struct dentry *d_spin_debug;
428 
429 static int __init xen_spinlock_debugfs(void)
430 {
431         struct dentry *d_xen = xen_init_debugfs();
432 
433         if (d_xen == NULL)
434                 return -ENOMEM;
435 
436         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
437 
438         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
439 
440         debugfs_create_u32("timeout", 0644, d_spin_debug, &lock_timeout);
441 
442         debugfs_create_u64("taken", 0444, d_spin_debug, &spinlock_stats.taken);
443         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
444                            &spinlock_stats.taken_slow);
445         debugfs_create_u32("taken_slow_nested", 0444, d_spin_debug,
446                            &spinlock_stats.taken_slow_nested);
447         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
448                            &spinlock_stats.taken_slow_pickup);
449         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
450                            &spinlock_stats.taken_slow_spurious);
451         debugfs_create_u32("taken_slow_irqenable", 0444, d_spin_debug,
452                            &spinlock_stats.taken_slow_irqenable);
453 
454         debugfs_create_u64("released", 0444, d_spin_debug, &spinlock_stats.released);
455         debugfs_create_u32("released_slow", 0444, d_spin_debug,
456                            &spinlock_stats.released_slow);
457         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
458                            &spinlock_stats.released_slow_kicked);
459 
460         debugfs_create_u64("time_spinning", 0444, d_spin_debug,
461                            &spinlock_stats.time_spinning);
462         debugfs_create_u64("time_blocked", 0444, d_spin_debug,
463                            &spinlock_stats.time_blocked);
464         debugfs_create_u64("time_total", 0444, d_spin_debug,
465                            &spinlock_stats.time_total);
466 
467         debugfs_create_u32_array("histo_total", 0444, d_spin_debug,
468                                 spinlock_stats.histo_spin_total, HISTO_BUCKETS + 1);
469         debugfs_create_u32_array("histo_spinning", 0444, d_spin_debug,
470                                 spinlock_stats.histo_spin_spinning, HISTO_BUCKETS + 1);
471         debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
472                                 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
473 
474         return 0;
475 }
476 fs_initcall(xen_spinlock_debugfs);
477 
478 #endif  /* CONFIG_XEN_DEBUG_FS */
479 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp