~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/mm/tlb.c

Version: ~ [ linux-5.11-rc3 ] ~ [ linux-5.10.7 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.89 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.167 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.215 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.251 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.251 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #include <linux/init.h>
  2 
  3 #include <linux/mm.h>
  4 #include <linux/spinlock.h>
  5 #include <linux/smp.h>
  6 #include <linux/interrupt.h>
  7 #include <linux/export.h>
  8 #include <linux/cpu.h>
  9 
 10 #include <asm/tlbflush.h>
 11 #include <asm/mmu_context.h>
 12 #include <asm/cache.h>
 13 #include <asm/apic.h>
 14 #include <asm/uv/uv.h>
 15 #include <linux/debugfs.h>
 16 
 17 /*
 18  *      Smarter SMP flushing macros.
 19  *              c/o Linus Torvalds.
 20  *
 21  *      These mean you can really definitely utterly forget about
 22  *      writing to user space from interrupts. (Its not allowed anyway).
 23  *
 24  *      Optimizations Manfred Spraul <manfred@colorfullife.com>
 25  *
 26  *      More scalable flush, from Andi Kleen
 27  *
 28  *      Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
 29  */
 30 
 31 #ifdef CONFIG_SMP
 32 
 33 struct flush_tlb_info {
 34         struct mm_struct *flush_mm;
 35         unsigned long flush_start;
 36         unsigned long flush_end;
 37 };
 38 
 39 /*
 40  * We cannot call mmdrop() because we are in interrupt context,
 41  * instead update mm->cpu_vm_mask.
 42  */
 43 void leave_mm(int cpu)
 44 {
 45         struct mm_struct *active_mm = this_cpu_read(cpu_tlbstate.active_mm);
 46         if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
 47                 BUG();
 48         if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) {
 49                 cpumask_clear_cpu(cpu, mm_cpumask(active_mm));
 50                 load_cr3(swapper_pg_dir);
 51                 /*
 52                  * This gets called in the idle path where RCU
 53                  * functions differently.  Tracing normally
 54                  * uses RCU, so we have to call the tracepoint
 55                  * specially here.
 56                  */
 57                 trace_tlb_flush_rcuidle(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
 58         }
 59 }
 60 EXPORT_SYMBOL_GPL(leave_mm);
 61 
 62 #endif /* CONFIG_SMP */
 63 
 64 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
 65                struct task_struct *tsk)
 66 {
 67         unsigned long flags;
 68 
 69         local_irq_save(flags);
 70         switch_mm_irqs_off(prev, next, tsk);
 71         local_irq_restore(flags);
 72 }
 73 
 74 void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
 75                         struct task_struct *tsk)
 76 {
 77         unsigned cpu = smp_processor_id();
 78 
 79         if (likely(prev != next)) {
 80 #ifdef CONFIG_SMP
 81                 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
 82                 this_cpu_write(cpu_tlbstate.active_mm, next);
 83 #endif
 84                 cpumask_set_cpu(cpu, mm_cpumask(next));
 85 
 86                 /*
 87                  * Re-load page tables.
 88                  *
 89                  * This logic has an ordering constraint:
 90                  *
 91                  *  CPU 0: Write to a PTE for 'next'
 92                  *  CPU 0: load bit 1 in mm_cpumask.  if nonzero, send IPI.
 93                  *  CPU 1: set bit 1 in next's mm_cpumask
 94                  *  CPU 1: load from the PTE that CPU 0 writes (implicit)
 95                  *
 96                  * We need to prevent an outcome in which CPU 1 observes
 97                  * the new PTE value and CPU 0 observes bit 1 clear in
 98                  * mm_cpumask.  (If that occurs, then the IPI will never
 99                  * be sent, and CPU 0's TLB will contain a stale entry.)
100                  *
101                  * The bad outcome can occur if either CPU's load is
102                  * reordered before that CPU's store, so both CPUs must
103                  * execute full barriers to prevent this from happening.
104                  *
105                  * Thus, switch_mm needs a full barrier between the
106                  * store to mm_cpumask and any operation that could load
107                  * from next->pgd.  TLB fills are special and can happen
108                  * due to instruction fetches or for no reason at all,
109                  * and neither LOCK nor MFENCE orders them.
110                  * Fortunately, load_cr3() is serializing and gives the
111                  * ordering guarantee we need.
112                  *
113                  */
114                 load_cr3(next->pgd);
115 
116                 trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
117 
118                 /* Stop flush ipis for the previous mm */
119                 cpumask_clear_cpu(cpu, mm_cpumask(prev));
120 
121                 /* Load per-mm CR4 state */
122                 load_mm_cr4(next);
123 
124 #ifdef CONFIG_MODIFY_LDT_SYSCALL
125                 /*
126                  * Load the LDT, if the LDT is different.
127                  *
128                  * It's possible that prev->context.ldt doesn't match
129                  * the LDT register.  This can happen if leave_mm(prev)
130                  * was called and then modify_ldt changed
131                  * prev->context.ldt but suppressed an IPI to this CPU.
132                  * In this case, prev->context.ldt != NULL, because we
133                  * never set context.ldt to NULL while the mm still
134                  * exists.  That means that next->context.ldt !=
135                  * prev->context.ldt, because mms never share an LDT.
136                  */
137                 if (unlikely(prev->context.ldt != next->context.ldt))
138                         load_mm_ldt(next);
139 #endif
140         }
141 #ifdef CONFIG_SMP
142           else {
143                 this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
144                 BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
145 
146                 if (!cpumask_test_cpu(cpu, mm_cpumask(next))) {
147                         /*
148                          * On established mms, the mm_cpumask is only changed
149                          * from irq context, from ptep_clear_flush() while in
150                          * lazy tlb mode, and here. Irqs are blocked during
151                          * schedule, protecting us from simultaneous changes.
152                          */
153                         cpumask_set_cpu(cpu, mm_cpumask(next));
154 
155                         /*
156                          * We were in lazy tlb mode and leave_mm disabled
157                          * tlb flush IPI delivery. We must reload CR3
158                          * to make sure to use no freed page tables.
159                          *
160                          * As above, load_cr3() is serializing and orders TLB
161                          * fills with respect to the mm_cpumask write.
162                          */
163                         load_cr3(next->pgd);
164                         trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL);
165                         load_mm_cr4(next);
166                         load_mm_ldt(next);
167                 }
168         }
169 #endif
170 }
171 
172 #ifdef CONFIG_SMP
173 
174 /*
175  * The flush IPI assumes that a thread switch happens in this order:
176  * [cpu0: the cpu that switches]
177  * 1) switch_mm() either 1a) or 1b)
178  * 1a) thread switch to a different mm
179  * 1a1) set cpu_tlbstate to TLBSTATE_OK
180  *      Now the tlb flush NMI handler flush_tlb_func won't call leave_mm
181  *      if cpu0 was in lazy tlb mode.
182  * 1a2) update cpu active_mm
183  *      Now cpu0 accepts tlb flushes for the new mm.
184  * 1a3) cpu_set(cpu, new_mm->cpu_vm_mask);
185  *      Now the other cpus will send tlb flush ipis.
186  * 1a4) change cr3.
187  * 1a5) cpu_clear(cpu, old_mm->cpu_vm_mask);
188  *      Stop ipi delivery for the old mm. This is not synchronized with
189  *      the other cpus, but flush_tlb_func ignore flush ipis for the wrong
190  *      mm, and in the worst case we perform a superfluous tlb flush.
191  * 1b) thread switch without mm change
192  *      cpu active_mm is correct, cpu0 already handles flush ipis.
193  * 1b1) set cpu_tlbstate to TLBSTATE_OK
194  * 1b2) test_and_set the cpu bit in cpu_vm_mask.
195  *      Atomically set the bit [other cpus will start sending flush ipis],
196  *      and test the bit.
197  * 1b3) if the bit was 0: leave_mm was called, flush the tlb.
198  * 2) switch %%esp, ie current
199  *
200  * The interrupt must handle 2 special cases:
201  * - cr3 is changed before %%esp, ie. it cannot use current->{active_,}mm.
202  * - the cpu performs speculative tlb reads, i.e. even if the cpu only
203  *   runs in kernel space, the cpu could load tlb entries for user space
204  *   pages.
205  *
206  * The good news is that cpu_tlbstate is local to each cpu, no
207  * write/read ordering problems.
208  */
209 
210 /*
211  * TLB flush funcation:
212  * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
213  * 2) Leave the mm if we are in the lazy tlb mode.
214  */
215 static void flush_tlb_func(void *info)
216 {
217         struct flush_tlb_info *f = info;
218 
219         inc_irq_stat(irq_tlb_count);
220 
221         if (f->flush_mm && f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
222                 return;
223 
224         count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
225         if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
226                 if (f->flush_end == TLB_FLUSH_ALL) {
227                         local_flush_tlb();
228                         trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, TLB_FLUSH_ALL);
229                 } else {
230                         unsigned long addr;
231                         unsigned long nr_pages =
232                                 (f->flush_end - f->flush_start) / PAGE_SIZE;
233                         addr = f->flush_start;
234                         while (addr < f->flush_end) {
235                                 __flush_tlb_single(addr);
236                                 addr += PAGE_SIZE;
237                         }
238                         trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, nr_pages);
239                 }
240         } else
241                 leave_mm(smp_processor_id());
242 
243 }
244 
245 void native_flush_tlb_others(const struct cpumask *cpumask,
246                                  struct mm_struct *mm, unsigned long start,
247                                  unsigned long end)
248 {
249         struct flush_tlb_info info;
250 
251         if (end == 0)
252                 end = start + PAGE_SIZE;
253         info.flush_mm = mm;
254         info.flush_start = start;
255         info.flush_end = end;
256 
257         count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
258         if (end == TLB_FLUSH_ALL)
259                 trace_tlb_flush(TLB_REMOTE_SEND_IPI, TLB_FLUSH_ALL);
260         else
261                 trace_tlb_flush(TLB_REMOTE_SEND_IPI,
262                                 (end - start) >> PAGE_SHIFT);
263 
264         if (is_uv_system()) {
265                 unsigned int cpu;
266 
267                 cpu = smp_processor_id();
268                 cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
269                 if (cpumask)
270                         smp_call_function_many(cpumask, flush_tlb_func,
271                                                                 &info, 1);
272                 return;
273         }
274         smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
275 }
276 
277 void flush_tlb_current_task(void)
278 {
279         struct mm_struct *mm = current->mm;
280 
281         preempt_disable();
282 
283         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
284 
285         /* This is an implicit full barrier that synchronizes with switch_mm. */
286         local_flush_tlb();
287 
288         trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
289         if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
290                 flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
291         preempt_enable();
292 }
293 
294 /*
295  * See Documentation/x86/tlb.txt for details.  We choose 33
296  * because it is large enough to cover the vast majority (at
297  * least 95%) of allocations, and is small enough that we are
298  * confident it will not cause too much overhead.  Each single
299  * flush is about 100 ns, so this caps the maximum overhead at
300  * _about_ 3,000 ns.
301  *
302  * This is in units of pages.
303  */
304 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
305 
306 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
307                                 unsigned long end, unsigned long vmflag)
308 {
309         unsigned long addr;
310         /* do a global flush by default */
311         unsigned long base_pages_to_flush = TLB_FLUSH_ALL;
312 
313         preempt_disable();
314         if (current->active_mm != mm) {
315                 /* Synchronize with switch_mm. */
316                 smp_mb();
317 
318                 goto out;
319         }
320 
321         if (!current->mm) {
322                 leave_mm(smp_processor_id());
323 
324                 /* Synchronize with switch_mm. */
325                 smp_mb();
326 
327                 goto out;
328         }
329 
330         if ((end != TLB_FLUSH_ALL) && !(vmflag & VM_HUGETLB))
331                 base_pages_to_flush = (end - start) >> PAGE_SHIFT;
332 
333         /*
334          * Both branches below are implicit full barriers (MOV to CR or
335          * INVLPG) that synchronize with switch_mm.
336          */
337         if (base_pages_to_flush > tlb_single_page_flush_ceiling) {
338                 base_pages_to_flush = TLB_FLUSH_ALL;
339                 count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
340                 local_flush_tlb();
341         } else {
342                 /* flush range by one by one 'invlpg' */
343                 for (addr = start; addr < end;  addr += PAGE_SIZE) {
344                         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
345                         __flush_tlb_single(addr);
346                 }
347         }
348         trace_tlb_flush(TLB_LOCAL_MM_SHOOTDOWN, base_pages_to_flush);
349 out:
350         if (base_pages_to_flush == TLB_FLUSH_ALL) {
351                 start = 0UL;
352                 end = TLB_FLUSH_ALL;
353         }
354         if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
355                 flush_tlb_others(mm_cpumask(mm), mm, start, end);
356         preempt_enable();
357 }
358 
359 void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
360 {
361         struct mm_struct *mm = vma->vm_mm;
362 
363         preempt_disable();
364 
365         if (current->active_mm == mm) {
366                 if (current->mm) {
367                         /*
368                          * Implicit full barrier (INVLPG) that synchronizes
369                          * with switch_mm.
370                          */
371                         __flush_tlb_one(start);
372                 } else {
373                         leave_mm(smp_processor_id());
374 
375                         /* Synchronize with switch_mm. */
376                         smp_mb();
377                 }
378         }
379 
380         if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
381                 flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
382 
383         preempt_enable();
384 }
385 
386 static void do_flush_tlb_all(void *info)
387 {
388         count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
389         __flush_tlb_all();
390         if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
391                 leave_mm(smp_processor_id());
392 }
393 
394 void flush_tlb_all(void)
395 {
396         count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
397         on_each_cpu(do_flush_tlb_all, NULL, 1);
398 }
399 
400 static void do_kernel_range_flush(void *info)
401 {
402         struct flush_tlb_info *f = info;
403         unsigned long addr;
404 
405         /* flush range by one by one 'invlpg' */
406         for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
407                 __flush_tlb_single(addr);
408 }
409 
410 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
411 {
412 
413         /* Balance as user space task's flush, a bit conservative */
414         if (end == TLB_FLUSH_ALL ||
415             (end - start) > tlb_single_page_flush_ceiling * PAGE_SIZE) {
416                 on_each_cpu(do_flush_tlb_all, NULL, 1);
417         } else {
418                 struct flush_tlb_info info;
419                 info.flush_start = start;
420                 info.flush_end = end;
421                 on_each_cpu(do_kernel_range_flush, &info, 1);
422         }
423 }
424 
425 static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
426                              size_t count, loff_t *ppos)
427 {
428         char buf[32];
429         unsigned int len;
430 
431         len = sprintf(buf, "%ld\n", tlb_single_page_flush_ceiling);
432         return simple_read_from_buffer(user_buf, count, ppos, buf, len);
433 }
434 
435 static ssize_t tlbflush_write_file(struct file *file,
436                  const char __user *user_buf, size_t count, loff_t *ppos)
437 {
438         char buf[32];
439         ssize_t len;
440         int ceiling;
441 
442         len = min(count, sizeof(buf) - 1);
443         if (copy_from_user(buf, user_buf, len))
444                 return -EFAULT;
445 
446         buf[len] = '\0';
447         if (kstrtoint(buf, 0, &ceiling))
448                 return -EINVAL;
449 
450         if (ceiling < 0)
451                 return -EINVAL;
452 
453         tlb_single_page_flush_ceiling = ceiling;
454         return count;
455 }
456 
457 static const struct file_operations fops_tlbflush = {
458         .read = tlbflush_read_file,
459         .write = tlbflush_write_file,
460         .llseek = default_llseek,
461 };
462 
463 static int __init create_tlb_single_page_flush_ceiling(void)
464 {
465         debugfs_create_file("tlb_single_page_flush_ceiling", S_IRUSR | S_IWUSR,
466                             arch_debugfs_dir, NULL, &fops_tlbflush);
467         return 0;
468 }
469 late_initcall(create_tlb_single_page_flush_ceiling);
470 
471 #endif /* CONFIG_SMP */
472 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp