~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/include/asm/tlbflush.h

Version: ~ [ linux-5.14-rc3 ] ~ [ linux-5.13.5 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.53 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.135 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.198 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.240 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.276 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.276 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #ifndef _ASM_X86_TLBFLUSH_H
  2 #define _ASM_X86_TLBFLUSH_H
  3 
  4 #include <linux/mm.h>
  5 #include <linux/sched.h>
  6 
  7 #include <asm/processor.h>
  8 #include <asm/special_insns.h>
  9 
 10 #ifdef CONFIG_PARAVIRT
 11 #include <asm/paravirt.h>
 12 #else
 13 #define __flush_tlb() __native_flush_tlb()
 14 #define __flush_tlb_global() __native_flush_tlb_global()
 15 #define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
 16 #endif
 17 
 18 struct tlb_state {
 19 #ifdef CONFIG_SMP
 20         struct mm_struct *active_mm;
 21         int state;
 22 #endif
 23 
 24         /*
 25          * Access to this CR4 shadow and to H/W CR4 is protected by
 26          * disabling interrupts when modifying either one.
 27          */
 28         unsigned long cr4;
 29 };
 30 DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
 31 
 32 /* Initialize cr4 shadow for this CPU. */
 33 static inline void cr4_init_shadow(void)
 34 {
 35         this_cpu_write(cpu_tlbstate.cr4, __read_cr4_safe());
 36 }
 37 
 38 /* Set in this cpu's CR4. */
 39 static inline void cr4_set_bits(unsigned long mask)
 40 {
 41         unsigned long cr4;
 42 
 43         cr4 = this_cpu_read(cpu_tlbstate.cr4);
 44         if ((cr4 | mask) != cr4) {
 45                 cr4 |= mask;
 46                 this_cpu_write(cpu_tlbstate.cr4, cr4);
 47                 __write_cr4(cr4);
 48         }
 49 }
 50 
 51 /* Clear in this cpu's CR4. */
 52 static inline void cr4_clear_bits(unsigned long mask)
 53 {
 54         unsigned long cr4;
 55 
 56         cr4 = this_cpu_read(cpu_tlbstate.cr4);
 57         if ((cr4 & ~mask) != cr4) {
 58                 cr4 &= ~mask;
 59                 this_cpu_write(cpu_tlbstate.cr4, cr4);
 60                 __write_cr4(cr4);
 61         }
 62 }
 63 
 64 /* Read the CR4 shadow. */
 65 static inline unsigned long cr4_read_shadow(void)
 66 {
 67         return this_cpu_read(cpu_tlbstate.cr4);
 68 }
 69 
 70 /*
 71  * Save some of cr4 feature set we're using (e.g.  Pentium 4MB
 72  * enable and PPro Global page enable), so that any CPU's that boot
 73  * up after us can get the correct flags.  This should only be used
 74  * during boot on the boot cpu.
 75  */
 76 extern unsigned long mmu_cr4_features;
 77 extern u32 *trampoline_cr4_features;
 78 
 79 static inline void cr4_set_bits_and_update_boot(unsigned long mask)
 80 {
 81         mmu_cr4_features |= mask;
 82         if (trampoline_cr4_features)
 83                 *trampoline_cr4_features = mmu_cr4_features;
 84         cr4_set_bits(mask);
 85 }
 86 
 87 static inline void __native_flush_tlb(void)
 88 {
 89         /*
 90          * If current->mm == NULL then we borrow a mm which may change during a
 91          * task switch and therefore we must not be preempted while we write CR3
 92          * back:
 93          */
 94         preempt_disable();
 95         native_write_cr3(native_read_cr3());
 96         preempt_enable();
 97 }
 98 
 99 static inline void __native_flush_tlb_global_irq_disabled(void)
100 {
101         unsigned long cr4;
102 
103         cr4 = this_cpu_read(cpu_tlbstate.cr4);
104         /* clear PGE */
105         native_write_cr4(cr4 & ~X86_CR4_PGE);
106         /* write old PGE again and flush TLBs */
107         native_write_cr4(cr4);
108 }
109 
110 static inline void __native_flush_tlb_global(void)
111 {
112         unsigned long flags;
113 
114         /*
115          * Read-modify-write to CR4 - protect it from preemption and
116          * from interrupts. (Use the raw variant because this code can
117          * be called from deep inside debugging code.)
118          */
119         raw_local_irq_save(flags);
120 
121         __native_flush_tlb_global_irq_disabled();
122 
123         raw_local_irq_restore(flags);
124 }
125 
126 static inline void __native_flush_tlb_single(unsigned long addr)
127 {
128         asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
129 }
130 
131 static inline void __flush_tlb_all(void)
132 {
133         if (cpu_has_pge)
134                 __flush_tlb_global();
135         else
136                 __flush_tlb();
137 }
138 
139 static inline void __flush_tlb_one(unsigned long addr)
140 {
141         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
142         __flush_tlb_single(addr);
143 }
144 
145 #define TLB_FLUSH_ALL   -1UL
146 
147 /*
148  * TLB flushing:
149  *
150  *  - flush_tlb() flushes the current mm struct TLBs
151  *  - flush_tlb_all() flushes all processes TLBs
152  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
153  *  - flush_tlb_page(vma, vmaddr) flushes one page
154  *  - flush_tlb_range(vma, start, end) flushes a range of pages
155  *  - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
156  *  - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
157  *
158  * ..but the i386 has somewhat limited tlb flushing capabilities,
159  * and page-granular flushes are available only on i486 and up.
160  */
161 
162 #ifndef CONFIG_SMP
163 
164 /* "_up" is for UniProcessor.
165  *
166  * This is a helper for other header functions.  *Not* intended to be called
167  * directly.  All global TLB flushes need to either call this, or to bump the
168  * vm statistics themselves.
169  */
170 static inline void __flush_tlb_up(void)
171 {
172         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
173         __flush_tlb();
174 }
175 
176 static inline void flush_tlb_all(void)
177 {
178         count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
179         __flush_tlb_all();
180 }
181 
182 static inline void flush_tlb(void)
183 {
184         __flush_tlb_up();
185 }
186 
187 static inline void local_flush_tlb(void)
188 {
189         __flush_tlb_up();
190 }
191 
192 static inline void flush_tlb_mm(struct mm_struct *mm)
193 {
194         if (mm == current->active_mm)
195                 __flush_tlb_up();
196 }
197 
198 static inline void flush_tlb_page(struct vm_area_struct *vma,
199                                   unsigned long addr)
200 {
201         if (vma->vm_mm == current->active_mm)
202                 __flush_tlb_one(addr);
203 }
204 
205 static inline void flush_tlb_range(struct vm_area_struct *vma,
206                                    unsigned long start, unsigned long end)
207 {
208         if (vma->vm_mm == current->active_mm)
209                 __flush_tlb_up();
210 }
211 
212 static inline void flush_tlb_mm_range(struct mm_struct *mm,
213            unsigned long start, unsigned long end, unsigned long vmflag)
214 {
215         if (mm == current->active_mm)
216                 __flush_tlb_up();
217 }
218 
219 static inline void native_flush_tlb_others(const struct cpumask *cpumask,
220                                            struct mm_struct *mm,
221                                            unsigned long start,
222                                            unsigned long end)
223 {
224 }
225 
226 static inline void reset_lazy_tlbstate(void)
227 {
228 }
229 
230 static inline void flush_tlb_kernel_range(unsigned long start,
231                                           unsigned long end)
232 {
233         flush_tlb_all();
234 }
235 
236 #else  /* SMP */
237 
238 #include <asm/smp.h>
239 
240 #define local_flush_tlb() __flush_tlb()
241 
242 #define flush_tlb_mm(mm)        flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
243 
244 #define flush_tlb_range(vma, start, end)        \
245                 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
246 
247 extern void flush_tlb_all(void);
248 extern void flush_tlb_current_task(void);
249 extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
250 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
251                                 unsigned long end, unsigned long vmflag);
252 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
253 
254 #define flush_tlb()     flush_tlb_current_task()
255 
256 void native_flush_tlb_others(const struct cpumask *cpumask,
257                                 struct mm_struct *mm,
258                                 unsigned long start, unsigned long end);
259 
260 #define TLBSTATE_OK     1
261 #define TLBSTATE_LAZY   2
262 
263 static inline void reset_lazy_tlbstate(void)
264 {
265         this_cpu_write(cpu_tlbstate.state, 0);
266         this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
267 }
268 
269 #endif  /* SMP */
270 
271 #ifndef CONFIG_PARAVIRT
272 #define flush_tlb_others(mask, mm, start, end)  \
273         native_flush_tlb_others(mask, mm, start, end)
274 #endif
275 
276 #endif /* _ASM_X86_TLBFLUSH_H */
277 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp