~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/mm/tlb-radix.c

Version: ~ [ linux-5.15-rc1 ] ~ [ linux-5.14.5 ] ~ [ linux-5.13.18 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.66 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.147 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.206 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.246 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.282 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.283 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * TLB flush routines for radix kernels.
  3  *
  4  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
  5  *
  6  * This program is free software; you can redistribute it and/or
  7  * modify it under the terms of the GNU General Public License
  8  * as published by the Free Software Foundation; either version
  9  * 2 of the License, or (at your option) any later version.
 10  */
 11 
 12 #include <linux/mm.h>
 13 #include <linux/hugetlb.h>
 14 #include <linux/memblock.h>
 15 #include <asm/ppc-opcode.h>
 16 
 17 #include <asm/tlb.h>
 18 #include <asm/tlbflush.h>
 19 
 20 static DEFINE_RAW_SPINLOCK(native_tlbie_lock);
 21 
 22 #define RIC_FLUSH_TLB 0
 23 #define RIC_FLUSH_PWC 1
 24 #define RIC_FLUSH_ALL 2
 25 
 26 static inline void __tlbiel_pid(unsigned long pid, int set,
 27                                 unsigned long ric)
 28 {
 29         unsigned long rb,rs,prs,r;
 30 
 31         rb = PPC_BIT(53); /* IS = 1 */
 32         rb |= set << PPC_BITLSHIFT(51);
 33         rs = ((unsigned long)pid) << PPC_BITLSHIFT(31);
 34         prs = 1; /* process scoped */
 35         r = 1;   /* raidx format */
 36 
 37         asm volatile("ptesync": : :"memory");
 38         asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
 39                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
 40         asm volatile("ptesync": : :"memory");
 41 }
 42 
 43 /*
 44  * We use 128 set in radix mode and 256 set in hpt mode.
 45  */
 46 static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 47 {
 48         int set;
 49 
 50         for (set = 0; set < POWER9_TLB_SETS_RADIX ; set++) {
 51                 __tlbiel_pid(pid, set, ric);
 52         }
 53         return;
 54 }
 55 
 56 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 57 {
 58         unsigned long rb,rs,prs,r;
 59 
 60         rb = PPC_BIT(53); /* IS = 1 */
 61         rs = pid << PPC_BITLSHIFT(31);
 62         prs = 1; /* process scoped */
 63         r = 1;   /* raidx format */
 64 
 65         asm volatile("ptesync": : :"memory");
 66         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
 67                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
 68         asm volatile("eieio; tlbsync; ptesync": : :"memory");
 69 }
 70 
 71 static inline void _tlbiel_va(unsigned long va, unsigned long pid,
 72                               unsigned long ap, unsigned long ric)
 73 {
 74         unsigned long rb,rs,prs,r;
 75 
 76         rb = va & ~(PPC_BITMASK(52, 63));
 77         rb |= ap << PPC_BITLSHIFT(58);
 78         rs = pid << PPC_BITLSHIFT(31);
 79         prs = 1; /* process scoped */
 80         r = 1;   /* raidx format */
 81 
 82         asm volatile("ptesync": : :"memory");
 83         asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
 84                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
 85         asm volatile("ptesync": : :"memory");
 86 }
 87 
 88 static inline void _tlbie_va(unsigned long va, unsigned long pid,
 89                              unsigned long ap, unsigned long ric)
 90 {
 91         unsigned long rb,rs,prs,r;
 92 
 93         rb = va & ~(PPC_BITMASK(52, 63));
 94         rb |= ap << PPC_BITLSHIFT(58);
 95         rs = pid << PPC_BITLSHIFT(31);
 96         prs = 1; /* process scoped */
 97         r = 1;   /* raidx format */
 98 
 99         asm volatile("ptesync": : :"memory");
100         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
101                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
102         asm volatile("eieio; tlbsync; ptesync": : :"memory");
103 }
104 
105 /*
106  * Base TLB flushing operations:
107  *
108  *  - flush_tlb_mm(mm) flushes the specified mm context TLB's
109  *  - flush_tlb_page(vma, vmaddr) flushes one page
110  *  - flush_tlb_range(vma, start, end) flushes a range of pages
111  *  - flush_tlb_kernel_range(start, end) flushes kernel pages
112  *
113  *  - local_* variants of page and mm only apply to the current
114  *    processor
115  */
116 void radix__local_flush_tlb_mm(struct mm_struct *mm)
117 {
118         unsigned long pid;
119 
120         preempt_disable();
121         pid = mm->context.id;
122         if (pid != MMU_NO_CONTEXT)
123                 _tlbiel_pid(pid, RIC_FLUSH_ALL);
124         preempt_enable();
125 }
126 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
127 
128 void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
129 {
130         unsigned long pid;
131         struct mm_struct *mm = tlb->mm;
132 
133         preempt_disable();
134 
135         pid = mm->context.id;
136         if (pid != MMU_NO_CONTEXT)
137                 _tlbiel_pid(pid, RIC_FLUSH_PWC);
138 
139         preempt_enable();
140 }
141 EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
142 
143 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
144                                        int psize)
145 {
146         unsigned long pid;
147         unsigned long ap = mmu_get_ap(psize);
148 
149         preempt_disable();
150         pid = mm ? mm->context.id : 0;
151         if (pid != MMU_NO_CONTEXT)
152                 _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
153         preempt_enable();
154 }
155 
156 void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
157 {
158 #ifdef CONFIG_HUGETLB_PAGE
159         /* need the return fix for nohash.c */
160         if (vma && is_vm_hugetlb_page(vma))
161                 return __local_flush_hugetlb_page(vma, vmaddr);
162 #endif
163         radix__local_flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
164                                           mmu_virtual_psize);
165 }
166 EXPORT_SYMBOL(radix__local_flush_tlb_page);
167 
168 #ifdef CONFIG_SMP
169 void radix__flush_tlb_mm(struct mm_struct *mm)
170 {
171         unsigned long pid;
172 
173         preempt_disable();
174         pid = mm->context.id;
175         if (unlikely(pid == MMU_NO_CONTEXT))
176                 goto no_context;
177 
178         if (!mm_is_thread_local(mm)) {
179                 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
180 
181                 if (lock_tlbie)
182                         raw_spin_lock(&native_tlbie_lock);
183                 _tlbie_pid(pid, RIC_FLUSH_ALL);
184                 if (lock_tlbie)
185                         raw_spin_unlock(&native_tlbie_lock);
186         } else
187                 _tlbiel_pid(pid, RIC_FLUSH_ALL);
188 no_context:
189         preempt_enable();
190 }
191 EXPORT_SYMBOL(radix__flush_tlb_mm);
192 
193 void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
194 {
195         unsigned long pid;
196         struct mm_struct *mm = tlb->mm;
197 
198         preempt_disable();
199 
200         pid = mm->context.id;
201         if (unlikely(pid == MMU_NO_CONTEXT))
202                 goto no_context;
203 
204         if (!mm_is_thread_local(mm)) {
205                 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
206 
207                 if (lock_tlbie)
208                         raw_spin_lock(&native_tlbie_lock);
209                 _tlbie_pid(pid, RIC_FLUSH_PWC);
210                 if (lock_tlbie)
211                         raw_spin_unlock(&native_tlbie_lock);
212         } else
213                 _tlbiel_pid(pid, RIC_FLUSH_PWC);
214 no_context:
215         preempt_enable();
216 }
217 EXPORT_SYMBOL(radix__flush_tlb_pwc);
218 
219 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
220                                  int psize)
221 {
222         unsigned long pid;
223         unsigned long ap = mmu_get_ap(psize);
224 
225         preempt_disable();
226         pid = mm ? mm->context.id : 0;
227         if (unlikely(pid == MMU_NO_CONTEXT))
228                 goto bail;
229         if (!mm_is_thread_local(mm)) {
230                 int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
231 
232                 if (lock_tlbie)
233                         raw_spin_lock(&native_tlbie_lock);
234                 _tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
235                 if (lock_tlbie)
236                         raw_spin_unlock(&native_tlbie_lock);
237         } else
238                 _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
239 bail:
240         preempt_enable();
241 }
242 
243 void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
244 {
245 #ifdef CONFIG_HUGETLB_PAGE
246         if (vma && is_vm_hugetlb_page(vma))
247                 return flush_hugetlb_page(vma, vmaddr);
248 #endif
249         radix__flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
250                                     mmu_virtual_psize);
251 }
252 EXPORT_SYMBOL(radix__flush_tlb_page);
253 
254 #endif /* CONFIG_SMP */
255 
256 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
257 {
258         int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
259 
260         if (lock_tlbie)
261                 raw_spin_lock(&native_tlbie_lock);
262         _tlbie_pid(0, RIC_FLUSH_ALL);
263         if (lock_tlbie)
264                 raw_spin_unlock(&native_tlbie_lock);
265 }
266 EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
267 
268 /*
269  * Currently, for range flushing, we just do a full mm flush. Because
270  * we use this in code path where we don' track the page size.
271  */
272 void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
273                      unsigned long end)
274 
275 {
276         struct mm_struct *mm = vma->vm_mm;
277         radix__flush_tlb_mm(mm);
278 }
279 EXPORT_SYMBOL(radix__flush_tlb_range);
280 
281 static int radix_get_mmu_psize(int page_size)
282 {
283         int psize;
284 
285         if (page_size == (1UL << mmu_psize_defs[mmu_virtual_psize].shift))
286                 psize = mmu_virtual_psize;
287         else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_2M].shift))
288                 psize = MMU_PAGE_2M;
289         else if (page_size == (1UL << mmu_psize_defs[MMU_PAGE_1G].shift))
290                 psize = MMU_PAGE_1G;
291         else
292                 return -1;
293         return psize;
294 }
295 
296 void radix__tlb_flush(struct mmu_gather *tlb)
297 {
298         int psize = 0;
299         struct mm_struct *mm = tlb->mm;
300         int page_size = tlb->page_size;
301 
302         psize = radix_get_mmu_psize(page_size);
303         /*
304          * if page size is not something we understand, do a full mm flush
305          */
306         if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
307                 radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
308         else
309                 radix__flush_tlb_mm(mm);
310 }
311 
312 #define TLB_FLUSH_ALL -1UL
313 /*
314  * Number of pages above which we will do a bcast tlbie. Just a
315  * number at this point copied from x86
316  */
317 static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
318 
319 void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
320                                   unsigned long end, int psize)
321 {
322         unsigned long pid;
323         unsigned long addr;
324         int local = mm_is_thread_local(mm);
325         unsigned long ap = mmu_get_ap(psize);
326         int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
327         unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
328 
329 
330         preempt_disable();
331         pid = mm ? mm->context.id : 0;
332         if (unlikely(pid == MMU_NO_CONTEXT))
333                 goto err_out;
334 
335         if (end == TLB_FLUSH_ALL ||
336             (end - start) > tlb_single_page_flush_ceiling * page_size) {
337                 if (local)
338                         _tlbiel_pid(pid, RIC_FLUSH_TLB);
339                 else
340                         _tlbie_pid(pid, RIC_FLUSH_TLB);
341                 goto err_out;
342         }
343         for (addr = start; addr < end; addr += page_size) {
344 
345                 if (local)
346                         _tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
347                 else {
348                         if (lock_tlbie)
349                                 raw_spin_lock(&native_tlbie_lock);
350                         _tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
351                         if (lock_tlbie)
352                                 raw_spin_unlock(&native_tlbie_lock);
353                 }
354         }
355 err_out:
356         preempt_enable();
357 }
358 
359 void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
360                               unsigned long page_size)
361 {
362         unsigned long rb,rs,prs,r;
363         unsigned long ap;
364         unsigned long ric = RIC_FLUSH_TLB;
365 
366         ap = mmu_get_ap(radix_get_mmu_psize(page_size));
367         rb = gpa & ~(PPC_BITMASK(52, 63));
368         rb |= ap << PPC_BITLSHIFT(58);
369         rs = lpid & ((1UL << 32) - 1);
370         prs = 0; /* process scoped */
371         r = 1;   /* raidx format */
372 
373         asm volatile("ptesync": : :"memory");
374         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
375                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
376         asm volatile("eieio; tlbsync; ptesync": : :"memory");
377 }
378 EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
379 
380 void radix__flush_tlb_lpid(unsigned long lpid)
381 {
382         unsigned long rb,rs,prs,r;
383         unsigned long ric = RIC_FLUSH_ALL;
384 
385         rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
386         rs = lpid & ((1UL << 32) - 1);
387         prs = 0; /* partition scoped */
388         r = 1;   /* raidx format */
389 
390         asm volatile("ptesync": : :"memory");
391         asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
392                      : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
393         asm volatile("eieio; tlbsync; ptesync": : :"memory");
394 }
395 EXPORT_SYMBOL(radix__flush_tlb_lpid);
396 
397 void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
398                                 unsigned long start, unsigned long end)
399 {
400         radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
401 }
402 EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
403 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp