~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/include/asm/tlb.h

Version: ~ [ linux-5.16-rc3 ] ~ [ linux-5.15.5 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.82 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.162 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.218 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.256 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.291 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.293 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  arch/arm/include/asm/tlb.h
  3  *
  4  *  Copyright (C) 2002 Russell King
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  *
 10  *  Experimentation shows that on a StrongARM, it appears to be faster
 11  *  to use the "invalidate whole tlb" rather than "invalidate single
 12  *  tlb" for this.
 13  *
 14  *  This appears true for both the process fork+exit case, as well as
 15  *  the munmap-large-area case.
 16  */
 17 #ifndef __ASMARM_TLB_H
 18 #define __ASMARM_TLB_H
 19 
 20 #include <asm/cacheflush.h>
 21 
 22 #ifndef CONFIG_MMU
 23 
 24 #include <linux/pagemap.h>
 25 
 26 #define tlb_flush(tlb)  ((void) tlb)
 27 
 28 #include <asm-generic/tlb.h>
 29 
 30 #else /* !CONFIG_MMU */
 31 
 32 #include <linux/swap.h>
 33 #include <asm/pgalloc.h>
 34 #include <asm/tlbflush.h>
 35 
 36 #define MMU_GATHER_BUNDLE       8
 37 
 38 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 39 static inline void __tlb_remove_table(void *_table)
 40 {
 41         free_page_and_swap_cache((struct page *)_table);
 42 }
 43 
 44 struct mmu_table_batch {
 45         struct rcu_head         rcu;
 46         unsigned int            nr;
 47         void                    *tables[0];
 48 };
 49 
 50 #define MAX_TABLE_BATCH         \
 51         ((PAGE_SIZE - sizeof(struct mmu_table_batch)) / sizeof(void *))
 52 
 53 extern void tlb_table_flush(struct mmu_gather *tlb);
 54 extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
 55 
 56 #define tlb_remove_entry(tlb, entry)    tlb_remove_table(tlb, entry)
 57 #else
 58 #define tlb_remove_entry(tlb, entry)    tlb_remove_page(tlb, entry)
 59 #endif /* CONFIG_HAVE_RCU_TABLE_FREE */
 60 
 61 /*
 62  * TLB handling.  This allows us to remove pages from the page
 63  * tables, and efficiently handle the TLB issues.
 64  */
 65 struct mmu_gather {
 66         struct mm_struct        *mm;
 67 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
 68         struct mmu_table_batch  *batch;
 69         unsigned int            need_flush;
 70 #endif
 71         unsigned int            fullmm;
 72         struct vm_area_struct   *vma;
 73         unsigned long           start, end;
 74         unsigned long           range_start;
 75         unsigned long           range_end;
 76         unsigned int            nr;
 77         unsigned int            max;
 78         struct page             **pages;
 79         struct page             *local[MMU_GATHER_BUNDLE];
 80 };
 81 
 82 DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
 83 
 84 /*
 85  * This is unnecessarily complex.  There's three ways the TLB shootdown
 86  * code is used:
 87  *  1. Unmapping a range of vmas.  See zap_page_range(), unmap_region().
 88  *     tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
 89  *     tlb->vma will be non-NULL.
 90  *  2. Unmapping all vmas.  See exit_mmap().
 91  *     tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
 92  *     tlb->vma will be non-NULL.  Additionally, page tables will be freed.
 93  *  3. Unmapping argument pages.  See shift_arg_pages().
 94  *     tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
 95  *     tlb->vma will be NULL.
 96  */
 97 static inline void tlb_flush(struct mmu_gather *tlb)
 98 {
 99         if (tlb->fullmm || !tlb->vma)
100                 flush_tlb_mm(tlb->mm);
101         else if (tlb->range_end > 0) {
102                 flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
103                 tlb->range_start = TASK_SIZE;
104                 tlb->range_end = 0;
105         }
106 }
107 
108 static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
109 {
110         if (!tlb->fullmm) {
111                 if (addr < tlb->range_start)
112                         tlb->range_start = addr;
113                 if (addr + PAGE_SIZE > tlb->range_end)
114                         tlb->range_end = addr + PAGE_SIZE;
115         }
116 }
117 
118 static inline void __tlb_alloc_page(struct mmu_gather *tlb)
119 {
120         unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
121 
122         if (addr) {
123                 tlb->pages = (void *)addr;
124                 tlb->max = PAGE_SIZE / sizeof(struct page *);
125         }
126 }
127 
128 static inline void tlb_flush_mmu_tlbonly(struct mmu_gather *tlb)
129 {
130         tlb_flush(tlb);
131 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
132         tlb_table_flush(tlb);
133 #endif
134 }
135 
136 static inline void tlb_flush_mmu_free(struct mmu_gather *tlb)
137 {
138         free_pages_and_swap_cache(tlb->pages, tlb->nr);
139         tlb->nr = 0;
140         if (tlb->pages == tlb->local)
141                 __tlb_alloc_page(tlb);
142 }
143 
144 static inline void tlb_flush_mmu(struct mmu_gather *tlb)
145 {
146         tlb_flush_mmu_tlbonly(tlb);
147         tlb_flush_mmu_free(tlb);
148 }
149 
150 static inline void
151 tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long start, unsigned long end)
152 {
153         tlb->mm = mm;
154         tlb->fullmm = !(start | (end+1));
155         tlb->start = start;
156         tlb->end = end;
157         tlb->vma = NULL;
158         tlb->max = ARRAY_SIZE(tlb->local);
159         tlb->pages = tlb->local;
160         tlb->nr = 0;
161         __tlb_alloc_page(tlb);
162 
163 #ifdef CONFIG_HAVE_RCU_TABLE_FREE
164         tlb->batch = NULL;
165 #endif
166 }
167 
168 static inline void
169 tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
170 {
171         tlb_flush_mmu(tlb);
172 
173         /* keep the page table cache within bounds */
174         check_pgt_cache();
175 
176         if (tlb->pages != tlb->local)
177                 free_pages((unsigned long)tlb->pages, 0);
178 }
179 
180 /*
181  * Memorize the range for the TLB flush.
182  */
183 static inline void
184 tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
185 {
186         tlb_add_flush(tlb, addr);
187 }
188 
189 /*
190  * In the case of tlb vma handling, we can optimise these away in the
191  * case where we're doing a full MM flush.  When we're doing a munmap,
192  * the vmas are adjusted to only cover the region to be torn down.
193  */
194 static inline void
195 tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
196 {
197         if (!tlb->fullmm) {
198                 flush_cache_range(vma, vma->vm_start, vma->vm_end);
199                 tlb->vma = vma;
200                 tlb->range_start = TASK_SIZE;
201                 tlb->range_end = 0;
202         }
203 }
204 
205 static inline void
206 tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
207 {
208         if (!tlb->fullmm)
209                 tlb_flush(tlb);
210 }
211 
212 static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
213 {
214         tlb->pages[tlb->nr++] = page;
215         VM_BUG_ON(tlb->nr > tlb->max);
216         return tlb->max - tlb->nr;
217 }
218 
219 static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
220 {
221         if (!__tlb_remove_page(tlb, page))
222                 tlb_flush_mmu(tlb);
223 }
224 
225 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
226         unsigned long addr)
227 {
228         pgtable_page_dtor(pte);
229 
230 #ifdef CONFIG_ARM_LPAE
231         tlb_add_flush(tlb, addr);
232 #else
233         /*
234          * With the classic ARM MMU, a pte page has two corresponding pmd
235          * entries, each covering 1MB.
236          */
237         addr &= PMD_MASK;
238         tlb_add_flush(tlb, addr + SZ_1M - PAGE_SIZE);
239         tlb_add_flush(tlb, addr + SZ_1M);
240 #endif
241 
242         tlb_remove_entry(tlb, pte);
243 }
244 
245 static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
246                                   unsigned long addr)
247 {
248 #ifdef CONFIG_ARM_LPAE
249         tlb_add_flush(tlb, addr);
250         tlb_remove_entry(tlb, virt_to_page(pmdp));
251 #endif
252 }
253 
254 static inline void
255 tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr)
256 {
257         tlb_add_flush(tlb, addr);
258 }
259 
260 #define pte_free_tlb(tlb, ptep, addr)   __pte_free_tlb(tlb, ptep, addr)
261 #define pmd_free_tlb(tlb, pmdp, addr)   __pmd_free_tlb(tlb, pmdp, addr)
262 #define pud_free_tlb(tlb, pudp, addr)   pud_free((tlb)->mm, pudp)
263 
264 #define tlb_migrate_finish(mm)          do { } while (0)
265 
266 #endif /* CONFIG_MMU */
267 #endif
268 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp