~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm/mm/flush.c

Version: ~ [ linux-5.19-rc3 ] ~ [ linux-5.18.5 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.48 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.123 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.199 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.248 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.284 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.319 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  *  linux/arch/arm/mm/flush.c
  4  *
  5  *  Copyright (C) 1995-2002 Russell King
  6  */
  7 #include <linux/module.h>
  8 #include <linux/mm.h>
  9 #include <linux/pagemap.h>
 10 #include <linux/highmem.h>
 11 
 12 #include <asm/cacheflush.h>
 13 #include <asm/cachetype.h>
 14 #include <asm/highmem.h>
 15 #include <asm/smp_plat.h>
 16 #include <asm/tlbflush.h>
 17 #include <linux/hugetlb.h>
 18 
 19 #include "mm.h"
 20 
 21 #ifdef CONFIG_ARM_HEAVY_MB
 22 void (*soc_mb)(void);
 23 
 24 void arm_heavy_mb(void)
 25 {
 26 #ifdef CONFIG_OUTER_CACHE_SYNC
 27         if (outer_cache.sync)
 28                 outer_cache.sync();
 29 #endif
 30         if (soc_mb)
 31                 soc_mb();
 32 }
 33 EXPORT_SYMBOL(arm_heavy_mb);
 34 #endif
 35 
 36 #ifdef CONFIG_CPU_CACHE_VIPT
 37 
 38 static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
 39 {
 40         unsigned long to = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
 41         const int zero = 0;
 42 
 43         set_top_pte(to, pfn_pte(pfn, PAGE_KERNEL));
 44 
 45         asm(    "mcrr   p15, 0, %1, %0, c14\n"
 46         "       mcr     p15, 0, %2, c7, c10, 4"
 47             :
 48             : "r" (to), "r" (to + PAGE_SIZE - 1), "r" (zero)
 49             : "cc");
 50 }
 51 
 52 static void flush_icache_alias(unsigned long pfn, unsigned long vaddr, unsigned long len)
 53 {
 54         unsigned long va = FLUSH_ALIAS_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
 55         unsigned long offset = vaddr & (PAGE_SIZE - 1);
 56         unsigned long to;
 57 
 58         set_top_pte(va, pfn_pte(pfn, PAGE_KERNEL));
 59         to = va + offset;
 60         flush_icache_range(to, to + len);
 61 }
 62 
 63 void flush_cache_mm(struct mm_struct *mm)
 64 {
 65         if (cache_is_vivt()) {
 66                 vivt_flush_cache_mm(mm);
 67                 return;
 68         }
 69 
 70         if (cache_is_vipt_aliasing()) {
 71                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
 72                 "       mcr     p15, 0, %0, c7, c10, 4"
 73                     :
 74                     : "r" (0)
 75                     : "cc");
 76         }
 77 }
 78 
 79 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 80 {
 81         if (cache_is_vivt()) {
 82                 vivt_flush_cache_range(vma, start, end);
 83                 return;
 84         }
 85 
 86         if (cache_is_vipt_aliasing()) {
 87                 asm(    "mcr    p15, 0, %0, c7, c14, 0\n"
 88                 "       mcr     p15, 0, %0, c7, c10, 4"
 89                     :
 90                     : "r" (0)
 91                     : "cc");
 92         }
 93 
 94         if (vma->vm_flags & VM_EXEC)
 95                 __flush_icache_all();
 96 }
 97 
 98 void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn)
 99 {
100         if (cache_is_vivt()) {
101                 vivt_flush_cache_page(vma, user_addr, pfn);
102                 return;
103         }
104 
105         if (cache_is_vipt_aliasing()) {
106                 flush_pfn_alias(pfn, user_addr);
107                 __flush_icache_all();
108         }
109 
110         if (vma->vm_flags & VM_EXEC && icache_is_vivt_asid_tagged())
111                 __flush_icache_all();
112 }
113 
114 #else
115 #define flush_pfn_alias(pfn,vaddr)              do { } while (0)
116 #define flush_icache_alias(pfn,vaddr,len)       do { } while (0)
117 #endif
118 
119 #define FLAG_PA_IS_EXEC 1
120 #define FLAG_PA_CORE_IN_MM 2
121 
122 static void flush_ptrace_access_other(void *args)
123 {
124         __flush_icache_all();
125 }
126 
127 static inline
128 void __flush_ptrace_access(struct page *page, unsigned long uaddr, void *kaddr,
129                            unsigned long len, unsigned int flags)
130 {
131         if (cache_is_vivt()) {
132                 if (flags & FLAG_PA_CORE_IN_MM) {
133                         unsigned long addr = (unsigned long)kaddr;
134                         __cpuc_coherent_kern_range(addr, addr + len);
135                 }
136                 return;
137         }
138 
139         if (cache_is_vipt_aliasing()) {
140                 flush_pfn_alias(page_to_pfn(page), uaddr);
141                 __flush_icache_all();
142                 return;
143         }
144 
145         /* VIPT non-aliasing D-cache */
146         if (flags & FLAG_PA_IS_EXEC) {
147                 unsigned long addr = (unsigned long)kaddr;
148                 if (icache_is_vipt_aliasing())
149                         flush_icache_alias(page_to_pfn(page), uaddr, len);
150                 else
151                         __cpuc_coherent_kern_range(addr, addr + len);
152                 if (cache_ops_need_broadcast())
153                         smp_call_function(flush_ptrace_access_other,
154                                           NULL, 1);
155         }
156 }
157 
158 static
159 void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
160                          unsigned long uaddr, void *kaddr, unsigned long len)
161 {
162         unsigned int flags = 0;
163         if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm)))
164                 flags |= FLAG_PA_CORE_IN_MM;
165         if (vma->vm_flags & VM_EXEC)
166                 flags |= FLAG_PA_IS_EXEC;
167         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
168 }
169 
170 void flush_uprobe_xol_access(struct page *page, unsigned long uaddr,
171                              void *kaddr, unsigned long len)
172 {
173         unsigned int flags = FLAG_PA_CORE_IN_MM|FLAG_PA_IS_EXEC;
174 
175         __flush_ptrace_access(page, uaddr, kaddr, len, flags);
176 }
177 
178 /*
179  * Copy user data from/to a page which is mapped into a different
180  * processes address space.  Really, we want to allow our "user
181  * space" model to handle this.
182  *
183  * Note that this code needs to run on the current CPU.
184  */
185 void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
186                        unsigned long uaddr, void *dst, const void *src,
187                        unsigned long len)
188 {
189 #ifdef CONFIG_SMP
190         preempt_disable();
191 #endif
192         memcpy(dst, src, len);
193         flush_ptrace_access(vma, page, uaddr, dst, len);
194 #ifdef CONFIG_SMP
195         preempt_enable();
196 #endif
197 }
198 
199 void __flush_dcache_page(struct address_space *mapping, struct page *page)
200 {
201         /*
202          * Writeback any data associated with the kernel mapping of this
203          * page.  This ensures that data in the physical page is mutually
204          * coherent with the kernels mapping.
205          */
206         if (!PageHighMem(page)) {
207                 __cpuc_flush_dcache_area(page_address(page), page_size(page));
208         } else {
209                 unsigned long i;
210                 if (cache_is_vipt_nonaliasing()) {
211                         for (i = 0; i < compound_nr(page); i++) {
212                                 void *addr = kmap_atomic(page + i);
213                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
214                                 kunmap_atomic(addr);
215                         }
216                 } else {
217                         for (i = 0; i < compound_nr(page); i++) {
218                                 void *addr = kmap_high_get(page + i);
219                                 if (addr) {
220                                         __cpuc_flush_dcache_area(addr, PAGE_SIZE);
221                                         kunmap_high(page + i);
222                                 }
223                         }
224                 }
225         }
226 
227         /*
228          * If this is a page cache page, and we have an aliasing VIPT cache,
229          * we only need to do one flush - which would be at the relevant
230          * userspace colour, which is congruent with page->index.
231          */
232         if (mapping && cache_is_vipt_aliasing())
233                 flush_pfn_alias(page_to_pfn(page),
234                                 page->index << PAGE_SHIFT);
235 }
236 
237 static void __flush_dcache_aliases(struct address_space *mapping, struct page *page)
238 {
239         struct mm_struct *mm = current->active_mm;
240         struct vm_area_struct *mpnt;
241         pgoff_t pgoff;
242 
243         /*
244          * There are possible user space mappings of this page:
245          * - VIVT cache: we need to also write back and invalidate all user
246          *   data in the current VM view associated with this page.
247          * - aliasing VIPT: we only need to find one mapping of this page.
248          */
249         pgoff = page->index;
250 
251         flush_dcache_mmap_lock(mapping);
252         vma_interval_tree_foreach(mpnt, &mapping->i_mmap, pgoff, pgoff) {
253                 unsigned long offset;
254 
255                 /*
256                  * If this VMA is not in our MM, we can ignore it.
257                  */
258                 if (mpnt->vm_mm != mm)
259                         continue;
260                 if (!(mpnt->vm_flags & VM_MAYSHARE))
261                         continue;
262                 offset = (pgoff - mpnt->vm_pgoff) << PAGE_SHIFT;
263                 flush_cache_page(mpnt, mpnt->vm_start + offset, page_to_pfn(page));
264         }
265         flush_dcache_mmap_unlock(mapping);
266 }
267 
268 #if __LINUX_ARM_ARCH__ >= 6
269 void __sync_icache_dcache(pte_t pteval)
270 {
271         unsigned long pfn;
272         struct page *page;
273         struct address_space *mapping;
274 
275         if (cache_is_vipt_nonaliasing() && !pte_exec(pteval))
276                 /* only flush non-aliasing VIPT caches for exec mappings */
277                 return;
278         pfn = pte_pfn(pteval);
279         if (!pfn_valid(pfn))
280                 return;
281 
282         page = pfn_to_page(pfn);
283         if (cache_is_vipt_aliasing())
284                 mapping = page_mapping_file(page);
285         else
286                 mapping = NULL;
287 
288         if (!test_and_set_bit(PG_dcache_clean, &page->flags))
289                 __flush_dcache_page(mapping, page);
290 
291         if (pte_exec(pteval))
292                 __flush_icache_all();
293 }
294 #endif
295 
296 /*
297  * Ensure cache coherency between kernel mapping and userspace mapping
298  * of this page.
299  *
300  * We have three cases to consider:
301  *  - VIPT non-aliasing cache: fully coherent so nothing required.
302  *  - VIVT: fully aliasing, so we need to handle every alias in our
303  *          current VM view.
304  *  - VIPT aliasing: need to handle one alias in our current VM view.
305  *
306  * If we need to handle aliasing:
307  *  If the page only exists in the page cache and there are no user
308  *  space mappings, we can be lazy and remember that we may have dirty
309  *  kernel cache lines for later.  Otherwise, we assume we have
310  *  aliasing mappings.
311  *
312  * Note that we disable the lazy flush for SMP configurations where
313  * the cache maintenance operations are not automatically broadcasted.
314  */
315 void flush_dcache_page(struct page *page)
316 {
317         struct address_space *mapping;
318 
319         /*
320          * The zero page is never written to, so never has any dirty
321          * cache lines, and therefore never needs to be flushed.
322          */
323         if (page == ZERO_PAGE(0))
324                 return;
325 
326         if (!cache_ops_need_broadcast() && cache_is_vipt_nonaliasing()) {
327                 if (test_bit(PG_dcache_clean, &page->flags))
328                         clear_bit(PG_dcache_clean, &page->flags);
329                 return;
330         }
331 
332         mapping = page_mapping_file(page);
333 
334         if (!cache_ops_need_broadcast() &&
335             mapping && !page_mapcount(page))
336                 clear_bit(PG_dcache_clean, &page->flags);
337         else {
338                 __flush_dcache_page(mapping, page);
339                 if (mapping && cache_is_vivt())
340                         __flush_dcache_aliases(mapping, page);
341                 else if (mapping)
342                         __flush_icache_all();
343                 set_bit(PG_dcache_clean, &page->flags);
344         }
345 }
346 EXPORT_SYMBOL(flush_dcache_page);
347 
348 /*
349  * Ensure cache coherency for the kernel mapping of this page. We can
350  * assume that the page is pinned via kmap.
351  *
352  * If the page only exists in the page cache and there are no user
353  * space mappings, this is a no-op since the page was already marked
354  * dirty at creation.  Otherwise, we need to flush the dirty kernel
355  * cache lines directly.
356  */
357 void flush_kernel_dcache_page(struct page *page)
358 {
359         if (cache_is_vivt() || cache_is_vipt_aliasing()) {
360                 struct address_space *mapping;
361 
362                 mapping = page_mapping_file(page);
363 
364                 if (!mapping || mapping_mapped(mapping)) {
365                         void *addr;
366 
367                         addr = page_address(page);
368                         /*
369                          * kmap_atomic() doesn't set the page virtual
370                          * address for highmem pages, and
371                          * kunmap_atomic() takes care of cache
372                          * flushing already.
373                          */
374                         if (!IS_ENABLED(CONFIG_HIGHMEM) || addr)
375                                 __cpuc_flush_dcache_area(addr, PAGE_SIZE);
376                 }
377         }
378 }
379 EXPORT_SYMBOL(flush_kernel_dcache_page);
380 
381 /*
382  * Flush an anonymous page so that users of get_user_pages()
383  * can safely access the data.  The expected sequence is:
384  *
385  *  get_user_pages()
386  *    -> flush_anon_page
387  *  memcpy() to/from page
388  *  if written to page, flush_dcache_page()
389  */
390 void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr)
391 {
392         unsigned long pfn;
393 
394         /* VIPT non-aliasing caches need do nothing */
395         if (cache_is_vipt_nonaliasing())
396                 return;
397 
398         /*
399          * Write back and invalidate userspace mapping.
400          */
401         pfn = page_to_pfn(page);
402         if (cache_is_vivt()) {
403                 flush_cache_page(vma, vmaddr, pfn);
404         } else {
405                 /*
406                  * For aliasing VIPT, we can flush an alias of the
407                  * userspace address only.
408                  */
409                 flush_pfn_alias(pfn, vmaddr);
410                 __flush_icache_all();
411         }
412 
413         /*
414          * Invalidate kernel mapping.  No data should be contained
415          * in this mapping of the page.  FIXME: this is overkill
416          * since we actually ask for a write-back and invalidate.
417          */
418         __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE);
419 }
420 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp