~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/sh/mm/gup.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Lockless get_user_pages_fast for SuperH
  3  *
  4  * Copyright (C) 2009 - 2010  Paul Mundt
  5  *
  6  * Cloned from the x86 and PowerPC versions, by:
  7  *
  8  *      Copyright (C) 2008 Nick Piggin
  9  *      Copyright (C) 2008 Novell Inc.
 10  */
 11 #include <linux/sched.h>
 12 #include <linux/mm.h>
 13 #include <linux/vmstat.h>
 14 #include <linux/highmem.h>
 15 #include <asm/pgtable.h>
 16 
 17 static inline pte_t gup_get_pte(pte_t *ptep)
 18 {
 19 #ifndef CONFIG_X2TLB
 20         return READ_ONCE(*ptep);
 21 #else
 22         /*
 23          * With get_user_pages_fast, we walk down the pagetables without
 24          * taking any locks.  For this we would like to load the pointers
 25          * atomically, but that is not possible with 64-bit PTEs.  What
 26          * we do have is the guarantee that a pte will only either go
 27          * from not present to present, or present to not present or both
 28          * -- it will not switch to a completely different present page
 29          * without a TLB flush in between; something that we are blocking
 30          * by holding interrupts off.
 31          *
 32          * Setting ptes from not present to present goes:
 33          * ptep->pte_high = h;
 34          * smp_wmb();
 35          * ptep->pte_low = l;
 36          *
 37          * And present to not present goes:
 38          * ptep->pte_low = 0;
 39          * smp_wmb();
 40          * ptep->pte_high = 0;
 41          *
 42          * We must ensure here that the load of pte_low sees l iff pte_high
 43          * sees h. We load pte_high *after* loading pte_low, which ensures we
 44          * don't see an older value of pte_high.  *Then* we recheck pte_low,
 45          * which ensures that we haven't picked up a changed pte high. We might
 46          * have got rubbish values from pte_low and pte_high, but we are
 47          * guaranteed that pte_low will not have the present bit set *unless*
 48          * it is 'l'. And get_user_pages_fast only operates on present ptes, so
 49          * we're safe.
 50          *
 51          * gup_get_pte should not be used or copied outside gup.c without being
 52          * very careful -- it does not atomically load the pte or anything that
 53          * is likely to be useful for you.
 54          */
 55         pte_t pte;
 56 
 57 retry:
 58         pte.pte_low = ptep->pte_low;
 59         smp_rmb();
 60         pte.pte_high = ptep->pte_high;
 61         smp_rmb();
 62         if (unlikely(pte.pte_low != ptep->pte_low))
 63                 goto retry;
 64 
 65         return pte;
 66 #endif
 67 }
 68 
 69 /*
 70  * The performance critical leaf functions are made noinline otherwise gcc
 71  * inlines everything into a single function which results in too much
 72  * register pressure.
 73  */
 74 static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
 75                 unsigned long end, int write, struct page **pages, int *nr)
 76 {
 77         u64 mask, result;
 78         pte_t *ptep;
 79 
 80 #ifdef CONFIG_X2TLB
 81         result = _PAGE_PRESENT | _PAGE_EXT(_PAGE_EXT_KERN_READ | _PAGE_EXT_USER_READ);
 82         if (write)
 83                 result |= _PAGE_EXT(_PAGE_EXT_KERN_WRITE | _PAGE_EXT_USER_WRITE);
 84 #elif defined(CONFIG_SUPERH64)
 85         result = _PAGE_PRESENT | _PAGE_USER | _PAGE_READ;
 86         if (write)
 87                 result |= _PAGE_WRITE;
 88 #else
 89         result = _PAGE_PRESENT | _PAGE_USER;
 90         if (write)
 91                 result |= _PAGE_RW;
 92 #endif
 93 
 94         mask = result | _PAGE_SPECIAL;
 95 
 96         ptep = pte_offset_map(&pmd, addr);
 97         do {
 98                 pte_t pte = gup_get_pte(ptep);
 99                 struct page *page;
100 
101                 if ((pte_val(pte) & mask) != result) {
102                         pte_unmap(ptep);
103                         return 0;
104                 }
105                 VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
106                 page = pte_page(pte);
107                 get_page(page);
108                 __flush_anon_page(page, addr);
109                 flush_dcache_page(page);
110                 pages[*nr] = page;
111                 (*nr)++;
112 
113         } while (ptep++, addr += PAGE_SIZE, addr != end);
114         pte_unmap(ptep - 1);
115 
116         return 1;
117 }
118 
119 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
120                 int write, struct page **pages, int *nr)
121 {
122         unsigned long next;
123         pmd_t *pmdp;
124 
125         pmdp = pmd_offset(&pud, addr);
126         do {
127                 pmd_t pmd = *pmdp;
128 
129                 next = pmd_addr_end(addr, end);
130                 if (pmd_none(pmd))
131                         return 0;
132                 if (!gup_pte_range(pmd, addr, next, write, pages, nr))
133                         return 0;
134         } while (pmdp++, addr = next, addr != end);
135 
136         return 1;
137 }
138 
139 static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
140                         int write, struct page **pages, int *nr)
141 {
142         unsigned long next;
143         pud_t *pudp;
144 
145         pudp = pud_offset(&pgd, addr);
146         do {
147                 pud_t pud = *pudp;
148 
149                 next = pud_addr_end(addr, end);
150                 if (pud_none(pud))
151                         return 0;
152                 if (!gup_pmd_range(pud, addr, next, write, pages, nr))
153                         return 0;
154         } while (pudp++, addr = next, addr != end);
155 
156         return 1;
157 }
158 
159 /*
160  * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
161  * back to the regular GUP.
162  */
163 int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
164                           struct page **pages)
165 {
166         struct mm_struct *mm = current->mm;
167         unsigned long addr, len, end;
168         unsigned long next;
169         unsigned long flags;
170         pgd_t *pgdp;
171         int nr = 0;
172 
173         start &= PAGE_MASK;
174         addr = start;
175         len = (unsigned long) nr_pages << PAGE_SHIFT;
176         end = start + len;
177         if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
178                                         (void __user *)start, len)))
179                 return 0;
180 
181         /*
182          * This doesn't prevent pagetable teardown, but does prevent
183          * the pagetables and pages from being freed.
184          */
185         local_irq_save(flags);
186         pgdp = pgd_offset(mm, addr);
187         do {
188                 pgd_t pgd = *pgdp;
189 
190                 next = pgd_addr_end(addr, end);
191                 if (pgd_none(pgd))
192                         break;
193                 if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
194                         break;
195         } while (pgdp++, addr = next, addr != end);
196         local_irq_restore(flags);
197 
198         return nr;
199 }
200 
201 /**
202  * get_user_pages_fast() - pin user pages in memory
203  * @start:      starting user address
204  * @nr_pages:   number of pages from start to pin
205  * @write:      whether pages will be written to
206  * @pages:      array that receives pointers to the pages pinned.
207  *              Should be at least nr_pages long.
208  *
209  * Attempt to pin user pages in memory without taking mm->mmap_sem.
210  * If not successful, it will fall back to taking the lock and
211  * calling get_user_pages().
212  *
213  * Returns number of pages pinned. This may be fewer than the number
214  * requested. If nr_pages is 0 or negative, returns 0. If no pages
215  * were pinned, returns -errno.
216  */
217 int get_user_pages_fast(unsigned long start, int nr_pages, int write,
218                         struct page **pages)
219 {
220         struct mm_struct *mm = current->mm;
221         unsigned long addr, len, end;
222         unsigned long next;
223         pgd_t *pgdp;
224         int nr = 0;
225 
226         start &= PAGE_MASK;
227         addr = start;
228         len = (unsigned long) nr_pages << PAGE_SHIFT;
229 
230         end = start + len;
231         if (end < start)
232                 goto slow_irqon;
233 
234         local_irq_disable();
235         pgdp = pgd_offset(mm, addr);
236         do {
237                 pgd_t pgd = *pgdp;
238 
239                 next = pgd_addr_end(addr, end);
240                 if (pgd_none(pgd))
241                         goto slow;
242                 if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
243                         goto slow;
244         } while (pgdp++, addr = next, addr != end);
245         local_irq_enable();
246 
247         VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
248         return nr;
249 
250         {
251                 int ret;
252 
253 slow:
254                 local_irq_enable();
255 slow_irqon:
256                 /* Try to get the remaining pages with get_user_pages */
257                 start += nr << PAGE_SHIFT;
258                 pages += nr;
259 
260                 down_read(&mm->mmap_sem);
261                 ret = get_user_pages(current, mm, start,
262                         (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
263                 up_read(&mm->mmap_sem);
264 
265                 /* Have to be a bit careful with return values */
266                 if (nr > 0) {
267                         if (ret < 0)
268                                 ret = nr;
269                         else
270                                 ret += nr;
271                 }
272 
273                 return ret;
274         }
275 }
276 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp