~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/mm/pgtable.c

Version: ~ [ linux-5.5-rc7 ] ~ [ linux-5.4.13 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.97 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.166 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.210 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.210 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.81 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *    Copyright IBM Corp. 2007, 2011
  4  *    Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
  5  */
  6 
  7 #include <linux/sched.h>
  8 #include <linux/kernel.h>
  9 #include <linux/errno.h>
 10 #include <linux/gfp.h>
 11 #include <linux/mm.h>
 12 #include <linux/swap.h>
 13 #include <linux/smp.h>
 14 #include <linux/spinlock.h>
 15 #include <linux/rcupdate.h>
 16 #include <linux/slab.h>
 17 #include <linux/swapops.h>
 18 #include <linux/sysctl.h>
 19 #include <linux/ksm.h>
 20 #include <linux/mman.h>
 21 
 22 #include <asm/pgtable.h>
 23 #include <asm/pgalloc.h>
 24 #include <asm/tlb.h>
 25 #include <asm/tlbflush.h>
 26 #include <asm/mmu_context.h>
 27 #include <asm/page-states.h>
 28 
 29 static inline void ptep_ipte_local(struct mm_struct *mm, unsigned long addr,
 30                                    pte_t *ptep, int nodat)
 31 {
 32         unsigned long opt, asce;
 33 
 34         if (MACHINE_HAS_TLB_GUEST) {
 35                 opt = 0;
 36                 asce = READ_ONCE(mm->context.gmap_asce);
 37                 if (asce == 0UL || nodat)
 38                         opt |= IPTE_NODAT;
 39                 if (asce != -1UL) {
 40                         asce = asce ? : mm->context.asce;
 41                         opt |= IPTE_GUEST_ASCE;
 42                 }
 43                 __ptep_ipte(addr, ptep, opt, asce, IPTE_LOCAL);
 44         } else {
 45                 __ptep_ipte(addr, ptep, 0, 0, IPTE_LOCAL);
 46         }
 47 }
 48 
 49 static inline void ptep_ipte_global(struct mm_struct *mm, unsigned long addr,
 50                                     pte_t *ptep, int nodat)
 51 {
 52         unsigned long opt, asce;
 53 
 54         if (MACHINE_HAS_TLB_GUEST) {
 55                 opt = 0;
 56                 asce = READ_ONCE(mm->context.gmap_asce);
 57                 if (asce == 0UL || nodat)
 58                         opt |= IPTE_NODAT;
 59                 if (asce != -1UL) {
 60                         asce = asce ? : mm->context.asce;
 61                         opt |= IPTE_GUEST_ASCE;
 62                 }
 63                 __ptep_ipte(addr, ptep, opt, asce, IPTE_GLOBAL);
 64         } else {
 65                 __ptep_ipte(addr, ptep, 0, 0, IPTE_GLOBAL);
 66         }
 67 }
 68 
 69 static inline pte_t ptep_flush_direct(struct mm_struct *mm,
 70                                       unsigned long addr, pte_t *ptep,
 71                                       int nodat)
 72 {
 73         pte_t old;
 74 
 75         old = *ptep;
 76         if (unlikely(pte_val(old) & _PAGE_INVALID))
 77                 return old;
 78         atomic_inc(&mm->context.flush_count);
 79         if (MACHINE_HAS_TLB_LC &&
 80             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
 81                 ptep_ipte_local(mm, addr, ptep, nodat);
 82         else
 83                 ptep_ipte_global(mm, addr, ptep, nodat);
 84         atomic_dec(&mm->context.flush_count);
 85         return old;
 86 }
 87 
 88 static inline pte_t ptep_flush_lazy(struct mm_struct *mm,
 89                                     unsigned long addr, pte_t *ptep,
 90                                     int nodat)
 91 {
 92         pte_t old;
 93 
 94         old = *ptep;
 95         if (unlikely(pte_val(old) & _PAGE_INVALID))
 96                 return old;
 97         atomic_inc(&mm->context.flush_count);
 98         if (cpumask_equal(&mm->context.cpu_attach_mask,
 99                           cpumask_of(smp_processor_id()))) {
100                 pte_val(*ptep) |= _PAGE_INVALID;
101                 mm->context.flush_mm = 1;
102         } else
103                 ptep_ipte_global(mm, addr, ptep, nodat);
104         atomic_dec(&mm->context.flush_count);
105         return old;
106 }
107 
108 static inline pgste_t pgste_get_lock(pte_t *ptep)
109 {
110         unsigned long new = 0;
111 #ifdef CONFIG_PGSTE
112         unsigned long old;
113 
114         asm(
115                 "       lg      %0,%2\n"
116                 "0:     lgr     %1,%0\n"
117                 "       nihh    %0,0xff7f\n"    /* clear PCL bit in old */
118                 "       oihh    %1,0x0080\n"    /* set PCL bit in new */
119                 "       csg     %0,%1,%2\n"
120                 "       jl      0b\n"
121                 : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE])
122                 : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory");
123 #endif
124         return __pgste(new);
125 }
126 
127 static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste)
128 {
129 #ifdef CONFIG_PGSTE
130         asm(
131                 "       nihh    %1,0xff7f\n"    /* clear PCL bit */
132                 "       stg     %1,%0\n"
133                 : "=Q" (ptep[PTRS_PER_PTE])
134                 : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE])
135                 : "cc", "memory");
136 #endif
137 }
138 
139 static inline pgste_t pgste_get(pte_t *ptep)
140 {
141         unsigned long pgste = 0;
142 #ifdef CONFIG_PGSTE
143         pgste = *(unsigned long *)(ptep + PTRS_PER_PTE);
144 #endif
145         return __pgste(pgste);
146 }
147 
148 static inline void pgste_set(pte_t *ptep, pgste_t pgste)
149 {
150 #ifdef CONFIG_PGSTE
151         *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste;
152 #endif
153 }
154 
155 static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste,
156                                        struct mm_struct *mm)
157 {
158 #ifdef CONFIG_PGSTE
159         unsigned long address, bits, skey;
160 
161         if (!mm_uses_skeys(mm) || pte_val(pte) & _PAGE_INVALID)
162                 return pgste;
163         address = pte_val(pte) & PAGE_MASK;
164         skey = (unsigned long) page_get_storage_key(address);
165         bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
166         /* Transfer page changed & referenced bit to guest bits in pgste */
167         pgste_val(pgste) |= bits << 48;         /* GR bit & GC bit */
168         /* Copy page access key and fetch protection bit to pgste */
169         pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
170         pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
171 #endif
172         return pgste;
173 
174 }
175 
176 static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry,
177                                  struct mm_struct *mm)
178 {
179 #ifdef CONFIG_PGSTE
180         unsigned long address;
181         unsigned long nkey;
182 
183         if (!mm_uses_skeys(mm) || pte_val(entry) & _PAGE_INVALID)
184                 return;
185         VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID));
186         address = pte_val(entry) & PAGE_MASK;
187         /*
188          * Set page access key and fetch protection bit from pgste.
189          * The guest C/R information is still in the PGSTE, set real
190          * key C/R to 0.
191          */
192         nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
193         nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
194         page_set_storage_key(address, nkey, 0);
195 #endif
196 }
197 
198 static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry)
199 {
200 #ifdef CONFIG_PGSTE
201         if ((pte_val(entry) & _PAGE_PRESENT) &&
202             (pte_val(entry) & _PAGE_WRITE) &&
203             !(pte_val(entry) & _PAGE_INVALID)) {
204                 if (!MACHINE_HAS_ESOP) {
205                         /*
206                          * Without enhanced suppression-on-protection force
207                          * the dirty bit on for all writable ptes.
208                          */
209                         pte_val(entry) |= _PAGE_DIRTY;
210                         pte_val(entry) &= ~_PAGE_PROTECT;
211                 }
212                 if (!(pte_val(entry) & _PAGE_PROTECT))
213                         /* This pte allows write access, set user-dirty */
214                         pgste_val(pgste) |= PGSTE_UC_BIT;
215         }
216 #endif
217         *ptep = entry;
218         return pgste;
219 }
220 
221 static inline pgste_t pgste_pte_notify(struct mm_struct *mm,
222                                        unsigned long addr,
223                                        pte_t *ptep, pgste_t pgste)
224 {
225 #ifdef CONFIG_PGSTE
226         unsigned long bits;
227 
228         bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT);
229         if (bits) {
230                 pgste_val(pgste) ^= bits;
231                 ptep_notify(mm, addr, ptep, bits);
232         }
233 #endif
234         return pgste;
235 }
236 
237 static inline pgste_t ptep_xchg_start(struct mm_struct *mm,
238                                       unsigned long addr, pte_t *ptep)
239 {
240         pgste_t pgste = __pgste(0);
241 
242         if (mm_has_pgste(mm)) {
243                 pgste = pgste_get_lock(ptep);
244                 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
245         }
246         return pgste;
247 }
248 
249 static inline pte_t ptep_xchg_commit(struct mm_struct *mm,
250                                     unsigned long addr, pte_t *ptep,
251                                     pgste_t pgste, pte_t old, pte_t new)
252 {
253         if (mm_has_pgste(mm)) {
254                 if (pte_val(old) & _PAGE_INVALID)
255                         pgste_set_key(ptep, pgste, new, mm);
256                 if (pte_val(new) & _PAGE_INVALID) {
257                         pgste = pgste_update_all(old, pgste, mm);
258                         if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) ==
259                             _PGSTE_GPS_USAGE_UNUSED)
260                                 pte_val(old) |= _PAGE_UNUSED;
261                 }
262                 pgste = pgste_set_pte(ptep, pgste, new);
263                 pgste_set_unlock(ptep, pgste);
264         } else {
265                 *ptep = new;
266         }
267         return old;
268 }
269 
270 pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr,
271                        pte_t *ptep, pte_t new)
272 {
273         pgste_t pgste;
274         pte_t old;
275         int nodat;
276 
277         preempt_disable();
278         pgste = ptep_xchg_start(mm, addr, ptep);
279         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
280         old = ptep_flush_direct(mm, addr, ptep, nodat);
281         old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
282         preempt_enable();
283         return old;
284 }
285 EXPORT_SYMBOL(ptep_xchg_direct);
286 
287 pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr,
288                      pte_t *ptep, pte_t new)
289 {
290         pgste_t pgste;
291         pte_t old;
292         int nodat;
293 
294         preempt_disable();
295         pgste = ptep_xchg_start(mm, addr, ptep);
296         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
297         old = ptep_flush_lazy(mm, addr, ptep, nodat);
298         old = ptep_xchg_commit(mm, addr, ptep, pgste, old, new);
299         preempt_enable();
300         return old;
301 }
302 EXPORT_SYMBOL(ptep_xchg_lazy);
303 
304 pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
305                              pte_t *ptep)
306 {
307         pgste_t pgste;
308         pte_t old;
309         int nodat;
310 
311         preempt_disable();
312         pgste = ptep_xchg_start(mm, addr, ptep);
313         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
314         old = ptep_flush_lazy(mm, addr, ptep, nodat);
315         if (mm_has_pgste(mm)) {
316                 pgste = pgste_update_all(old, pgste, mm);
317                 pgste_set(ptep, pgste);
318         }
319         return old;
320 }
321 EXPORT_SYMBOL(ptep_modify_prot_start);
322 
323 void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
324                              pte_t *ptep, pte_t pte)
325 {
326         pgste_t pgste;
327 
328         if (!MACHINE_HAS_NX)
329                 pte_val(pte) &= ~_PAGE_NOEXEC;
330         if (mm_has_pgste(mm)) {
331                 pgste = pgste_get(ptep);
332                 pgste_set_key(ptep, pgste, pte, mm);
333                 pgste = pgste_set_pte(ptep, pgste, pte);
334                 pgste_set_unlock(ptep, pgste);
335         } else {
336                 *ptep = pte;
337         }
338         preempt_enable();
339 }
340 EXPORT_SYMBOL(ptep_modify_prot_commit);
341 
342 static inline void pmdp_idte_local(struct mm_struct *mm,
343                                    unsigned long addr, pmd_t *pmdp)
344 {
345         if (MACHINE_HAS_TLB_GUEST)
346                 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
347                             mm->context.asce, IDTE_LOCAL);
348         else
349                 __pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
350 }
351 
352 static inline void pmdp_idte_global(struct mm_struct *mm,
353                                     unsigned long addr, pmd_t *pmdp)
354 {
355         if (MACHINE_HAS_TLB_GUEST)
356                 __pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
357                             mm->context.asce, IDTE_GLOBAL);
358         else if (MACHINE_HAS_IDTE)
359                 __pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
360         else
361                 __pmdp_csp(pmdp);
362 }
363 
364 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
365                                       unsigned long addr, pmd_t *pmdp)
366 {
367         pmd_t old;
368 
369         old = *pmdp;
370         if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
371                 return old;
372         atomic_inc(&mm->context.flush_count);
373         if (MACHINE_HAS_TLB_LC &&
374             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
375                 pmdp_idte_local(mm, addr, pmdp);
376         else
377                 pmdp_idte_global(mm, addr, pmdp);
378         atomic_dec(&mm->context.flush_count);
379         return old;
380 }
381 
382 static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm,
383                                     unsigned long addr, pmd_t *pmdp)
384 {
385         pmd_t old;
386 
387         old = *pmdp;
388         if (pmd_val(old) & _SEGMENT_ENTRY_INVALID)
389                 return old;
390         atomic_inc(&mm->context.flush_count);
391         if (cpumask_equal(&mm->context.cpu_attach_mask,
392                           cpumask_of(smp_processor_id()))) {
393                 pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
394                 mm->context.flush_mm = 1;
395         } else {
396                 pmdp_idte_global(mm, addr, pmdp);
397         }
398         atomic_dec(&mm->context.flush_count);
399         return old;
400 }
401 
402 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
403                        pmd_t *pmdp, pmd_t new)
404 {
405         pmd_t old;
406 
407         preempt_disable();
408         old = pmdp_flush_direct(mm, addr, pmdp);
409         *pmdp = new;
410         preempt_enable();
411         return old;
412 }
413 EXPORT_SYMBOL(pmdp_xchg_direct);
414 
415 pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr,
416                      pmd_t *pmdp, pmd_t new)
417 {
418         pmd_t old;
419 
420         preempt_disable();
421         old = pmdp_flush_lazy(mm, addr, pmdp);
422         *pmdp = new;
423         preempt_enable();
424         return old;
425 }
426 EXPORT_SYMBOL(pmdp_xchg_lazy);
427 
428 static inline void pudp_idte_local(struct mm_struct *mm,
429                                    unsigned long addr, pud_t *pudp)
430 {
431         if (MACHINE_HAS_TLB_GUEST)
432                 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
433                             mm->context.asce, IDTE_LOCAL);
434         else
435                 __pudp_idte(addr, pudp, 0, 0, IDTE_LOCAL);
436 }
437 
438 static inline void pudp_idte_global(struct mm_struct *mm,
439                                     unsigned long addr, pud_t *pudp)
440 {
441         if (MACHINE_HAS_TLB_GUEST)
442                 __pudp_idte(addr, pudp, IDTE_NODAT | IDTE_GUEST_ASCE,
443                             mm->context.asce, IDTE_GLOBAL);
444         else if (MACHINE_HAS_IDTE)
445                 __pudp_idte(addr, pudp, 0, 0, IDTE_GLOBAL);
446         else
447                 /*
448                  * Invalid bit position is the same for pmd and pud, so we can
449                  * re-use _pmd_csp() here
450                  */
451                 __pmdp_csp((pmd_t *) pudp);
452 }
453 
454 static inline pud_t pudp_flush_direct(struct mm_struct *mm,
455                                       unsigned long addr, pud_t *pudp)
456 {
457         pud_t old;
458 
459         old = *pudp;
460         if (pud_val(old) & _REGION_ENTRY_INVALID)
461                 return old;
462         atomic_inc(&mm->context.flush_count);
463         if (MACHINE_HAS_TLB_LC &&
464             cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id())))
465                 pudp_idte_local(mm, addr, pudp);
466         else
467                 pudp_idte_global(mm, addr, pudp);
468         atomic_dec(&mm->context.flush_count);
469         return old;
470 }
471 
472 pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr,
473                        pud_t *pudp, pud_t new)
474 {
475         pud_t old;
476 
477         preempt_disable();
478         old = pudp_flush_direct(mm, addr, pudp);
479         *pudp = new;
480         preempt_enable();
481         return old;
482 }
483 EXPORT_SYMBOL(pudp_xchg_direct);
484 
485 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
486 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
487                                 pgtable_t pgtable)
488 {
489         struct list_head *lh = (struct list_head *) pgtable;
490 
491         assert_spin_locked(pmd_lockptr(mm, pmdp));
492 
493         /* FIFO */
494         if (!pmd_huge_pte(mm, pmdp))
495                 INIT_LIST_HEAD(lh);
496         else
497                 list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
498         pmd_huge_pte(mm, pmdp) = pgtable;
499 }
500 
501 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
502 {
503         struct list_head *lh;
504         pgtable_t pgtable;
505         pte_t *ptep;
506 
507         assert_spin_locked(pmd_lockptr(mm, pmdp));
508 
509         /* FIFO */
510         pgtable = pmd_huge_pte(mm, pmdp);
511         lh = (struct list_head *) pgtable;
512         if (list_empty(lh))
513                 pmd_huge_pte(mm, pmdp) = NULL;
514         else {
515                 pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
516                 list_del(lh);
517         }
518         ptep = (pte_t *) pgtable;
519         pte_val(*ptep) = _PAGE_INVALID;
520         ptep++;
521         pte_val(*ptep) = _PAGE_INVALID;
522         return pgtable;
523 }
524 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
525 
526 #ifdef CONFIG_PGSTE
527 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr,
528                      pte_t *ptep, pte_t entry)
529 {
530         pgste_t pgste;
531 
532         /* the mm_has_pgste() check is done in set_pte_at() */
533         preempt_disable();
534         pgste = pgste_get_lock(ptep);
535         pgste_val(pgste) &= ~_PGSTE_GPS_ZERO;
536         pgste_set_key(ptep, pgste, entry, mm);
537         pgste = pgste_set_pte(ptep, pgste, entry);
538         pgste_set_unlock(ptep, pgste);
539         preempt_enable();
540 }
541 
542 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
543 {
544         pgste_t pgste;
545 
546         preempt_disable();
547         pgste = pgste_get_lock(ptep);
548         pgste_val(pgste) |= PGSTE_IN_BIT;
549         pgste_set_unlock(ptep, pgste);
550         preempt_enable();
551 }
552 
553 /**
554  * ptep_force_prot - change access rights of a locked pte
555  * @mm: pointer to the process mm_struct
556  * @addr: virtual address in the guest address space
557  * @ptep: pointer to the page table entry
558  * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE
559  * @bit: pgste bit to set (e.g. for notification)
560  *
561  * Returns 0 if the access rights were changed and -EAGAIN if the current
562  * and requested access rights are incompatible.
563  */
564 int ptep_force_prot(struct mm_struct *mm, unsigned long addr,
565                     pte_t *ptep, int prot, unsigned long bit)
566 {
567         pte_t entry;
568         pgste_t pgste;
569         int pte_i, pte_p, nodat;
570 
571         pgste = pgste_get_lock(ptep);
572         entry = *ptep;
573         /* Check pte entry after all locks have been acquired */
574         pte_i = pte_val(entry) & _PAGE_INVALID;
575         pte_p = pte_val(entry) & _PAGE_PROTECT;
576         if ((pte_i && (prot != PROT_NONE)) ||
577             (pte_p && (prot & PROT_WRITE))) {
578                 pgste_set_unlock(ptep, pgste);
579                 return -EAGAIN;
580         }
581         /* Change access rights and set pgste bit */
582         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
583         if (prot == PROT_NONE && !pte_i) {
584                 ptep_flush_direct(mm, addr, ptep, nodat);
585                 pgste = pgste_update_all(entry, pgste, mm);
586                 pte_val(entry) |= _PAGE_INVALID;
587         }
588         if (prot == PROT_READ && !pte_p) {
589                 ptep_flush_direct(mm, addr, ptep, nodat);
590                 pte_val(entry) &= ~_PAGE_INVALID;
591                 pte_val(entry) |= _PAGE_PROTECT;
592         }
593         pgste_val(pgste) |= bit;
594         pgste = pgste_set_pte(ptep, pgste, entry);
595         pgste_set_unlock(ptep, pgste);
596         return 0;
597 }
598 
599 int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr,
600                     pte_t *sptep, pte_t *tptep, pte_t pte)
601 {
602         pgste_t spgste, tpgste;
603         pte_t spte, tpte;
604         int rc = -EAGAIN;
605 
606         if (!(pte_val(*tptep) & _PAGE_INVALID))
607                 return 0;       /* already shadowed */
608         spgste = pgste_get_lock(sptep);
609         spte = *sptep;
610         if (!(pte_val(spte) & _PAGE_INVALID) &&
611             !((pte_val(spte) & _PAGE_PROTECT) &&
612               !(pte_val(pte) & _PAGE_PROTECT))) {
613                 pgste_val(spgste) |= PGSTE_VSIE_BIT;
614                 tpgste = pgste_get_lock(tptep);
615                 pte_val(tpte) = (pte_val(spte) & PAGE_MASK) |
616                                 (pte_val(pte) & _PAGE_PROTECT);
617                 /* don't touch the storage key - it belongs to parent pgste */
618                 tpgste = pgste_set_pte(tptep, tpgste, tpte);
619                 pgste_set_unlock(tptep, tpgste);
620                 rc = 1;
621         }
622         pgste_set_unlock(sptep, spgste);
623         return rc;
624 }
625 
626 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep)
627 {
628         pgste_t pgste;
629         int nodat;
630 
631         pgste = pgste_get_lock(ptep);
632         /* notifier is called by the caller */
633         nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
634         ptep_flush_direct(mm, saddr, ptep, nodat);
635         /* don't touch the storage key - it belongs to parent pgste */
636         pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID));
637         pgste_set_unlock(ptep, pgste);
638 }
639 
640 static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry)
641 {
642         if (!non_swap_entry(entry))
643                 dec_mm_counter(mm, MM_SWAPENTS);
644         else if (is_migration_entry(entry)) {
645                 struct page *page = migration_entry_to_page(entry);
646 
647                 dec_mm_counter(mm, mm_counter(page));
648         }
649         free_swap_and_cache(entry);
650 }
651 
652 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr,
653                      pte_t *ptep, int reset)
654 {
655         unsigned long pgstev;
656         pgste_t pgste;
657         pte_t pte;
658 
659         /* Zap unused and logically-zero pages */
660         preempt_disable();
661         pgste = pgste_get_lock(ptep);
662         pgstev = pgste_val(pgste);
663         pte = *ptep;
664         if (!reset && pte_swap(pte) &&
665             ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED ||
666              (pgstev & _PGSTE_GPS_ZERO))) {
667                 ptep_zap_swap_entry(mm, pte_to_swp_entry(pte));
668                 pte_clear(mm, addr, ptep);
669         }
670         if (reset)
671                 pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK;
672         pgste_set_unlock(ptep, pgste);
673         preempt_enable();
674 }
675 
676 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
677 {
678         unsigned long ptev;
679         pgste_t pgste;
680 
681         /* Clear storage key ACC and F, but set R/C */
682         preempt_disable();
683         pgste = pgste_get_lock(ptep);
684         pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT);
685         pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT;
686         ptev = pte_val(*ptep);
687         if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
688                 page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
689         pgste_set_unlock(ptep, pgste);
690         preempt_enable();
691 }
692 
693 /*
694  * Test and reset if a guest page is dirty
695  */
696 bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
697 {
698         spinlock_t *ptl;
699         pgd_t *pgd;
700         p4d_t *p4d;
701         pud_t *pud;
702         pmd_t *pmd;
703         pgste_t pgste;
704         pte_t *ptep;
705         pte_t pte;
706         bool dirty;
707         int nodat;
708 
709         pgd = pgd_offset(mm, addr);
710         p4d = p4d_alloc(mm, pgd, addr);
711         if (!p4d)
712                 return false;
713         pud = pud_alloc(mm, p4d, addr);
714         if (!pud)
715                 return false;
716         pmd = pmd_alloc(mm, pud, addr);
717         if (!pmd)
718                 return false;
719         /* We can't run guests backed by huge pages, but userspace can
720          * still set them up and then try to migrate them without any
721          * migration support.
722          */
723         if (pmd_large(*pmd))
724                 return true;
725 
726         ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl);
727         if (unlikely(!ptep))
728                 return false;
729 
730         pgste = pgste_get_lock(ptep);
731         dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
732         pgste_val(pgste) &= ~PGSTE_UC_BIT;
733         pte = *ptep;
734         if (dirty && (pte_val(pte) & _PAGE_PRESENT)) {
735                 pgste = pgste_pte_notify(mm, addr, ptep, pgste);
736                 nodat = !!(pgste_val(pgste) & _PGSTE_GPS_NODAT);
737                 ptep_ipte_global(mm, addr, ptep, nodat);
738                 if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE))
739                         pte_val(pte) |= _PAGE_PROTECT;
740                 else
741                         pte_val(pte) |= _PAGE_INVALID;
742                 *ptep = pte;
743         }
744         pgste_set_unlock(ptep, pgste);
745 
746         spin_unlock(ptl);
747         return dirty;
748 }
749 EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
750 
751 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
752                           unsigned char key, bool nq)
753 {
754         unsigned long keyul;
755         spinlock_t *ptl;
756         pgste_t old, new;
757         pte_t *ptep;
758 
759         ptep = get_locked_pte(mm, addr, &ptl);
760         if (unlikely(!ptep))
761                 return -EFAULT;
762 
763         new = old = pgste_get_lock(ptep);
764         pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT |
765                             PGSTE_ACC_BITS | PGSTE_FP_BIT);
766         keyul = (unsigned long) key;
767         pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
768         pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
769         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
770                 unsigned long address, bits, skey;
771 
772                 address = pte_val(*ptep) & PAGE_MASK;
773                 skey = (unsigned long) page_get_storage_key(address);
774                 bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
775                 skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
776                 /* Set storage key ACC and FP */
777                 page_set_storage_key(address, skey, !nq);
778                 /* Merge host changed & referenced into pgste  */
779                 pgste_val(new) |= bits << 52;
780         }
781         /* changing the guest storage key is considered a change of the page */
782         if ((pgste_val(new) ^ pgste_val(old)) &
783             (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT))
784                 pgste_val(new) |= PGSTE_UC_BIT;
785 
786         pgste_set_unlock(ptep, new);
787         pte_unmap_unlock(ptep, ptl);
788         return 0;
789 }
790 EXPORT_SYMBOL(set_guest_storage_key);
791 
792 /**
793  * Conditionally set a guest storage key (handling csske).
794  * oldkey will be updated when either mr or mc is set and a pointer is given.
795  *
796  * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest
797  * storage key was updated and -EFAULT on access errors.
798  */
799 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
800                                unsigned char key, unsigned char *oldkey,
801                                bool nq, bool mr, bool mc)
802 {
803         unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT;
804         int rc;
805 
806         /* we can drop the pgste lock between getting and setting the key */
807         if (mr | mc) {
808                 rc = get_guest_storage_key(current->mm, addr, &tmp);
809                 if (rc)
810                         return rc;
811                 if (oldkey)
812                         *oldkey = tmp;
813                 if (!mr)
814                         mask |= _PAGE_REFERENCED;
815                 if (!mc)
816                         mask |= _PAGE_CHANGED;
817                 if (!((tmp ^ key) & mask))
818                         return 0;
819         }
820         rc = set_guest_storage_key(current->mm, addr, key, nq);
821         return rc < 0 ? rc : 1;
822 }
823 EXPORT_SYMBOL(cond_set_guest_storage_key);
824 
825 /**
826  * Reset a guest reference bit (rrbe), returning the reference and changed bit.
827  *
828  * Returns < 0 in case of error, otherwise the cc to be reported to the guest.
829  */
830 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
831 {
832         spinlock_t *ptl;
833         pgste_t old, new;
834         pte_t *ptep;
835         int cc = 0;
836 
837         ptep = get_locked_pte(mm, addr, &ptl);
838         if (unlikely(!ptep))
839                 return -EFAULT;
840 
841         new = old = pgste_get_lock(ptep);
842         /* Reset guest reference bit only */
843         pgste_val(new) &= ~PGSTE_GR_BIT;
844 
845         if (!(pte_val(*ptep) & _PAGE_INVALID)) {
846                 cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
847                 /* Merge real referenced bit into host-set */
848                 pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
849         }
850         /* Reflect guest's logical view, not physical */
851         cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49;
852         /* Changing the guest storage key is considered a change of the page */
853         if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT)
854                 pgste_val(new) |= PGSTE_UC_BIT;
855 
856         pgste_set_unlock(ptep, new);
857         pte_unmap_unlock(ptep, ptl);
858         return cc;
859 }
860 EXPORT_SYMBOL(reset_guest_reference_bit);
861 
862 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
863                           unsigned char *key)
864 {
865         spinlock_t *ptl;
866         pgste_t pgste;
867         pte_t *ptep;
868 
869         ptep = get_locked_pte(mm, addr, &ptl);
870         if (unlikely(!ptep))
871                 return -EFAULT;
872 
873         pgste = pgste_get_lock(ptep);
874         *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
875         if (!(pte_val(*ptep) & _PAGE_INVALID))
876                 *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);
877         /* Reflect guest's logical view, not physical */
878         *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
879         pgste_set_unlock(ptep, pgste);
880         pte_unmap_unlock(ptep, ptl);
881         return 0;
882 }
883 EXPORT_SYMBOL(get_guest_storage_key);
884 
885 /**
886  * pgste_perform_essa - perform ESSA actions on the PGSTE.
887  * @mm: the memory context. It must have PGSTEs, no check is performed here!
888  * @hva: the host virtual address of the page whose PGSTE is to be processed
889  * @orc: the specific action to perform, see the ESSA_SET_* macros.
890  * @oldpte: the PTE will be saved there if the pointer is not NULL.
891  * @oldpgste: the old PGSTE will be saved there if the pointer is not NULL.
892  *
893  * Return: 1 if the page is to be added to the CBRL, otherwise 0,
894  *         or < 0 in case of error. -EINVAL is returned for invalid values
895  *         of orc, -EFAULT for invalid addresses.
896  */
897 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
898                         unsigned long *oldpte, unsigned long *oldpgste)
899 {
900         unsigned long pgstev;
901         spinlock_t *ptl;
902         pgste_t pgste;
903         pte_t *ptep;
904         int res = 0;
905 
906         WARN_ON_ONCE(orc > ESSA_MAX);
907         if (unlikely(orc > ESSA_MAX))
908                 return -EINVAL;
909         ptep = get_locked_pte(mm, hva, &ptl);
910         if (unlikely(!ptep))
911                 return -EFAULT;
912         pgste = pgste_get_lock(ptep);
913         pgstev = pgste_val(pgste);
914         if (oldpte)
915                 *oldpte = pte_val(*ptep);
916         if (oldpgste)
917                 *oldpgste = pgstev;
918 
919         switch (orc) {
920         case ESSA_GET_STATE:
921                 break;
922         case ESSA_SET_STABLE:
923                 pgstev &= ~(_PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT);
924                 pgstev |= _PGSTE_GPS_USAGE_STABLE;
925                 break;
926         case ESSA_SET_UNUSED:
927                 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
928                 pgstev |= _PGSTE_GPS_USAGE_UNUSED;
929                 if (pte_val(*ptep) & _PAGE_INVALID)
930                         res = 1;
931                 break;
932         case ESSA_SET_VOLATILE:
933                 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
934                 pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
935                 if (pte_val(*ptep) & _PAGE_INVALID)
936                         res = 1;
937                 break;
938         case ESSA_SET_POT_VOLATILE:
939                 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
940                 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
941                         pgstev |= _PGSTE_GPS_USAGE_POT_VOLATILE;
942                         break;
943                 }
944                 if (pgstev & _PGSTE_GPS_ZERO) {
945                         pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
946                         break;
947                 }
948                 if (!(pgstev & PGSTE_GC_BIT)) {
949                         pgstev |= _PGSTE_GPS_USAGE_VOLATILE;
950                         res = 1;
951                         break;
952                 }
953                 break;
954         case ESSA_SET_STABLE_RESIDENT:
955                 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
956                 pgstev |= _PGSTE_GPS_USAGE_STABLE;
957                 /*
958                  * Since the resident state can go away any time after this
959                  * call, we will not make this page resident. We can revisit
960                  * this decision if a guest will ever start using this.
961                  */
962                 break;
963         case ESSA_SET_STABLE_IF_RESIDENT:
964                 if (!(pte_val(*ptep) & _PAGE_INVALID)) {
965                         pgstev &= ~_PGSTE_GPS_USAGE_MASK;
966                         pgstev |= _PGSTE_GPS_USAGE_STABLE;
967                 }
968                 break;
969         case ESSA_SET_STABLE_NODAT:
970                 pgstev &= ~_PGSTE_GPS_USAGE_MASK;
971                 pgstev |= _PGSTE_GPS_USAGE_STABLE | _PGSTE_GPS_NODAT;
972                 break;
973         default:
974                 /* we should never get here! */
975                 break;
976         }
977         /* If we are discarding a page, set it to logical zero */
978         if (res)
979                 pgstev |= _PGSTE_GPS_ZERO;
980 
981         pgste_val(pgste) = pgstev;
982         pgste_set_unlock(ptep, pgste);
983         pte_unmap_unlock(ptep, ptl);
984         return res;
985 }
986 EXPORT_SYMBOL(pgste_perform_essa);
987 
988 /**
989  * set_pgste_bits - set specific PGSTE bits.
990  * @mm: the memory context. It must have PGSTEs, no check is performed here!
991  * @hva: the host virtual address of the page whose PGSTE is to be processed
992  * @bits: a bitmask representing the bits that will be touched
993  * @value: the values of the bits to be written. Only the bits in the mask
994  *         will be written.
995  *
996  * Return: 0 on success, < 0 in case of error.
997  */
998 int set_pgste_bits(struct mm_struct *mm, unsigned long hva,
999                         unsigned long bits, unsigned long value)
1000 {
1001         spinlock_t *ptl;
1002         pgste_t new;
1003         pte_t *ptep;
1004 
1005         ptep = get_locked_pte(mm, hva, &ptl);
1006         if (unlikely(!ptep))
1007                 return -EFAULT;
1008         new = pgste_get_lock(ptep);
1009 
1010         pgste_val(new) &= ~bits;
1011         pgste_val(new) |= value & bits;
1012 
1013         pgste_set_unlock(ptep, new);
1014         pte_unmap_unlock(ptep, ptl);
1015         return 0;
1016 }
1017 EXPORT_SYMBOL(set_pgste_bits);
1018 
1019 /**
1020  * get_pgste - get the current PGSTE for the given address.
1021  * @mm: the memory context. It must have PGSTEs, no check is performed here!
1022  * @hva: the host virtual address of the page whose PGSTE is to be processed
1023  * @pgstep: will be written with the current PGSTE for the given address.
1024  *
1025  * Return: 0 on success, < 0 in case of error.
1026  */
1027 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep)
1028 {
1029         spinlock_t *ptl;
1030         pte_t *ptep;
1031 
1032         ptep = get_locked_pte(mm, hva, &ptl);
1033         if (unlikely(!ptep))
1034                 return -EFAULT;
1035         *pgstep = pgste_val(pgste_get(ptep));
1036         pte_unmap_unlock(ptep, ptl);
1037         return 0;
1038 }
1039 EXPORT_SYMBOL(get_pgste);
1040 #endif
1041 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp