~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/include/asm/pgtable.h

Version: ~ [ linux-5.9 ] ~ [ linux-5.8.14 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.70 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.150 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.200 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.238 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.238 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 #ifndef _ASM_X86_PGTABLE_H
  3 #define _ASM_X86_PGTABLE_H
  4 
  5 #include <linux/mem_encrypt.h>
  6 #include <asm/page.h>
  7 #include <asm/pgtable_types.h>
  8 
  9 /*
 10  * Macro to mark a page protection value as UC-
 11  */
 12 #define pgprot_noncached(prot)                                          \
 13         ((boot_cpu_data.x86 > 3)                                        \
 14          ? (__pgprot(pgprot_val(prot) |                                 \
 15                      cachemode2protval(_PAGE_CACHE_MODE_UC_MINUS)))     \
 16          : (prot))
 17 
 18 /*
 19  * Macros to add or remove encryption attribute
 20  */
 21 #define pgprot_encrypted(prot)  __pgprot(__sme_set(pgprot_val(prot)))
 22 #define pgprot_decrypted(prot)  __pgprot(__sme_clr(pgprot_val(prot)))
 23 
 24 #ifndef __ASSEMBLY__
 25 #include <asm/x86_init.h>
 26 
 27 extern pgd_t early_top_pgt[PTRS_PER_PGD];
 28 int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
 29 
 30 void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
 31 void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
 32 void ptdump_walk_pgd_level_checkwx(void);
 33 
 34 #ifdef CONFIG_DEBUG_WX
 35 #define debug_checkwx() ptdump_walk_pgd_level_checkwx()
 36 #else
 37 #define debug_checkwx() do { } while (0)
 38 #endif
 39 
 40 /*
 41  * ZERO_PAGE is a global shared page that is always zero: used
 42  * for zero-mapped memory areas etc..
 43  */
 44 extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)]
 45         __visible;
 46 #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page))
 47 
 48 extern spinlock_t pgd_lock;
 49 extern struct list_head pgd_list;
 50 
 51 extern struct mm_struct *pgd_page_get_mm(struct page *page);
 52 
 53 extern pmdval_t early_pmd_flags;
 54 
 55 #ifdef CONFIG_PARAVIRT
 56 #include <asm/paravirt.h>
 57 #else  /* !CONFIG_PARAVIRT */
 58 #define set_pte(ptep, pte)              native_set_pte(ptep, pte)
 59 #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte)
 60 
 61 #define set_pte_atomic(ptep, pte)                                       \
 62         native_set_pte_atomic(ptep, pte)
 63 
 64 #define set_pmd(pmdp, pmd)              native_set_pmd(pmdp, pmd)
 65 
 66 #ifndef __PAGETABLE_P4D_FOLDED
 67 #define set_pgd(pgdp, pgd)              native_set_pgd(pgdp, pgd)
 68 #define pgd_clear(pgd)                  (pgtable_l5_enabled() ? native_pgd_clear(pgd) : 0)
 69 #endif
 70 
 71 #ifndef set_p4d
 72 # define set_p4d(p4dp, p4d)             native_set_p4d(p4dp, p4d)
 73 #endif
 74 
 75 #ifndef __PAGETABLE_PUD_FOLDED
 76 #define p4d_clear(p4d)                  native_p4d_clear(p4d)
 77 #endif
 78 
 79 #ifndef set_pud
 80 # define set_pud(pudp, pud)             native_set_pud(pudp, pud)
 81 #endif
 82 
 83 #ifndef __PAGETABLE_PUD_FOLDED
 84 #define pud_clear(pud)                  native_pud_clear(pud)
 85 #endif
 86 
 87 #define pte_clear(mm, addr, ptep)       native_pte_clear(mm, addr, ptep)
 88 #define pmd_clear(pmd)                  native_pmd_clear(pmd)
 89 
 90 #define pgd_val(x)      native_pgd_val(x)
 91 #define __pgd(x)        native_make_pgd(x)
 92 
 93 #ifndef __PAGETABLE_P4D_FOLDED
 94 #define p4d_val(x)      native_p4d_val(x)
 95 #define __p4d(x)        native_make_p4d(x)
 96 #endif
 97 
 98 #ifndef __PAGETABLE_PUD_FOLDED
 99 #define pud_val(x)      native_pud_val(x)
100 #define __pud(x)        native_make_pud(x)
101 #endif
102 
103 #ifndef __PAGETABLE_PMD_FOLDED
104 #define pmd_val(x)      native_pmd_val(x)
105 #define __pmd(x)        native_make_pmd(x)
106 #endif
107 
108 #define pte_val(x)      native_pte_val(x)
109 #define __pte(x)        native_make_pte(x)
110 
111 #define arch_end_context_switch(prev)   do {} while(0)
112 
113 #endif  /* CONFIG_PARAVIRT */
114 
115 /*
116  * The following only work if pte_present() is true.
117  * Undefined behaviour if not..
118  */
119 static inline int pte_dirty(pte_t pte)
120 {
121         return pte_flags(pte) & _PAGE_DIRTY;
122 }
123 
124 
125 static inline u32 read_pkru(void)
126 {
127         if (boot_cpu_has(X86_FEATURE_OSPKE))
128                 return __read_pkru();
129         return 0;
130 }
131 
132 static inline void write_pkru(u32 pkru)
133 {
134         if (boot_cpu_has(X86_FEATURE_OSPKE))
135                 __write_pkru(pkru);
136 }
137 
138 static inline int pte_young(pte_t pte)
139 {
140         return pte_flags(pte) & _PAGE_ACCESSED;
141 }
142 
143 static inline int pmd_dirty(pmd_t pmd)
144 {
145         return pmd_flags(pmd) & _PAGE_DIRTY;
146 }
147 
148 static inline int pmd_young(pmd_t pmd)
149 {
150         return pmd_flags(pmd) & _PAGE_ACCESSED;
151 }
152 
153 static inline int pud_dirty(pud_t pud)
154 {
155         return pud_flags(pud) & _PAGE_DIRTY;
156 }
157 
158 static inline int pud_young(pud_t pud)
159 {
160         return pud_flags(pud) & _PAGE_ACCESSED;
161 }
162 
163 static inline int pte_write(pte_t pte)
164 {
165         return pte_flags(pte) & _PAGE_RW;
166 }
167 
168 static inline int pte_huge(pte_t pte)
169 {
170         return pte_flags(pte) & _PAGE_PSE;
171 }
172 
173 static inline int pte_global(pte_t pte)
174 {
175         return pte_flags(pte) & _PAGE_GLOBAL;
176 }
177 
178 static inline int pte_exec(pte_t pte)
179 {
180         return !(pte_flags(pte) & _PAGE_NX);
181 }
182 
183 static inline int pte_special(pte_t pte)
184 {
185         return pte_flags(pte) & _PAGE_SPECIAL;
186 }
187 
188 /* Entries that were set to PROT_NONE are inverted */
189 
190 static inline u64 protnone_mask(u64 val);
191 
192 static inline unsigned long pte_pfn(pte_t pte)
193 {
194         phys_addr_t pfn = pte_val(pte);
195         pfn ^= protnone_mask(pfn);
196         return (pfn & PTE_PFN_MASK) >> PAGE_SHIFT;
197 }
198 
199 static inline unsigned long pmd_pfn(pmd_t pmd)
200 {
201         phys_addr_t pfn = pmd_val(pmd);
202         pfn ^= protnone_mask(pfn);
203         return (pfn & pmd_pfn_mask(pmd)) >> PAGE_SHIFT;
204 }
205 
206 static inline unsigned long pud_pfn(pud_t pud)
207 {
208         phys_addr_t pfn = pud_val(pud);
209         pfn ^= protnone_mask(pfn);
210         return (pfn & pud_pfn_mask(pud)) >> PAGE_SHIFT;
211 }
212 
213 static inline unsigned long p4d_pfn(p4d_t p4d)
214 {
215         return (p4d_val(p4d) & p4d_pfn_mask(p4d)) >> PAGE_SHIFT;
216 }
217 
218 static inline unsigned long pgd_pfn(pgd_t pgd)
219 {
220         return (pgd_val(pgd) & PTE_PFN_MASK) >> PAGE_SHIFT;
221 }
222 
223 static inline int p4d_large(p4d_t p4d)
224 {
225         /* No 512 GiB pages yet */
226         return 0;
227 }
228 
229 #define pte_page(pte)   pfn_to_page(pte_pfn(pte))
230 
231 static inline int pmd_large(pmd_t pte)
232 {
233         return pmd_flags(pte) & _PAGE_PSE;
234 }
235 
236 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
237 static inline int pmd_trans_huge(pmd_t pmd)
238 {
239         return (pmd_val(pmd) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
240 }
241 
242 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
243 static inline int pud_trans_huge(pud_t pud)
244 {
245         return (pud_val(pud) & (_PAGE_PSE|_PAGE_DEVMAP)) == _PAGE_PSE;
246 }
247 #endif
248 
249 #define has_transparent_hugepage has_transparent_hugepage
250 static inline int has_transparent_hugepage(void)
251 {
252         return boot_cpu_has(X86_FEATURE_PSE);
253 }
254 
255 #ifdef __HAVE_ARCH_PTE_DEVMAP
256 static inline int pmd_devmap(pmd_t pmd)
257 {
258         return !!(pmd_val(pmd) & _PAGE_DEVMAP);
259 }
260 
261 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
262 static inline int pud_devmap(pud_t pud)
263 {
264         return !!(pud_val(pud) & _PAGE_DEVMAP);
265 }
266 #else
267 static inline int pud_devmap(pud_t pud)
268 {
269         return 0;
270 }
271 #endif
272 
273 static inline int pgd_devmap(pgd_t pgd)
274 {
275         return 0;
276 }
277 #endif
278 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
279 
280 static inline pte_t pte_set_flags(pte_t pte, pteval_t set)
281 {
282         pteval_t v = native_pte_val(pte);
283 
284         return native_make_pte(v | set);
285 }
286 
287 static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear)
288 {
289         pteval_t v = native_pte_val(pte);
290 
291         return native_make_pte(v & ~clear);
292 }
293 
294 static inline pte_t pte_mkclean(pte_t pte)
295 {
296         return pte_clear_flags(pte, _PAGE_DIRTY);
297 }
298 
299 static inline pte_t pte_mkold(pte_t pte)
300 {
301         return pte_clear_flags(pte, _PAGE_ACCESSED);
302 }
303 
304 static inline pte_t pte_wrprotect(pte_t pte)
305 {
306         return pte_clear_flags(pte, _PAGE_RW);
307 }
308 
309 static inline pte_t pte_mkexec(pte_t pte)
310 {
311         return pte_clear_flags(pte, _PAGE_NX);
312 }
313 
314 static inline pte_t pte_mkdirty(pte_t pte)
315 {
316         return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
317 }
318 
319 static inline pte_t pte_mkyoung(pte_t pte)
320 {
321         return pte_set_flags(pte, _PAGE_ACCESSED);
322 }
323 
324 static inline pte_t pte_mkwrite(pte_t pte)
325 {
326         return pte_set_flags(pte, _PAGE_RW);
327 }
328 
329 static inline pte_t pte_mkhuge(pte_t pte)
330 {
331         return pte_set_flags(pte, _PAGE_PSE);
332 }
333 
334 static inline pte_t pte_clrhuge(pte_t pte)
335 {
336         return pte_clear_flags(pte, _PAGE_PSE);
337 }
338 
339 static inline pte_t pte_mkglobal(pte_t pte)
340 {
341         return pte_set_flags(pte, _PAGE_GLOBAL);
342 }
343 
344 static inline pte_t pte_clrglobal(pte_t pte)
345 {
346         return pte_clear_flags(pte, _PAGE_GLOBAL);
347 }
348 
349 static inline pte_t pte_mkspecial(pte_t pte)
350 {
351         return pte_set_flags(pte, _PAGE_SPECIAL);
352 }
353 
354 static inline pte_t pte_mkdevmap(pte_t pte)
355 {
356         return pte_set_flags(pte, _PAGE_SPECIAL|_PAGE_DEVMAP);
357 }
358 
359 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set)
360 {
361         pmdval_t v = native_pmd_val(pmd);
362 
363         return native_make_pmd(v | set);
364 }
365 
366 static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear)
367 {
368         pmdval_t v = native_pmd_val(pmd);
369 
370         return native_make_pmd(v & ~clear);
371 }
372 
373 static inline pmd_t pmd_mkold(pmd_t pmd)
374 {
375         return pmd_clear_flags(pmd, _PAGE_ACCESSED);
376 }
377 
378 static inline pmd_t pmd_mkclean(pmd_t pmd)
379 {
380         return pmd_clear_flags(pmd, _PAGE_DIRTY);
381 }
382 
383 static inline pmd_t pmd_wrprotect(pmd_t pmd)
384 {
385         return pmd_clear_flags(pmd, _PAGE_RW);
386 }
387 
388 static inline pmd_t pmd_mkdirty(pmd_t pmd)
389 {
390         return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
391 }
392 
393 static inline pmd_t pmd_mkdevmap(pmd_t pmd)
394 {
395         return pmd_set_flags(pmd, _PAGE_DEVMAP);
396 }
397 
398 static inline pmd_t pmd_mkhuge(pmd_t pmd)
399 {
400         return pmd_set_flags(pmd, _PAGE_PSE);
401 }
402 
403 static inline pmd_t pmd_mkyoung(pmd_t pmd)
404 {
405         return pmd_set_flags(pmd, _PAGE_ACCESSED);
406 }
407 
408 static inline pmd_t pmd_mkwrite(pmd_t pmd)
409 {
410         return pmd_set_flags(pmd, _PAGE_RW);
411 }
412 
413 static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
414 {
415         pudval_t v = native_pud_val(pud);
416 
417         return native_make_pud(v | set);
418 }
419 
420 static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear)
421 {
422         pudval_t v = native_pud_val(pud);
423 
424         return native_make_pud(v & ~clear);
425 }
426 
427 static inline pud_t pud_mkold(pud_t pud)
428 {
429         return pud_clear_flags(pud, _PAGE_ACCESSED);
430 }
431 
432 static inline pud_t pud_mkclean(pud_t pud)
433 {
434         return pud_clear_flags(pud, _PAGE_DIRTY);
435 }
436 
437 static inline pud_t pud_wrprotect(pud_t pud)
438 {
439         return pud_clear_flags(pud, _PAGE_RW);
440 }
441 
442 static inline pud_t pud_mkdirty(pud_t pud)
443 {
444         return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
445 }
446 
447 static inline pud_t pud_mkdevmap(pud_t pud)
448 {
449         return pud_set_flags(pud, _PAGE_DEVMAP);
450 }
451 
452 static inline pud_t pud_mkhuge(pud_t pud)
453 {
454         return pud_set_flags(pud, _PAGE_PSE);
455 }
456 
457 static inline pud_t pud_mkyoung(pud_t pud)
458 {
459         return pud_set_flags(pud, _PAGE_ACCESSED);
460 }
461 
462 static inline pud_t pud_mkwrite(pud_t pud)
463 {
464         return pud_set_flags(pud, _PAGE_RW);
465 }
466 
467 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
468 static inline int pte_soft_dirty(pte_t pte)
469 {
470         return pte_flags(pte) & _PAGE_SOFT_DIRTY;
471 }
472 
473 static inline int pmd_soft_dirty(pmd_t pmd)
474 {
475         return pmd_flags(pmd) & _PAGE_SOFT_DIRTY;
476 }
477 
478 static inline int pud_soft_dirty(pud_t pud)
479 {
480         return pud_flags(pud) & _PAGE_SOFT_DIRTY;
481 }
482 
483 static inline pte_t pte_mksoft_dirty(pte_t pte)
484 {
485         return pte_set_flags(pte, _PAGE_SOFT_DIRTY);
486 }
487 
488 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd)
489 {
490         return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY);
491 }
492 
493 static inline pud_t pud_mksoft_dirty(pud_t pud)
494 {
495         return pud_set_flags(pud, _PAGE_SOFT_DIRTY);
496 }
497 
498 static inline pte_t pte_clear_soft_dirty(pte_t pte)
499 {
500         return pte_clear_flags(pte, _PAGE_SOFT_DIRTY);
501 }
502 
503 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd)
504 {
505         return pmd_clear_flags(pmd, _PAGE_SOFT_DIRTY);
506 }
507 
508 static inline pud_t pud_clear_soft_dirty(pud_t pud)
509 {
510         return pud_clear_flags(pud, _PAGE_SOFT_DIRTY);
511 }
512 
513 #endif /* CONFIG_HAVE_ARCH_SOFT_DIRTY */
514 
515 /*
516  * Mask out unsupported bits in a present pgprot.  Non-present pgprots
517  * can use those bits for other purposes, so leave them be.
518  */
519 static inline pgprotval_t massage_pgprot(pgprot_t pgprot)
520 {
521         pgprotval_t protval = pgprot_val(pgprot);
522 
523         if (protval & _PAGE_PRESENT)
524                 protval &= __supported_pte_mask;
525 
526         return protval;
527 }
528 
529 static inline pgprotval_t check_pgprot(pgprot_t pgprot)
530 {
531         pgprotval_t massaged_val = massage_pgprot(pgprot);
532 
533         /* mmdebug.h can not be included here because of dependencies */
534 #ifdef CONFIG_DEBUG_VM
535         WARN_ONCE(pgprot_val(pgprot) != massaged_val,
536                   "attempted to set unsupported pgprot: %016llx "
537                   "bits: %016llx supported: %016llx\n",
538                   (u64)pgprot_val(pgprot),
539                   (u64)pgprot_val(pgprot) ^ massaged_val,
540                   (u64)__supported_pte_mask);
541 #endif
542 
543         return massaged_val;
544 }
545 
546 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot)
547 {
548         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
549         pfn ^= protnone_mask(pgprot_val(pgprot));
550         pfn &= PTE_PFN_MASK;
551         return __pte(pfn | check_pgprot(pgprot));
552 }
553 
554 static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
555 {
556         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
557         pfn ^= protnone_mask(pgprot_val(pgprot));
558         pfn &= PHYSICAL_PMD_PAGE_MASK;
559         return __pmd(pfn | check_pgprot(pgprot));
560 }
561 
562 static inline pud_t pfn_pud(unsigned long page_nr, pgprot_t pgprot)
563 {
564         phys_addr_t pfn = (phys_addr_t)page_nr << PAGE_SHIFT;
565         pfn ^= protnone_mask(pgprot_val(pgprot));
566         pfn &= PHYSICAL_PUD_PAGE_MASK;
567         return __pud(pfn | check_pgprot(pgprot));
568 }
569 
570 static inline pmd_t pmd_mknotpresent(pmd_t pmd)
571 {
572         return pfn_pmd(pmd_pfn(pmd),
573                       __pgprot(pmd_flags(pmd) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
574 }
575 
576 static inline pud_t pud_mknotpresent(pud_t pud)
577 {
578         return pfn_pud(pud_pfn(pud),
579               __pgprot(pud_flags(pud) & ~(_PAGE_PRESENT|_PAGE_PROTNONE)));
580 }
581 
582 static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask);
583 
584 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
585 {
586         pteval_t val = pte_val(pte), oldval = val;
587 
588         /*
589          * Chop off the NX bit (if present), and add the NX portion of
590          * the newprot (if present):
591          */
592         val &= _PAGE_CHG_MASK;
593         val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
594         val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
595         return __pte(val);
596 }
597 
598 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
599 {
600         pmdval_t val = pmd_val(pmd), oldval = val;
601 
602         val &= _HPAGE_CHG_MASK;
603         val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
604         val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
605         return __pmd(val);
606 }
607 
608 /* mprotect needs to preserve PAT bits when updating vm_page_prot */
609 #define pgprot_modify pgprot_modify
610 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot)
611 {
612         pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK;
613         pgprotval_t addbits = pgprot_val(newprot);
614         return __pgprot(preservebits | addbits);
615 }
616 
617 #define pte_pgprot(x) __pgprot(pte_flags(x))
618 #define pmd_pgprot(x) __pgprot(pmd_flags(x))
619 #define pud_pgprot(x) __pgprot(pud_flags(x))
620 #define p4d_pgprot(x) __pgprot(p4d_flags(x))
621 
622 #define canon_pgprot(p) __pgprot(massage_pgprot(p))
623 
624 static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
625 {
626         return canon_pgprot(prot);
627 }
628 
629 static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
630                                          enum page_cache_mode pcm,
631                                          enum page_cache_mode new_pcm)
632 {
633         /*
634          * PAT type is always WB for untracked ranges, so no need to check.
635          */
636         if (x86_platform.is_untracked_pat_range(paddr, paddr + size))
637                 return 1;
638 
639         /*
640          * Certain new memtypes are not allowed with certain
641          * requested memtype:
642          * - request is uncached, return cannot be write-back
643          * - request is write-combine, return cannot be write-back
644          * - request is write-through, return cannot be write-back
645          * - request is write-through, return cannot be write-combine
646          */
647         if ((pcm == _PAGE_CACHE_MODE_UC_MINUS &&
648              new_pcm == _PAGE_CACHE_MODE_WB) ||
649             (pcm == _PAGE_CACHE_MODE_WC &&
650              new_pcm == _PAGE_CACHE_MODE_WB) ||
651             (pcm == _PAGE_CACHE_MODE_WT &&
652              new_pcm == _PAGE_CACHE_MODE_WB) ||
653             (pcm == _PAGE_CACHE_MODE_WT &&
654              new_pcm == _PAGE_CACHE_MODE_WC)) {
655                 return 0;
656         }
657 
658         return 1;
659 }
660 
661 pmd_t *populate_extra_pmd(unsigned long vaddr);
662 pte_t *populate_extra_pte(unsigned long vaddr);
663 #endif  /* __ASSEMBLY__ */
664 
665 #ifdef CONFIG_X86_32
666 # include <asm/pgtable_32.h>
667 #else
668 # include <asm/pgtable_64.h>
669 #endif
670 
671 #ifndef __ASSEMBLY__
672 #include <linux/mm_types.h>
673 #include <linux/mmdebug.h>
674 #include <linux/log2.h>
675 #include <asm/fixmap.h>
676 
677 static inline int pte_none(pte_t pte)
678 {
679         return !(pte.pte & ~(_PAGE_KNL_ERRATUM_MASK));
680 }
681 
682 #define __HAVE_ARCH_PTE_SAME
683 static inline int pte_same(pte_t a, pte_t b)
684 {
685         return a.pte == b.pte;
686 }
687 
688 static inline int pte_present(pte_t a)
689 {
690         return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE);
691 }
692 
693 #ifdef __HAVE_ARCH_PTE_DEVMAP
694 static inline int pte_devmap(pte_t a)
695 {
696         return (pte_flags(a) & _PAGE_DEVMAP) == _PAGE_DEVMAP;
697 }
698 #endif
699 
700 #define pte_accessible pte_accessible
701 static inline bool pte_accessible(struct mm_struct *mm, pte_t a)
702 {
703         if (pte_flags(a) & _PAGE_PRESENT)
704                 return true;
705 
706         if ((pte_flags(a) & _PAGE_PROTNONE) &&
707                         mm_tlb_flush_pending(mm))
708                 return true;
709 
710         return false;
711 }
712 
713 static inline int pmd_present(pmd_t pmd)
714 {
715         /*
716          * Checking for _PAGE_PSE is needed too because
717          * split_huge_page will temporarily clear the present bit (but
718          * the _PAGE_PSE flag will remain set at all times while the
719          * _PAGE_PRESENT bit is clear).
720          */
721         return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
722 }
723 
724 #ifdef CONFIG_NUMA_BALANCING
725 /*
726  * These work without NUMA balancing but the kernel does not care. See the
727  * comment in include/asm-generic/pgtable.h
728  */
729 static inline int pte_protnone(pte_t pte)
730 {
731         return (pte_flags(pte) & (_PAGE_PROTNONE | _PAGE_PRESENT))
732                 == _PAGE_PROTNONE;
733 }
734 
735 static inline int pmd_protnone(pmd_t pmd)
736 {
737         return (pmd_flags(pmd) & (_PAGE_PROTNONE | _PAGE_PRESENT))
738                 == _PAGE_PROTNONE;
739 }
740 #endif /* CONFIG_NUMA_BALANCING */
741 
742 static inline int pmd_none(pmd_t pmd)
743 {
744         /* Only check low word on 32-bit platforms, since it might be
745            out of sync with upper half. */
746         unsigned long val = native_pmd_val(pmd);
747         return (val & ~_PAGE_KNL_ERRATUM_MASK) == 0;
748 }
749 
750 static inline unsigned long pmd_page_vaddr(pmd_t pmd)
751 {
752         return (unsigned long)__va(pmd_val(pmd) & pmd_pfn_mask(pmd));
753 }
754 
755 /*
756  * Currently stuck as a macro due to indirect forward reference to
757  * linux/mmzone.h's __section_mem_map_addr() definition:
758  */
759 #define pmd_page(pmd)   pfn_to_page(pmd_pfn(pmd))
760 
761 /*
762  * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD]
763  *
764  * this macro returns the index of the entry in the pmd page which would
765  * control the given virtual address
766  */
767 static inline unsigned long pmd_index(unsigned long address)
768 {
769         return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1);
770 }
771 
772 /*
773  * Conversion functions: convert a page and protection to a page entry,
774  * and a page entry and page directory to the page they refer to.
775  *
776  * (Currently stuck as a macro because of indirect forward reference
777  * to linux/mm.h:page_to_nid())
778  */
779 #define mk_pte(page, pgprot)   pfn_pte(page_to_pfn(page), (pgprot))
780 
781 /*
782  * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE]
783  *
784  * this function returns the index of the entry in the pte page which would
785  * control the given virtual address
786  */
787 static inline unsigned long pte_index(unsigned long address)
788 {
789         return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1);
790 }
791 
792 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
793 {
794         return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address);
795 }
796 
797 static inline int pmd_bad(pmd_t pmd)
798 {
799         return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE;
800 }
801 
802 static inline unsigned long pages_to_mb(unsigned long npg)
803 {
804         return npg >> (20 - PAGE_SHIFT);
805 }
806 
807 #if CONFIG_PGTABLE_LEVELS > 2
808 static inline int pud_none(pud_t pud)
809 {
810         return (native_pud_val(pud) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
811 }
812 
813 static inline int pud_present(pud_t pud)
814 {
815         return pud_flags(pud) & _PAGE_PRESENT;
816 }
817 
818 static inline unsigned long pud_page_vaddr(pud_t pud)
819 {
820         return (unsigned long)__va(pud_val(pud) & pud_pfn_mask(pud));
821 }
822 
823 /*
824  * Currently stuck as a macro due to indirect forward reference to
825  * linux/mmzone.h's __section_mem_map_addr() definition:
826  */
827 #define pud_page(pud)   pfn_to_page(pud_pfn(pud))
828 
829 /* Find an entry in the second-level page table.. */
830 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
831 {
832         return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address);
833 }
834 
835 static inline int pud_large(pud_t pud)
836 {
837         return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) ==
838                 (_PAGE_PSE | _PAGE_PRESENT);
839 }
840 
841 static inline int pud_bad(pud_t pud)
842 {
843         return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0;
844 }
845 #else
846 static inline int pud_large(pud_t pud)
847 {
848         return 0;
849 }
850 #endif  /* CONFIG_PGTABLE_LEVELS > 2 */
851 
852 static inline unsigned long pud_index(unsigned long address)
853 {
854         return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1);
855 }
856 
857 #if CONFIG_PGTABLE_LEVELS > 3
858 static inline int p4d_none(p4d_t p4d)
859 {
860         return (native_p4d_val(p4d) & ~(_PAGE_KNL_ERRATUM_MASK)) == 0;
861 }
862 
863 static inline int p4d_present(p4d_t p4d)
864 {
865         return p4d_flags(p4d) & _PAGE_PRESENT;
866 }
867 
868 static inline unsigned long p4d_page_vaddr(p4d_t p4d)
869 {
870         return (unsigned long)__va(p4d_val(p4d) & p4d_pfn_mask(p4d));
871 }
872 
873 /*
874  * Currently stuck as a macro due to indirect forward reference to
875  * linux/mmzone.h's __section_mem_map_addr() definition:
876  */
877 #define p4d_page(p4d)   pfn_to_page(p4d_pfn(p4d))
878 
879 /* Find an entry in the third-level page table.. */
880 static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address)
881 {
882         return (pud_t *)p4d_page_vaddr(*p4d) + pud_index(address);
883 }
884 
885 static inline int p4d_bad(p4d_t p4d)
886 {
887         unsigned long ignore_flags = _KERNPG_TABLE | _PAGE_USER;
888 
889         if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
890                 ignore_flags |= _PAGE_NX;
891 
892         return (p4d_flags(p4d) & ~ignore_flags) != 0;
893 }
894 #endif  /* CONFIG_PGTABLE_LEVELS > 3 */
895 
896 static inline unsigned long p4d_index(unsigned long address)
897 {
898         return (address >> P4D_SHIFT) & (PTRS_PER_P4D - 1);
899 }
900 
901 #if CONFIG_PGTABLE_LEVELS > 4
902 static inline int pgd_present(pgd_t pgd)
903 {
904         if (!pgtable_l5_enabled())
905                 return 1;
906         return pgd_flags(pgd) & _PAGE_PRESENT;
907 }
908 
909 static inline unsigned long pgd_page_vaddr(pgd_t pgd)
910 {
911         return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK);
912 }
913 
914 /*
915  * Currently stuck as a macro due to indirect forward reference to
916  * linux/mmzone.h's __section_mem_map_addr() definition:
917  */
918 #define pgd_page(pgd)   pfn_to_page(pgd_pfn(pgd))
919 
920 /* to find an entry in a page-table-directory. */
921 static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long address)
922 {
923         if (!pgtable_l5_enabled())
924                 return (p4d_t *)pgd;
925         return (p4d_t *)pgd_page_vaddr(*pgd) + p4d_index(address);
926 }
927 
928 static inline int pgd_bad(pgd_t pgd)
929 {
930         unsigned long ignore_flags = _PAGE_USER;
931 
932         if (!pgtable_l5_enabled())
933                 return 0;
934 
935         if (IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
936                 ignore_flags |= _PAGE_NX;
937 
938         return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
939 }
940 
941 static inline int pgd_none(pgd_t pgd)
942 {
943         if (!pgtable_l5_enabled())
944                 return 0;
945         /*
946          * There is no need to do a workaround for the KNL stray
947          * A/D bit erratum here.  PGDs only point to page tables
948          * except on 32-bit non-PAE which is not supported on
949          * KNL.
950          */
951         return !native_pgd_val(pgd);
952 }
953 #endif  /* CONFIG_PGTABLE_LEVELS > 4 */
954 
955 #endif  /* __ASSEMBLY__ */
956 
957 /*
958  * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD]
959  *
960  * this macro returns the index of the entry in the pgd page which would
961  * control the given virtual address
962  */
963 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1))
964 
965 /*
966  * pgd_offset() returns a (pgd_t *)
967  * pgd_index() is used get the offset into the pgd page's array of pgd_t's;
968  */
969 #define pgd_offset_pgd(pgd, address) (pgd + pgd_index((address)))
970 /*
971  * a shortcut to get a pgd_t in a given mm
972  */
973 #define pgd_offset(mm, address) pgd_offset_pgd((mm)->pgd, (address))
974 /*
975  * a shortcut which implies the use of the kernel's pgd, instead
976  * of a process's
977  */
978 #define pgd_offset_k(address) pgd_offset(&init_mm, (address))
979 
980 
981 #define KERNEL_PGD_BOUNDARY     pgd_index(PAGE_OFFSET)
982 #define KERNEL_PGD_PTRS         (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY)
983 
984 #ifndef __ASSEMBLY__
985 
986 extern int direct_gbpages;
987 void init_mem_mapping(void);
988 void early_alloc_pgt_buf(void);
989 extern void memblock_find_dma_reserve(void);
990 
991 #ifdef CONFIG_X86_64
992 /* Realmode trampoline initialization. */
993 extern pgd_t trampoline_pgd_entry;
994 static inline void __meminit init_trampoline_default(void)
995 {
996         /* Default trampoline pgd value */
997         trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)];
998 }
999 # ifdef CONFIG_RANDOMIZE_MEMORY
1000 void __meminit init_trampoline(void);
1001 # else
1002 #  define init_trampoline init_trampoline_default
1003 # endif
1004 #else
1005 static inline void init_trampoline(void) { }
1006 #endif
1007 
1008 /* local pte updates need not use xchg for locking */
1009 static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
1010 {
1011         pte_t res = *ptep;
1012 
1013         /* Pure native function needs no input for mm, addr */
1014         native_pte_clear(NULL, 0, ptep);
1015         return res;
1016 }
1017 
1018 static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp)
1019 {
1020         pmd_t res = *pmdp;
1021 
1022         native_pmd_clear(pmdp);
1023         return res;
1024 }
1025 
1026 static inline pud_t native_local_pudp_get_and_clear(pud_t *pudp)
1027 {
1028         pud_t res = *pudp;
1029 
1030         native_pud_clear(pudp);
1031         return res;
1032 }
1033 
1034 static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr,
1035                                      pte_t *ptep , pte_t pte)
1036 {
1037         native_set_pte(ptep, pte);
1038 }
1039 
1040 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
1041                               pmd_t *pmdp, pmd_t pmd)
1042 {
1043         native_set_pmd(pmdp, pmd);
1044 }
1045 
1046 static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
1047                               pud_t *pudp, pud_t pud)
1048 {
1049         native_set_pud(pudp, pud);
1050 }
1051 
1052 /*
1053  * We only update the dirty/accessed state if we set
1054  * the dirty bit by hand in the kernel, since the hardware
1055  * will do the accessed bit for us, and we don't want to
1056  * race with other CPU's that might be updating the dirty
1057  * bit at the same time.
1058  */
1059 struct vm_area_struct;
1060 
1061 #define  __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
1062 extern int ptep_set_access_flags(struct vm_area_struct *vma,
1063                                  unsigned long address, pte_t *ptep,
1064                                  pte_t entry, int dirty);
1065 
1066 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
1067 extern int ptep_test_and_clear_young(struct vm_area_struct *vma,
1068                                      unsigned long addr, pte_t *ptep);
1069 
1070 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
1071 extern int ptep_clear_flush_young(struct vm_area_struct *vma,
1072                                   unsigned long address, pte_t *ptep);
1073 
1074 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
1075 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
1076                                        pte_t *ptep)
1077 {
1078         pte_t pte = native_ptep_get_and_clear(ptep);
1079         return pte;
1080 }
1081 
1082 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
1083 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
1084                                             unsigned long addr, pte_t *ptep,
1085                                             int full)
1086 {
1087         pte_t pte;
1088         if (full) {
1089                 /*
1090                  * Full address destruction in progress; paravirt does not
1091                  * care about updates and native needs no locking
1092                  */
1093                 pte = native_local_ptep_get_and_clear(ptep);
1094         } else {
1095                 pte = ptep_get_and_clear(mm, addr, ptep);
1096         }
1097         return pte;
1098 }
1099 
1100 #define __HAVE_ARCH_PTEP_SET_WRPROTECT
1101 static inline void ptep_set_wrprotect(struct mm_struct *mm,
1102                                       unsigned long addr, pte_t *ptep)
1103 {
1104         clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
1105 }
1106 
1107 #define flush_tlb_fix_spurious_fault(vma, address) do { } while (0)
1108 
1109 #define mk_pmd(page, pgprot)   pfn_pmd(page_to_pfn(page), (pgprot))
1110 
1111 #define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
1112 extern int pmdp_set_access_flags(struct vm_area_struct *vma,
1113                                  unsigned long address, pmd_t *pmdp,
1114                                  pmd_t entry, int dirty);
1115 extern int pudp_set_access_flags(struct vm_area_struct *vma,
1116                                  unsigned long address, pud_t *pudp,
1117                                  pud_t entry, int dirty);
1118 
1119 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
1120 extern int pmdp_test_and_clear_young(struct vm_area_struct *vma,
1121                                      unsigned long addr, pmd_t *pmdp);
1122 extern int pudp_test_and_clear_young(struct vm_area_struct *vma,
1123                                      unsigned long addr, pud_t *pudp);
1124 
1125 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
1126 extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
1127                                   unsigned long address, pmd_t *pmdp);
1128 
1129 
1130 #define pmd_write pmd_write
1131 static inline int pmd_write(pmd_t pmd)
1132 {
1133         return pmd_flags(pmd) & _PAGE_RW;
1134 }
1135 
1136 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1137 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1138                                        pmd_t *pmdp)
1139 {
1140         return native_pmdp_get_and_clear(pmdp);
1141 }
1142 
1143 #define __HAVE_ARCH_PUDP_HUGE_GET_AND_CLEAR
1144 static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm,
1145                                         unsigned long addr, pud_t *pudp)
1146 {
1147         return native_pudp_get_and_clear(pudp);
1148 }
1149 
1150 #define __HAVE_ARCH_PMDP_SET_WRPROTECT
1151 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1152                                       unsigned long addr, pmd_t *pmdp)
1153 {
1154         clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
1155 }
1156 
1157 #define pud_write pud_write
1158 static inline int pud_write(pud_t pud)
1159 {
1160         return pud_flags(pud) & _PAGE_RW;
1161 }
1162 
1163 #ifndef pmdp_establish
1164 #define pmdp_establish pmdp_establish
1165 static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
1166                 unsigned long address, pmd_t *pmdp, pmd_t pmd)
1167 {
1168         if (IS_ENABLED(CONFIG_SMP)) {
1169                 return xchg(pmdp, pmd);
1170         } else {
1171                 pmd_t old = *pmdp;
1172                 *pmdp = pmd;
1173                 return old;
1174         }
1175 }
1176 #endif
1177 
1178 /*
1179  * clone_pgd_range(pgd_t *dst, pgd_t *src, int count);
1180  *
1181  *  dst - pointer to pgd range anwhere on a pgd page
1182  *  src - ""
1183  *  count - the number of pgds to copy.
1184  *
1185  * dst and src can be on the same page, but the range must not overlap,
1186  * and must not cross a page boundary.
1187  */
1188 static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
1189 {
1190         memcpy(dst, src, count * sizeof(pgd_t));
1191 #ifdef CONFIG_PAGE_TABLE_ISOLATION
1192         if (!static_cpu_has(X86_FEATURE_PTI))
1193                 return;
1194         /* Clone the user space pgd as well */
1195         memcpy(kernel_to_user_pgdp(dst), kernel_to_user_pgdp(src),
1196                count * sizeof(pgd_t));
1197 #endif
1198 }
1199 
1200 #define PTE_SHIFT ilog2(PTRS_PER_PTE)
1201 static inline int page_level_shift(enum pg_level level)
1202 {
1203         return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT;
1204 }
1205 static inline unsigned long page_level_size(enum pg_level level)
1206 {
1207         return 1UL << page_level_shift(level);
1208 }
1209 static inline unsigned long page_level_mask(enum pg_level level)
1210 {
1211         return ~(page_level_size(level) - 1);
1212 }
1213 
1214 /*
1215  * The x86 doesn't have any external MMU info: the kernel page
1216  * tables contain all the necessary information.
1217  */
1218 static inline void update_mmu_cache(struct vm_area_struct *vma,
1219                 unsigned long addr, pte_t *ptep)
1220 {
1221 }
1222 static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
1223                 unsigned long addr, pmd_t *pmd)
1224 {
1225 }
1226 static inline void update_mmu_cache_pud(struct vm_area_struct *vma,
1227                 unsigned long addr, pud_t *pud)
1228 {
1229 }
1230 
1231 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1232 static inline pte_t pte_swp_mksoft_dirty(pte_t pte)
1233 {
1234         return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1235 }
1236 
1237 static inline int pte_swp_soft_dirty(pte_t pte)
1238 {
1239         return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY;
1240 }
1241 
1242 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte)
1243 {
1244         return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY);
1245 }
1246 
1247 #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1248 static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd)
1249 {
1250         return pmd_set_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1251 }
1252 
1253 static inline int pmd_swp_soft_dirty(pmd_t pmd)
1254 {
1255         return pmd_flags(pmd) & _PAGE_SWP_SOFT_DIRTY;
1256 }
1257 
1258 static inline pmd_t pmd_swp_clear_soft_dirty(pmd_t pmd)
1259 {
1260         return pmd_clear_flags(pmd, _PAGE_SWP_SOFT_DIRTY);
1261 }
1262 #endif
1263 #endif
1264 
1265 #define PKRU_AD_BIT 0x1
1266 #define PKRU_WD_BIT 0x2
1267 #define PKRU_BITS_PER_PKEY 2
1268 
1269 static inline bool __pkru_allows_read(u32 pkru, u16 pkey)
1270 {
1271         int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1272         return !(pkru & (PKRU_AD_BIT << pkru_pkey_bits));
1273 }
1274 
1275 static inline bool __pkru_allows_write(u32 pkru, u16 pkey)
1276 {
1277         int pkru_pkey_bits = pkey * PKRU_BITS_PER_PKEY;
1278         /*
1279          * Access-disable disables writes too so we need to check
1280          * both bits here.
1281          */
1282         return !(pkru & ((PKRU_AD_BIT|PKRU_WD_BIT) << pkru_pkey_bits));
1283 }
1284 
1285 static inline u16 pte_flags_pkey(unsigned long pte_flags)
1286 {
1287 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
1288         /* ifdef to avoid doing 59-bit shift on 32-bit values */
1289         return (pte_flags & _PAGE_PKEY_MASK) >> _PAGE_BIT_PKEY_BIT0;
1290 #else
1291         return 0;
1292 #endif
1293 }
1294 
1295 static inline bool __pkru_allows_pkey(u16 pkey, bool write)
1296 {
1297         u32 pkru = read_pkru();
1298 
1299         if (!__pkru_allows_read(pkru, pkey))
1300                 return false;
1301         if (write && !__pkru_allows_write(pkru, pkey))
1302                 return false;
1303 
1304         return true;
1305 }
1306 
1307 /*
1308  * 'pteval' can come from a PTE, PMD or PUD.  We only check
1309  * _PAGE_PRESENT, _PAGE_USER, and _PAGE_RW in here which are the
1310  * same value on all 3 types.
1311  */
1312 static inline bool __pte_access_permitted(unsigned long pteval, bool write)
1313 {
1314         unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1315 
1316         if (write)
1317                 need_pte_bits |= _PAGE_RW;
1318 
1319         if ((pteval & need_pte_bits) != need_pte_bits)
1320                 return 0;
1321 
1322         return __pkru_allows_pkey(pte_flags_pkey(pteval), write);
1323 }
1324 
1325 #define pte_access_permitted pte_access_permitted
1326 static inline bool pte_access_permitted(pte_t pte, bool write)
1327 {
1328         return __pte_access_permitted(pte_val(pte), write);
1329 }
1330 
1331 #define pmd_access_permitted pmd_access_permitted
1332 static inline bool pmd_access_permitted(pmd_t pmd, bool write)
1333 {
1334         return __pte_access_permitted(pmd_val(pmd), write);
1335 }
1336 
1337 #define pud_access_permitted pud_access_permitted
1338 static inline bool pud_access_permitted(pud_t pud, bool write)
1339 {
1340         return __pte_access_permitted(pud_val(pud), write);
1341 }
1342 
1343 #define __HAVE_ARCH_PFN_MODIFY_ALLOWED 1
1344 extern bool pfn_modify_allowed(unsigned long pfn, pgprot_t prot);
1345 
1346 static inline bool arch_has_pfn_modify_check(void)
1347 {
1348         return boot_cpu_has_bug(X86_BUG_L1TF);
1349 }
1350 
1351 #include <asm-generic/pgtable.h>
1352 #endif  /* __ASSEMBLY__ */
1353 
1354 #endif /* _ASM_X86_PGTABLE_H */
1355 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp