~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/mm/hugetlbpage.c

Version: ~ [ linux-5.15-rc6 ] ~ [ linux-5.14.14 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.75 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.155 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.213 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.252 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.287 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.289 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  *  IBM System z Huge TLB Page Support for Kernel.
  4  *
  5  *    Copyright IBM Corp. 2007,2016
  6  *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
  7  */
  8 
  9 #define KMSG_COMPONENT "hugetlb"
 10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 11 
 12 #include <linux/mm.h>
 13 #include <linux/hugetlb.h>
 14 
 15 /*
 16  * If the bit selected by single-bit bitmask "a" is set within "x", move
 17  * it to the position indicated by single-bit bitmask "b".
 18  */
 19 #define move_set_bit(x, a, b)   (((x) & (a)) >> ilog2(a) << ilog2(b))
 20 
 21 static inline unsigned long __pte_to_rste(pte_t pte)
 22 {
 23         unsigned long rste;
 24 
 25         /*
 26          * Convert encoding               pte bits      pmd / pud bits
 27          *                              lIR.uswrdy.p    dy..R...I...wr
 28          * empty                        010.000000.0 -> 00..0...1...00
 29          * prot-none, clean, old        111.000000.1 -> 00..1...1...00
 30          * prot-none, clean, young      111.000001.1 -> 01..1...1...00
 31          * prot-none, dirty, old        111.000010.1 -> 10..1...1...00
 32          * prot-none, dirty, young      111.000011.1 -> 11..1...1...00
 33          * read-only, clean, old        111.000100.1 -> 00..1...1...01
 34          * read-only, clean, young      101.000101.1 -> 01..1...0...01
 35          * read-only, dirty, old        111.000110.1 -> 10..1...1...01
 36          * read-only, dirty, young      101.000111.1 -> 11..1...0...01
 37          * read-write, clean, old       111.001100.1 -> 00..1...1...11
 38          * read-write, clean, young     101.001101.1 -> 01..1...0...11
 39          * read-write, dirty, old       110.001110.1 -> 10..0...1...11
 40          * read-write, dirty, young     100.001111.1 -> 11..0...0...11
 41          * HW-bits: R read-only, I invalid
 42          * SW-bits: p present, y young, d dirty, r read, w write, s special,
 43          *          u unused, l large
 44          */
 45         if (pte_present(pte)) {
 46                 rste = pte_val(pte) & PAGE_MASK;
 47                 rste |= move_set_bit(pte_val(pte), _PAGE_READ,
 48                                      _SEGMENT_ENTRY_READ);
 49                 rste |= move_set_bit(pte_val(pte), _PAGE_WRITE,
 50                                      _SEGMENT_ENTRY_WRITE);
 51                 rste |= move_set_bit(pte_val(pte), _PAGE_INVALID,
 52                                      _SEGMENT_ENTRY_INVALID);
 53                 rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT,
 54                                      _SEGMENT_ENTRY_PROTECT);
 55                 rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY,
 56                                      _SEGMENT_ENTRY_DIRTY);
 57                 rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG,
 58                                      _SEGMENT_ENTRY_YOUNG);
 59 #ifdef CONFIG_MEM_SOFT_DIRTY
 60                 rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
 61                                      _SEGMENT_ENTRY_SOFT_DIRTY);
 62 #endif
 63                 rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC,
 64                                      _SEGMENT_ENTRY_NOEXEC);
 65         } else
 66                 rste = _SEGMENT_ENTRY_EMPTY;
 67         return rste;
 68 }
 69 
 70 static inline pte_t __rste_to_pte(unsigned long rste)
 71 {
 72         int present;
 73         pte_t pte;
 74 
 75         if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
 76                 present = pud_present(__pud(rste));
 77         else
 78                 present = pmd_present(__pmd(rste));
 79 
 80         /*
 81          * Convert encoding             pmd / pud bits      pte bits
 82          *                              dy..R...I...wr    lIR.uswrdy.p
 83          * empty                        00..0...1...00 -> 010.000000.0
 84          * prot-none, clean, old        00..1...1...00 -> 111.000000.1
 85          * prot-none, clean, young      01..1...1...00 -> 111.000001.1
 86          * prot-none, dirty, old        10..1...1...00 -> 111.000010.1
 87          * prot-none, dirty, young      11..1...1...00 -> 111.000011.1
 88          * read-only, clean, old        00..1...1...01 -> 111.000100.1
 89          * read-only, clean, young      01..1...0...01 -> 101.000101.1
 90          * read-only, dirty, old        10..1...1...01 -> 111.000110.1
 91          * read-only, dirty, young      11..1...0...01 -> 101.000111.1
 92          * read-write, clean, old       00..1...1...11 -> 111.001100.1
 93          * read-write, clean, young     01..1...0...11 -> 101.001101.1
 94          * read-write, dirty, old       10..0...1...11 -> 110.001110.1
 95          * read-write, dirty, young     11..0...0...11 -> 100.001111.1
 96          * HW-bits: R read-only, I invalid
 97          * SW-bits: p present, y young, d dirty, r read, w write, s special,
 98          *          u unused, l large
 99          */
100         if (present) {
101                 pte_val(pte) = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
102                 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
103                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_READ,
104                                              _PAGE_READ);
105                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE,
106                                              _PAGE_WRITE);
107                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID,
108                                              _PAGE_INVALID);
109                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT,
110                                              _PAGE_PROTECT);
111                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY,
112                                              _PAGE_DIRTY);
113                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG,
114                                              _PAGE_YOUNG);
115 #ifdef CONFIG_MEM_SOFT_DIRTY
116                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY,
117                                              _PAGE_DIRTY);
118 #endif
119                 pte_val(pte) |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC,
120                                              _PAGE_NOEXEC);
121         } else
122                 pte_val(pte) = _PAGE_INVALID;
123         return pte;
124 }
125 
126 static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
127 {
128         struct page *page;
129         unsigned long size, paddr;
130 
131         if (!mm_uses_skeys(mm) ||
132             rste & _SEGMENT_ENTRY_INVALID)
133                 return;
134 
135         if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
136                 page = pud_page(__pud(rste));
137                 size = PUD_SIZE;
138                 paddr = rste & PUD_MASK;
139         } else {
140                 page = pmd_page(__pmd(rste));
141                 size = PMD_SIZE;
142                 paddr = rste & PMD_MASK;
143         }
144 
145         if (!test_and_set_bit(PG_arch_1, &page->flags))
146                 __storage_key_init_range(paddr, paddr + size - 1);
147 }
148 
149 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
150                      pte_t *ptep, pte_t pte)
151 {
152         unsigned long rste;
153 
154         rste = __pte_to_rste(pte);
155         if (!MACHINE_HAS_NX)
156                 rste &= ~_SEGMENT_ENTRY_NOEXEC;
157 
158         /* Set correct table type for 2G hugepages */
159         if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
160                 rste |= _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE;
161         else
162                 rste |= _SEGMENT_ENTRY_LARGE;
163         clear_huge_pte_skeys(mm, rste);
164         pte_val(*ptep) = rste;
165 }
166 
167 pte_t huge_ptep_get(pte_t *ptep)
168 {
169         return __rste_to_pte(pte_val(*ptep));
170 }
171 
172 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
173                               unsigned long addr, pte_t *ptep)
174 {
175         pte_t pte = huge_ptep_get(ptep);
176         pmd_t *pmdp = (pmd_t *) ptep;
177         pud_t *pudp = (pud_t *) ptep;
178 
179         if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
180                 pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
181         else
182                 pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
183         return pte;
184 }
185 
186 pte_t *huge_pte_alloc(struct mm_struct *mm,
187                         unsigned long addr, unsigned long sz)
188 {
189         pgd_t *pgdp;
190         p4d_t *p4dp;
191         pud_t *pudp;
192         pmd_t *pmdp = NULL;
193 
194         pgdp = pgd_offset(mm, addr);
195         p4dp = p4d_alloc(mm, pgdp, addr);
196         if (p4dp) {
197                 pudp = pud_alloc(mm, p4dp, addr);
198                 if (pudp) {
199                         if (sz == PUD_SIZE)
200                                 return (pte_t *) pudp;
201                         else if (sz == PMD_SIZE)
202                                 pmdp = pmd_alloc(mm, pudp, addr);
203                 }
204         }
205         return (pte_t *) pmdp;
206 }
207 
208 pte_t *huge_pte_offset(struct mm_struct *mm,
209                        unsigned long addr, unsigned long sz)
210 {
211         pgd_t *pgdp;
212         p4d_t *p4dp;
213         pud_t *pudp;
214         pmd_t *pmdp = NULL;
215 
216         pgdp = pgd_offset(mm, addr);
217         if (pgd_present(*pgdp)) {
218                 p4dp = p4d_offset(pgdp, addr);
219                 if (p4d_present(*p4dp)) {
220                         pudp = pud_offset(p4dp, addr);
221                         if (pud_present(*pudp)) {
222                                 if (pud_large(*pudp))
223                                         return (pte_t *) pudp;
224                                 pmdp = pmd_offset(pudp, addr);
225                         }
226                 }
227         }
228         return (pte_t *) pmdp;
229 }
230 
231 int pmd_huge(pmd_t pmd)
232 {
233         return pmd_large(pmd);
234 }
235 
236 int pud_huge(pud_t pud)
237 {
238         return pud_large(pud);
239 }
240 
241 struct page *
242 follow_huge_pud(struct mm_struct *mm, unsigned long address,
243                 pud_t *pud, int flags)
244 {
245         if (flags & FOLL_GET)
246                 return NULL;
247 
248         return pud_page(*pud) + ((address & ~PUD_MASK) >> PAGE_SHIFT);
249 }
250 
251 static __init int setup_hugepagesz(char *opt)
252 {
253         unsigned long size;
254         char *string = opt;
255 
256         size = memparse(opt, &opt);
257         if (MACHINE_HAS_EDAT1 && size == PMD_SIZE) {
258                 hugetlb_add_hstate(PMD_SHIFT - PAGE_SHIFT);
259         } else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE) {
260                 hugetlb_add_hstate(PUD_SHIFT - PAGE_SHIFT);
261         } else {
262                 hugetlb_bad_size();
263                 pr_err("hugepagesz= specifies an unsupported page size %s\n",
264                         string);
265                 return 0;
266         }
267         return 1;
268 }
269 __setup("hugepagesz=", setup_hugepagesz);
270 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp