~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/mm/vmem.c

Version: ~ [ linux-5.8-rc5 ] ~ [ linux-5.7.8 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.51 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.132 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.188 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.230 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.230 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *    Copyright IBM Corp. 2006
  3  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
  4  */
  5 
  6 #include <linux/bootmem.h>
  7 #include <linux/pfn.h>
  8 #include <linux/mm.h>
  9 #include <linux/module.h>
 10 #include <linux/list.h>
 11 #include <linux/hugetlb.h>
 12 #include <linux/slab.h>
 13 #include <linux/memblock.h>
 14 #include <asm/pgalloc.h>
 15 #include <asm/pgtable.h>
 16 #include <asm/setup.h>
 17 #include <asm/tlbflush.h>
 18 #include <asm/sections.h>
 19 
 20 static DEFINE_MUTEX(vmem_mutex);
 21 
 22 struct memory_segment {
 23         struct list_head list;
 24         unsigned long start;
 25         unsigned long size;
 26 };
 27 
 28 static LIST_HEAD(mem_segs);
 29 
 30 static void __ref *vmem_alloc_pages(unsigned int order)
 31 {
 32         if (slab_is_available())
 33                 return (void *)__get_free_pages(GFP_KERNEL, order);
 34         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
 35 }
 36 
 37 static inline pud_t *vmem_pud_alloc(void)
 38 {
 39         pud_t *pud = NULL;
 40 
 41         pud = vmem_alloc_pages(2);
 42         if (!pud)
 43                 return NULL;
 44         clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
 45         return pud;
 46 }
 47 
 48 static inline pmd_t *vmem_pmd_alloc(void)
 49 {
 50         pmd_t *pmd = NULL;
 51 
 52         pmd = vmem_alloc_pages(2);
 53         if (!pmd)
 54                 return NULL;
 55         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
 56         return pmd;
 57 }
 58 
 59 static pte_t __ref *vmem_pte_alloc(void)
 60 {
 61         pte_t *pte;
 62 
 63         if (slab_is_available())
 64                 pte = (pte_t *) page_table_alloc(&init_mm);
 65         else
 66                 pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t),
 67                                           PTRS_PER_PTE * sizeof(pte_t));
 68         if (!pte)
 69                 return NULL;
 70         clear_table((unsigned long *) pte, _PAGE_INVALID,
 71                     PTRS_PER_PTE * sizeof(pte_t));
 72         return pte;
 73 }
 74 
 75 /*
 76  * Add a physical memory range to the 1:1 mapping.
 77  */
 78 static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
 79 {
 80         unsigned long end = start + size;
 81         unsigned long address = start;
 82         pgd_t *pg_dir;
 83         pud_t *pu_dir;
 84         pmd_t *pm_dir;
 85         pte_t *pt_dir;
 86         int ret = -ENOMEM;
 87 
 88         while (address < end) {
 89                 pg_dir = pgd_offset_k(address);
 90                 if (pgd_none(*pg_dir)) {
 91                         pu_dir = vmem_pud_alloc();
 92                         if (!pu_dir)
 93                                 goto out;
 94                         pgd_populate(&init_mm, pg_dir, pu_dir);
 95                 }
 96                 pu_dir = pud_offset(pg_dir, address);
 97                 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
 98                     !(address & ~PUD_MASK) && (address + PUD_SIZE <= end) &&
 99                      !debug_pagealloc_enabled()) {
100                         pud_val(*pu_dir) = __pa(address) |
101                                 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
102                                 (ro ? _REGION_ENTRY_PROTECT : 0);
103                         address += PUD_SIZE;
104                         continue;
105                 }
106                 if (pud_none(*pu_dir)) {
107                         pm_dir = vmem_pmd_alloc();
108                         if (!pm_dir)
109                                 goto out;
110                         pud_populate(&init_mm, pu_dir, pm_dir);
111                 }
112                 pm_dir = pmd_offset(pu_dir, address);
113                 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
114                     !(address & ~PMD_MASK) && (address + PMD_SIZE <= end) &&
115                     !debug_pagealloc_enabled()) {
116                         pmd_val(*pm_dir) = __pa(address) |
117                                 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
118                                 _SEGMENT_ENTRY_YOUNG |
119                                 (ro ? _SEGMENT_ENTRY_PROTECT : 0);
120                         address += PMD_SIZE;
121                         continue;
122                 }
123                 if (pmd_none(*pm_dir)) {
124                         pt_dir = vmem_pte_alloc();
125                         if (!pt_dir)
126                                 goto out;
127                         pmd_populate(&init_mm, pm_dir, pt_dir);
128                 }
129 
130                 pt_dir = pte_offset_kernel(pm_dir, address);
131                 pte_val(*pt_dir) = __pa(address) |
132                         pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
133                 address += PAGE_SIZE;
134         }
135         ret = 0;
136 out:
137         return ret;
138 }
139 
140 /*
141  * Remove a physical memory range from the 1:1 mapping.
142  * Currently only invalidates page table entries.
143  */
144 static void vmem_remove_range(unsigned long start, unsigned long size)
145 {
146         unsigned long end = start + size;
147         unsigned long address = start;
148         pgd_t *pg_dir;
149         pud_t *pu_dir;
150         pmd_t *pm_dir;
151         pte_t *pt_dir;
152         pte_t  pte;
153 
154         pte_val(pte) = _PAGE_INVALID;
155         while (address < end) {
156                 pg_dir = pgd_offset_k(address);
157                 if (pgd_none(*pg_dir)) {
158                         address += PGDIR_SIZE;
159                         continue;
160                 }
161                 pu_dir = pud_offset(pg_dir, address);
162                 if (pud_none(*pu_dir)) {
163                         address += PUD_SIZE;
164                         continue;
165                 }
166                 if (pud_large(*pu_dir)) {
167                         pud_clear(pu_dir);
168                         address += PUD_SIZE;
169                         continue;
170                 }
171                 pm_dir = pmd_offset(pu_dir, address);
172                 if (pmd_none(*pm_dir)) {
173                         address += PMD_SIZE;
174                         continue;
175                 }
176                 if (pmd_large(*pm_dir)) {
177                         pmd_clear(pm_dir);
178                         address += PMD_SIZE;
179                         continue;
180                 }
181                 pt_dir = pte_offset_kernel(pm_dir, address);
182                 *pt_dir = pte;
183                 address += PAGE_SIZE;
184         }
185         flush_tlb_kernel_range(start, end);
186 }
187 
188 /*
189  * Add a backed mem_map array to the virtual mem_map array.
190  */
191 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
192 {
193         unsigned long address = start;
194         pgd_t *pg_dir;
195         pud_t *pu_dir;
196         pmd_t *pm_dir;
197         pte_t *pt_dir;
198         int ret = -ENOMEM;
199 
200         for (address = start; address < end;) {
201                 pg_dir = pgd_offset_k(address);
202                 if (pgd_none(*pg_dir)) {
203                         pu_dir = vmem_pud_alloc();
204                         if (!pu_dir)
205                                 goto out;
206                         pgd_populate(&init_mm, pg_dir, pu_dir);
207                 }
208 
209                 pu_dir = pud_offset(pg_dir, address);
210                 if (pud_none(*pu_dir)) {
211                         pm_dir = vmem_pmd_alloc();
212                         if (!pm_dir)
213                                 goto out;
214                         pud_populate(&init_mm, pu_dir, pm_dir);
215                 }
216 
217                 pm_dir = pmd_offset(pu_dir, address);
218                 if (pmd_none(*pm_dir)) {
219                         /* Use 1MB frames for vmemmap if available. We always
220                          * use large frames even if they are only partially
221                          * used.
222                          * Otherwise we would have also page tables since
223                          * vmemmap_populate gets called for each section
224                          * separately. */
225                         if (MACHINE_HAS_EDAT1) {
226                                 void *new_page;
227 
228                                 new_page = vmemmap_alloc_block(PMD_SIZE, node);
229                                 if (!new_page)
230                                         goto out;
231                                 pmd_val(*pm_dir) = __pa(new_page) |
232                                         _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE;
233                                 address = (address + PMD_SIZE) & PMD_MASK;
234                                 continue;
235                         }
236                         pt_dir = vmem_pte_alloc();
237                         if (!pt_dir)
238                                 goto out;
239                         pmd_populate(&init_mm, pm_dir, pt_dir);
240                 } else if (pmd_large(*pm_dir)) {
241                         address = (address + PMD_SIZE) & PMD_MASK;
242                         continue;
243                 }
244 
245                 pt_dir = pte_offset_kernel(pm_dir, address);
246                 if (pte_none(*pt_dir)) {
247                         void *new_page;
248 
249                         new_page = vmemmap_alloc_block(PAGE_SIZE, node);
250                         if (!new_page)
251                                 goto out;
252                         pte_val(*pt_dir) =
253                                 __pa(new_page) | pgprot_val(PAGE_KERNEL);
254                 }
255                 address += PAGE_SIZE;
256         }
257         ret = 0;
258 out:
259         return ret;
260 }
261 
262 void vmemmap_free(unsigned long start, unsigned long end)
263 {
264 }
265 
266 /*
267  * Add memory segment to the segment list if it doesn't overlap with
268  * an already present segment.
269  */
270 static int insert_memory_segment(struct memory_segment *seg)
271 {
272         struct memory_segment *tmp;
273 
274         if (seg->start + seg->size > VMEM_MAX_PHYS ||
275             seg->start + seg->size < seg->start)
276                 return -ERANGE;
277 
278         list_for_each_entry(tmp, &mem_segs, list) {
279                 if (seg->start >= tmp->start + tmp->size)
280                         continue;
281                 if (seg->start + seg->size <= tmp->start)
282                         continue;
283                 return -ENOSPC;
284         }
285         list_add(&seg->list, &mem_segs);
286         return 0;
287 }
288 
289 /*
290  * Remove memory segment from the segment list.
291  */
292 static void remove_memory_segment(struct memory_segment *seg)
293 {
294         list_del(&seg->list);
295 }
296 
297 static void __remove_shared_memory(struct memory_segment *seg)
298 {
299         remove_memory_segment(seg);
300         vmem_remove_range(seg->start, seg->size);
301 }
302 
303 int vmem_remove_mapping(unsigned long start, unsigned long size)
304 {
305         struct memory_segment *seg;
306         int ret;
307 
308         mutex_lock(&vmem_mutex);
309 
310         ret = -ENOENT;
311         list_for_each_entry(seg, &mem_segs, list) {
312                 if (seg->start == start && seg->size == size)
313                         break;
314         }
315 
316         if (seg->start != start || seg->size != size)
317                 goto out;
318 
319         ret = 0;
320         __remove_shared_memory(seg);
321         kfree(seg);
322 out:
323         mutex_unlock(&vmem_mutex);
324         return ret;
325 }
326 
327 int vmem_add_mapping(unsigned long start, unsigned long size)
328 {
329         struct memory_segment *seg;
330         int ret;
331 
332         mutex_lock(&vmem_mutex);
333         ret = -ENOMEM;
334         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
335         if (!seg)
336                 goto out;
337         seg->start = start;
338         seg->size = size;
339 
340         ret = insert_memory_segment(seg);
341         if (ret)
342                 goto out_free;
343 
344         ret = vmem_add_mem(start, size, 0);
345         if (ret)
346                 goto out_remove;
347         goto out;
348 
349 out_remove:
350         __remove_shared_memory(seg);
351 out_free:
352         kfree(seg);
353 out:
354         mutex_unlock(&vmem_mutex);
355         return ret;
356 }
357 
358 /*
359  * map whole physical memory to virtual memory (identity mapping)
360  * we reserve enough space in the vmalloc area for vmemmap to hotplug
361  * additional memory segments.
362  */
363 void __init vmem_map_init(void)
364 {
365         unsigned long ro_start, ro_end;
366         struct memblock_region *reg;
367         phys_addr_t start, end;
368 
369         ro_start = PFN_ALIGN((unsigned long)&_stext);
370         ro_end = (unsigned long)&_eshared & PAGE_MASK;
371         for_each_memblock(memory, reg) {
372                 start = reg->base;
373                 end = reg->base + reg->size;
374                 if (start >= ro_end || end <= ro_start)
375                         vmem_add_mem(start, end - start, 0);
376                 else if (start >= ro_start && end <= ro_end)
377                         vmem_add_mem(start, end - start, 1);
378                 else if (start >= ro_start) {
379                         vmem_add_mem(start, ro_end - start, 1);
380                         vmem_add_mem(ro_end, end - ro_end, 0);
381                 } else if (end < ro_end) {
382                         vmem_add_mem(start, ro_start - start, 0);
383                         vmem_add_mem(ro_start, end - ro_start, 1);
384                 } else {
385                         vmem_add_mem(start, ro_start - start, 0);
386                         vmem_add_mem(ro_start, ro_end - ro_start, 1);
387                         vmem_add_mem(ro_end, end - ro_end, 0);
388                 }
389         }
390 }
391 
392 /*
393  * Convert memblock.memory  to a memory segment list so there is a single
394  * list that contains all memory segments.
395  */
396 static int __init vmem_convert_memory_chunk(void)
397 {
398         struct memblock_region *reg;
399         struct memory_segment *seg;
400 
401         mutex_lock(&vmem_mutex);
402         for_each_memblock(memory, reg) {
403                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
404                 if (!seg)
405                         panic("Out of memory...\n");
406                 seg->start = reg->base;
407                 seg->size = reg->size;
408                 insert_memory_segment(seg);
409         }
410         mutex_unlock(&vmem_mutex);
411         return 0;
412 }
413 
414 core_initcall(vmem_convert_memory_chunk);
415 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp