~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/vm/khugepaged.c

Version: ~ [ linux-6.6-rc1 ] ~ [ linux-6.5.2 ] ~ [ linux-6.4.15 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.52 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.131 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.194 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.256 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.294 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.325 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #define _GNU_SOURCE
  2 #include <fcntl.h>
  3 #include <limits.h>
  4 #include <signal.h>
  5 #include <stdio.h>
  6 #include <stdlib.h>
  7 #include <stdbool.h>
  8 #include <string.h>
  9 #include <unistd.h>
 10 
 11 #include <sys/mman.h>
 12 #include <sys/wait.h>
 13 
 14 #ifndef MADV_PAGEOUT
 15 #define MADV_PAGEOUT 21
 16 #endif
 17 
 18 #define BASE_ADDR ((void *)(1UL << 30))
 19 static unsigned long hpage_pmd_size;
 20 static unsigned long page_size;
 21 static int hpage_pmd_nr;
 22 
 23 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
 24 #define PID_SMAPS "/proc/self/smaps"
 25 
 26 enum thp_enabled {
 27         THP_ALWAYS,
 28         THP_MADVISE,
 29         THP_NEVER,
 30 };
 31 
 32 static const char *thp_enabled_strings[] = {
 33         "always",
 34         "madvise",
 35         "never",
 36         NULL
 37 };
 38 
 39 enum thp_defrag {
 40         THP_DEFRAG_ALWAYS,
 41         THP_DEFRAG_DEFER,
 42         THP_DEFRAG_DEFER_MADVISE,
 43         THP_DEFRAG_MADVISE,
 44         THP_DEFRAG_NEVER,
 45 };
 46 
 47 static const char *thp_defrag_strings[] = {
 48         "always",
 49         "defer",
 50         "defer+madvise",
 51         "madvise",
 52         "never",
 53         NULL
 54 };
 55 
 56 enum shmem_enabled {
 57         SHMEM_ALWAYS,
 58         SHMEM_WITHIN_SIZE,
 59         SHMEM_ADVISE,
 60         SHMEM_NEVER,
 61         SHMEM_DENY,
 62         SHMEM_FORCE,
 63 };
 64 
 65 static const char *shmem_enabled_strings[] = {
 66         "always",
 67         "within_size",
 68         "advise",
 69         "never",
 70         "deny",
 71         "force",
 72         NULL
 73 };
 74 
 75 struct khugepaged_settings {
 76         bool defrag;
 77         unsigned int alloc_sleep_millisecs;
 78         unsigned int scan_sleep_millisecs;
 79         unsigned int max_ptes_none;
 80         unsigned int max_ptes_swap;
 81         unsigned int max_ptes_shared;
 82         unsigned long pages_to_scan;
 83 };
 84 
 85 struct settings {
 86         enum thp_enabled thp_enabled;
 87         enum thp_defrag thp_defrag;
 88         enum shmem_enabled shmem_enabled;
 89         bool use_zero_page;
 90         struct khugepaged_settings khugepaged;
 91 };
 92 
 93 static struct settings default_settings = {
 94         .thp_enabled = THP_MADVISE,
 95         .thp_defrag = THP_DEFRAG_ALWAYS,
 96         .shmem_enabled = SHMEM_NEVER,
 97         .use_zero_page = 0,
 98         .khugepaged = {
 99                 .defrag = 1,
100                 .alloc_sleep_millisecs = 10,
101                 .scan_sleep_millisecs = 10,
102         },
103 };
104 
105 static struct settings saved_settings;
106 static bool skip_settings_restore;
107 
108 static int exit_status;
109 
110 static void success(const char *msg)
111 {
112         printf(" \e[32m%s\e[0m\n", msg);
113 }
114 
115 static void fail(const char *msg)
116 {
117         printf(" \e[31m%s\e[0m\n", msg);
118         exit_status++;
119 }
120 
121 static int read_file(const char *path, char *buf, size_t buflen)
122 {
123         int fd;
124         ssize_t numread;
125 
126         fd = open(path, O_RDONLY);
127         if (fd == -1)
128                 return 0;
129 
130         numread = read(fd, buf, buflen - 1);
131         if (numread < 1) {
132                 close(fd);
133                 return 0;
134         }
135 
136         buf[numread] = '\0';
137         close(fd);
138 
139         return (unsigned int) numread;
140 }
141 
142 static int write_file(const char *path, const char *buf, size_t buflen)
143 {
144         int fd;
145         ssize_t numwritten;
146 
147         fd = open(path, O_WRONLY);
148         if (fd == -1)
149                 return 0;
150 
151         numwritten = write(fd, buf, buflen - 1);
152         close(fd);
153         if (numwritten < 1)
154                 return 0;
155 
156         return (unsigned int) numwritten;
157 }
158 
159 static int read_string(const char *name, const char *strings[])
160 {
161         char path[PATH_MAX];
162         char buf[256];
163         char *c;
164         int ret;
165 
166         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
167         if (ret >= PATH_MAX) {
168                 printf("%s: Pathname is too long\n", __func__);
169                 exit(EXIT_FAILURE);
170         }
171 
172         if (!read_file(path, buf, sizeof(buf))) {
173                 perror(path);
174                 exit(EXIT_FAILURE);
175         }
176 
177         c = strchr(buf, '[');
178         if (!c) {
179                 printf("%s: Parse failure\n", __func__);
180                 exit(EXIT_FAILURE);
181         }
182 
183         c++;
184         memmove(buf, c, sizeof(buf) - (c - buf));
185 
186         c = strchr(buf, ']');
187         if (!c) {
188                 printf("%s: Parse failure\n", __func__);
189                 exit(EXIT_FAILURE);
190         }
191         *c = '\0';
192 
193         ret = 0;
194         while (strings[ret]) {
195                 if (!strcmp(strings[ret], buf))
196                         return ret;
197                 ret++;
198         }
199 
200         printf("Failed to parse %s\n", name);
201         exit(EXIT_FAILURE);
202 }
203 
204 static void write_string(const char *name, const char *val)
205 {
206         char path[PATH_MAX];
207         int ret;
208 
209         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
210         if (ret >= PATH_MAX) {
211                 printf("%s: Pathname is too long\n", __func__);
212                 exit(EXIT_FAILURE);
213         }
214 
215         if (!write_file(path, val, strlen(val) + 1)) {
216                 perror(path);
217                 exit(EXIT_FAILURE);
218         }
219 }
220 
221 static const unsigned long read_num(const char *name)
222 {
223         char path[PATH_MAX];
224         char buf[21];
225         int ret;
226 
227         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
228         if (ret >= PATH_MAX) {
229                 printf("%s: Pathname is too long\n", __func__);
230                 exit(EXIT_FAILURE);
231         }
232 
233         ret = read_file(path, buf, sizeof(buf));
234         if (ret < 0) {
235                 perror("read_file(read_num)");
236                 exit(EXIT_FAILURE);
237         }
238 
239         return strtoul(buf, NULL, 10);
240 }
241 
242 static void write_num(const char *name, unsigned long num)
243 {
244         char path[PATH_MAX];
245         char buf[21];
246         int ret;
247 
248         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
249         if (ret >= PATH_MAX) {
250                 printf("%s: Pathname is too long\n", __func__);
251                 exit(EXIT_FAILURE);
252         }
253 
254         sprintf(buf, "%ld", num);
255         if (!write_file(path, buf, strlen(buf) + 1)) {
256                 perror(path);
257                 exit(EXIT_FAILURE);
258         }
259 }
260 
261 static void write_settings(struct settings *settings)
262 {
263         struct khugepaged_settings *khugepaged = &settings->khugepaged;
264 
265         write_string("enabled", thp_enabled_strings[settings->thp_enabled]);
266         write_string("defrag", thp_defrag_strings[settings->thp_defrag]);
267         write_string("shmem_enabled",
268                         shmem_enabled_strings[settings->shmem_enabled]);
269         write_num("use_zero_page", settings->use_zero_page);
270 
271         write_num("khugepaged/defrag", khugepaged->defrag);
272         write_num("khugepaged/alloc_sleep_millisecs",
273                         khugepaged->alloc_sleep_millisecs);
274         write_num("khugepaged/scan_sleep_millisecs",
275                         khugepaged->scan_sleep_millisecs);
276         write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);
277         write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
278         write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
279         write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
280 }
281 
282 static void restore_settings(int sig)
283 {
284         if (skip_settings_restore)
285                 goto out;
286 
287         printf("Restore THP and khugepaged settings...");
288         write_settings(&saved_settings);
289         success("OK");
290         if (sig)
291                 exit(EXIT_FAILURE);
292 out:
293         exit(exit_status);
294 }
295 
296 static void save_settings(void)
297 {
298         printf("Save THP and khugepaged settings...");
299         saved_settings = (struct settings) {
300                 .thp_enabled = read_string("enabled", thp_enabled_strings),
301                 .thp_defrag = read_string("defrag", thp_defrag_strings),
302                 .shmem_enabled =
303                         read_string("shmem_enabled", shmem_enabled_strings),
304                 .use_zero_page = read_num("use_zero_page"),
305         };
306         saved_settings.khugepaged = (struct khugepaged_settings) {
307                 .defrag = read_num("khugepaged/defrag"),
308                 .alloc_sleep_millisecs =
309                         read_num("khugepaged/alloc_sleep_millisecs"),
310                 .scan_sleep_millisecs =
311                         read_num("khugepaged/scan_sleep_millisecs"),
312                 .max_ptes_none = read_num("khugepaged/max_ptes_none"),
313                 .max_ptes_swap = read_num("khugepaged/max_ptes_swap"),
314                 .max_ptes_shared = read_num("khugepaged/max_ptes_shared"),
315                 .pages_to_scan = read_num("khugepaged/pages_to_scan"),
316         };
317         success("OK");
318 
319         signal(SIGTERM, restore_settings);
320         signal(SIGINT, restore_settings);
321         signal(SIGHUP, restore_settings);
322         signal(SIGQUIT, restore_settings);
323 }
324 
325 static void adjust_settings(void)
326 {
327 
328         printf("Adjust settings...");
329         write_settings(&default_settings);
330         success("OK");
331 }
332 
333 #define MAX_LINE_LENGTH 500
334 
335 static bool check_for_pattern(FILE *fp, char *pattern, char *buf)
336 {
337         while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) {
338                 if (!strncmp(buf, pattern, strlen(pattern)))
339                         return true;
340         }
341         return false;
342 }
343 
344 static bool check_huge(void *addr)
345 {
346         bool thp = false;
347         int ret;
348         FILE *fp;
349         char buffer[MAX_LINE_LENGTH];
350         char addr_pattern[MAX_LINE_LENGTH];
351 
352         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
353                        (unsigned long) addr);
354         if (ret >= MAX_LINE_LENGTH) {
355                 printf("%s: Pattern is too long\n", __func__);
356                 exit(EXIT_FAILURE);
357         }
358 
359 
360         fp = fopen(PID_SMAPS, "r");
361         if (!fp) {
362                 printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
363                 exit(EXIT_FAILURE);
364         }
365         if (!check_for_pattern(fp, addr_pattern, buffer))
366                 goto err_out;
367 
368         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB",
369                        hpage_pmd_size >> 10);
370         if (ret >= MAX_LINE_LENGTH) {
371                 printf("%s: Pattern is too long\n", __func__);
372                 exit(EXIT_FAILURE);
373         }
374         /*
375          * Fetch the AnonHugePages: in the same block and check whether it got
376          * the expected number of hugeepages next.
377          */
378         if (!check_for_pattern(fp, "AnonHugePages:", buffer))
379                 goto err_out;
380 
381         if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
382                 goto err_out;
383 
384         thp = true;
385 err_out:
386         fclose(fp);
387         return thp;
388 }
389 
390 
391 static bool check_swap(void *addr, unsigned long size)
392 {
393         bool swap = false;
394         int ret;
395         FILE *fp;
396         char buffer[MAX_LINE_LENGTH];
397         char addr_pattern[MAX_LINE_LENGTH];
398 
399         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
400                        (unsigned long) addr);
401         if (ret >= MAX_LINE_LENGTH) {
402                 printf("%s: Pattern is too long\n", __func__);
403                 exit(EXIT_FAILURE);
404         }
405 
406 
407         fp = fopen(PID_SMAPS, "r");
408         if (!fp) {
409                 printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
410                 exit(EXIT_FAILURE);
411         }
412         if (!check_for_pattern(fp, addr_pattern, buffer))
413                 goto err_out;
414 
415         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB",
416                        size >> 10);
417         if (ret >= MAX_LINE_LENGTH) {
418                 printf("%s: Pattern is too long\n", __func__);
419                 exit(EXIT_FAILURE);
420         }
421         /*
422          * Fetch the Swap: in the same block and check whether it got
423          * the expected number of hugeepages next.
424          */
425         if (!check_for_pattern(fp, "Swap:", buffer))
426                 goto err_out;
427 
428         if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
429                 goto err_out;
430 
431         swap = true;
432 err_out:
433         fclose(fp);
434         return swap;
435 }
436 
437 static void *alloc_mapping(void)
438 {
439         void *p;
440 
441         p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE,
442                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
443         if (p != BASE_ADDR) {
444                 printf("Failed to allocate VMA at %p\n", BASE_ADDR);
445                 exit(EXIT_FAILURE);
446         }
447 
448         return p;
449 }
450 
451 static void fill_memory(int *p, unsigned long start, unsigned long end)
452 {
453         int i;
454 
455         for (i = start / page_size; i < end / page_size; i++)
456                 p[i * page_size / sizeof(*p)] = i + 0xdead0000;
457 }
458 
459 static void validate_memory(int *p, unsigned long start, unsigned long end)
460 {
461         int i;
462 
463         for (i = start / page_size; i < end / page_size; i++) {
464                 if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) {
465                         printf("Page %d is corrupted: %#x\n",
466                                         i, p[i * page_size / sizeof(*p)]);
467                         exit(EXIT_FAILURE);
468                 }
469         }
470 }
471 
472 #define TICK 500000
473 static bool wait_for_scan(const char *msg, char *p)
474 {
475         int full_scans;
476         int timeout = 6; /* 3 seconds */
477 
478         /* Sanity check */
479         if (check_huge(p)) {
480                 printf("Unexpected huge page\n");
481                 exit(EXIT_FAILURE);
482         }
483 
484         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
485 
486         /* Wait until the second full_scan completed */
487         full_scans = read_num("khugepaged/full_scans") + 2;
488 
489         printf("%s...", msg);
490         while (timeout--) {
491                 if (check_huge(p))
492                         break;
493                 if (read_num("khugepaged/full_scans") >= full_scans)
494                         break;
495                 printf(".");
496                 usleep(TICK);
497         }
498 
499         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
500 
501         return timeout == -1;
502 }
503 
504 static void alloc_at_fault(void)
505 {
506         struct settings settings = default_settings;
507         char *p;
508 
509         settings.thp_enabled = THP_ALWAYS;
510         write_settings(&settings);
511 
512         p = alloc_mapping();
513         *p = 1;
514         printf("Allocate huge page on fault...");
515         if (check_huge(p))
516                 success("OK");
517         else
518                 fail("Fail");
519 
520         write_settings(&default_settings);
521 
522         madvise(p, page_size, MADV_DONTNEED);
523         printf("Split huge PMD on MADV_DONTNEED...");
524         if (!check_huge(p))
525                 success("OK");
526         else
527                 fail("Fail");
528         munmap(p, hpage_pmd_size);
529 }
530 
531 static void collapse_full(void)
532 {
533         void *p;
534 
535         p = alloc_mapping();
536         fill_memory(p, 0, hpage_pmd_size);
537         if (wait_for_scan("Collapse fully populated PTE table", p))
538                 fail("Timeout");
539         else if (check_huge(p))
540                 success("OK");
541         else
542                 fail("Fail");
543         validate_memory(p, 0, hpage_pmd_size);
544         munmap(p, hpage_pmd_size);
545 }
546 
547 static void collapse_empty(void)
548 {
549         void *p;
550 
551         p = alloc_mapping();
552         if (wait_for_scan("Do not collapse empty PTE table", p))
553                 fail("Timeout");
554         else if (check_huge(p))
555                 fail("Fail");
556         else
557                 success("OK");
558         munmap(p, hpage_pmd_size);
559 }
560 
561 static void collapse_single_pte_entry(void)
562 {
563         void *p;
564 
565         p = alloc_mapping();
566         fill_memory(p, 0, page_size);
567         if (wait_for_scan("Collapse PTE table with single PTE entry present", p))
568                 fail("Timeout");
569         else if (check_huge(p))
570                 success("OK");
571         else
572                 fail("Fail");
573         validate_memory(p, 0, page_size);
574         munmap(p, hpage_pmd_size);
575 }
576 
577 static void collapse_max_ptes_none(void)
578 {
579         int max_ptes_none = hpage_pmd_nr / 2;
580         struct settings settings = default_settings;
581         void *p;
582 
583         settings.khugepaged.max_ptes_none = max_ptes_none;
584         write_settings(&settings);
585 
586         p = alloc_mapping();
587 
588         fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
589         if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p))
590                 fail("Timeout");
591         else if (check_huge(p))
592                 fail("Fail");
593         else
594                 success("OK");
595         validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
596 
597         fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
598         if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p))
599                 fail("Timeout");
600         else if (check_huge(p))
601                 success("OK");
602         else
603                 fail("Fail");
604         validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
605 
606         munmap(p, hpage_pmd_size);
607         write_settings(&default_settings);
608 }
609 
610 static void collapse_swapin_single_pte(void)
611 {
612         void *p;
613         p = alloc_mapping();
614         fill_memory(p, 0, hpage_pmd_size);
615 
616         printf("Swapout one page...");
617         if (madvise(p, page_size, MADV_PAGEOUT)) {
618                 perror("madvise(MADV_PAGEOUT)");
619                 exit(EXIT_FAILURE);
620         }
621         if (check_swap(p, page_size)) {
622                 success("OK");
623         } else {
624                 fail("Fail");
625                 goto out;
626         }
627 
628         if (wait_for_scan("Collapse with swapping in single PTE entry", p))
629                 fail("Timeout");
630         else if (check_huge(p))
631                 success("OK");
632         else
633                 fail("Fail");
634         validate_memory(p, 0, hpage_pmd_size);
635 out:
636         munmap(p, hpage_pmd_size);
637 }
638 
639 static void collapse_max_ptes_swap(void)
640 {
641         int max_ptes_swap = read_num("khugepaged/max_ptes_swap");
642         void *p;
643 
644         p = alloc_mapping();
645 
646         fill_memory(p, 0, hpage_pmd_size);
647         printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr);
648         if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) {
649                 perror("madvise(MADV_PAGEOUT)");
650                 exit(EXIT_FAILURE);
651         }
652         if (check_swap(p, (max_ptes_swap + 1) * page_size)) {
653                 success("OK");
654         } else {
655                 fail("Fail");
656                 goto out;
657         }
658 
659         if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p))
660                 fail("Timeout");
661         else if (check_huge(p))
662                 fail("Fail");
663         else
664                 success("OK");
665         validate_memory(p, 0, hpage_pmd_size);
666 
667         fill_memory(p, 0, hpage_pmd_size);
668         printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr);
669         if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) {
670                 perror("madvise(MADV_PAGEOUT)");
671                 exit(EXIT_FAILURE);
672         }
673         if (check_swap(p, max_ptes_swap * page_size)) {
674                 success("OK");
675         } else {
676                 fail("Fail");
677                 goto out;
678         }
679 
680         if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p))
681                 fail("Timeout");
682         else if (check_huge(p))
683                 success("OK");
684         else
685                 fail("Fail");
686         validate_memory(p, 0, hpage_pmd_size);
687 out:
688         munmap(p, hpage_pmd_size);
689 }
690 
691 static void collapse_single_pte_entry_compound(void)
692 {
693         void *p;
694 
695         p = alloc_mapping();
696 
697         printf("Allocate huge page...");
698         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
699         fill_memory(p, 0, hpage_pmd_size);
700         if (check_huge(p))
701                 success("OK");
702         else
703                 fail("Fail");
704         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
705 
706         printf("Split huge page leaving single PTE mapping compound page...");
707         madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED);
708         if (!check_huge(p))
709                 success("OK");
710         else
711                 fail("Fail");
712 
713         if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p))
714                 fail("Timeout");
715         else if (check_huge(p))
716                 success("OK");
717         else
718                 fail("Fail");
719         validate_memory(p, 0, page_size);
720         munmap(p, hpage_pmd_size);
721 }
722 
723 static void collapse_full_of_compound(void)
724 {
725         void *p;
726 
727         p = alloc_mapping();
728 
729         printf("Allocate huge page...");
730         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
731         fill_memory(p, 0, hpage_pmd_size);
732         if (check_huge(p))
733                 success("OK");
734         else
735                 fail("Fail");
736 
737         printf("Split huge page leaving single PTE page table full of compound pages...");
738         madvise(p, page_size, MADV_NOHUGEPAGE);
739         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
740         if (!check_huge(p))
741                 success("OK");
742         else
743                 fail("Fail");
744 
745         if (wait_for_scan("Collapse PTE table full of compound pages", p))
746                 fail("Timeout");
747         else if (check_huge(p))
748                 success("OK");
749         else
750                 fail("Fail");
751         validate_memory(p, 0, hpage_pmd_size);
752         munmap(p, hpage_pmd_size);
753 }
754 
755 static void collapse_compound_extreme(void)
756 {
757         void *p;
758         int i;
759 
760         p = alloc_mapping();
761         for (i = 0; i < hpage_pmd_nr; i++) {
762                 printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...",
763                                 i + 1, hpage_pmd_nr);
764 
765                 madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE);
766                 fill_memory(BASE_ADDR, 0, hpage_pmd_size);
767                 if (!check_huge(BASE_ADDR)) {
768                         printf("Failed to allocate huge page\n");
769                         exit(EXIT_FAILURE);
770                 }
771                 madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE);
772 
773                 p = mremap(BASE_ADDR - i * page_size,
774                                 i * page_size + hpage_pmd_size,
775                                 (i + 1) * page_size,
776                                 MREMAP_MAYMOVE | MREMAP_FIXED,
777                                 BASE_ADDR + 2 * hpage_pmd_size);
778                 if (p == MAP_FAILED) {
779                         perror("mremap+unmap");
780                         exit(EXIT_FAILURE);
781                 }
782 
783                 p = mremap(BASE_ADDR + 2 * hpage_pmd_size,
784                                 (i + 1) * page_size,
785                                 (i + 1) * page_size + hpage_pmd_size,
786                                 MREMAP_MAYMOVE | MREMAP_FIXED,
787                                 BASE_ADDR - (i + 1) * page_size);
788                 if (p == MAP_FAILED) {
789                         perror("mremap+alloc");
790                         exit(EXIT_FAILURE);
791                 }
792         }
793 
794         munmap(BASE_ADDR, hpage_pmd_size);
795         fill_memory(p, 0, hpage_pmd_size);
796         if (!check_huge(p))
797                 success("OK");
798         else
799                 fail("Fail");
800 
801         if (wait_for_scan("Collapse PTE table full of different compound pages", p))
802                 fail("Timeout");
803         else if (check_huge(p))
804                 success("OK");
805         else
806                 fail("Fail");
807 
808         validate_memory(p, 0, hpage_pmd_size);
809         munmap(p, hpage_pmd_size);
810 }
811 
812 static void collapse_fork(void)
813 {
814         int wstatus;
815         void *p;
816 
817         p = alloc_mapping();
818 
819         printf("Allocate small page...");
820         fill_memory(p, 0, page_size);
821         if (!check_huge(p))
822                 success("OK");
823         else
824                 fail("Fail");
825 
826         printf("Share small page over fork()...");
827         if (!fork()) {
828                 /* Do not touch settings on child exit */
829                 skip_settings_restore = true;
830                 exit_status = 0;
831 
832                 if (!check_huge(p))
833                         success("OK");
834                 else
835                         fail("Fail");
836 
837                 fill_memory(p, page_size, 2 * page_size);
838 
839                 if (wait_for_scan("Collapse PTE table with single page shared with parent process", p))
840                         fail("Timeout");
841                 else if (check_huge(p))
842                         success("OK");
843                 else
844                         fail("Fail");
845 
846                 validate_memory(p, 0, page_size);
847                 munmap(p, hpage_pmd_size);
848                 exit(exit_status);
849         }
850 
851         wait(&wstatus);
852         exit_status += WEXITSTATUS(wstatus);
853 
854         printf("Check if parent still has small page...");
855         if (!check_huge(p))
856                 success("OK");
857         else
858                 fail("Fail");
859         validate_memory(p, 0, page_size);
860         munmap(p, hpage_pmd_size);
861 }
862 
863 static void collapse_fork_compound(void)
864 {
865         int wstatus;
866         void *p;
867 
868         p = alloc_mapping();
869 
870         printf("Allocate huge page...");
871         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
872         fill_memory(p, 0, hpage_pmd_size);
873         if (check_huge(p))
874                 success("OK");
875         else
876                 fail("Fail");
877 
878         printf("Share huge page over fork()...");
879         if (!fork()) {
880                 /* Do not touch settings on child exit */
881                 skip_settings_restore = true;
882                 exit_status = 0;
883 
884                 if (check_huge(p))
885                         success("OK");
886                 else
887                         fail("Fail");
888 
889                 printf("Split huge page PMD in child process...");
890                 madvise(p, page_size, MADV_NOHUGEPAGE);
891                 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
892                 if (!check_huge(p))
893                         success("OK");
894                 else
895                         fail("Fail");
896                 fill_memory(p, 0, page_size);
897 
898                 write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1);
899                 if (wait_for_scan("Collapse PTE table full of compound pages in child", p))
900                         fail("Timeout");
901                 else if (check_huge(p))
902                         success("OK");
903                 else
904                         fail("Fail");
905                 write_num("khugepaged/max_ptes_shared",
906                                 default_settings.khugepaged.max_ptes_shared);
907 
908                 validate_memory(p, 0, hpage_pmd_size);
909                 munmap(p, hpage_pmd_size);
910                 exit(exit_status);
911         }
912 
913         wait(&wstatus);
914         exit_status += WEXITSTATUS(wstatus);
915 
916         printf("Check if parent still has huge page...");
917         if (check_huge(p))
918                 success("OK");
919         else
920                 fail("Fail");
921         validate_memory(p, 0, hpage_pmd_size);
922         munmap(p, hpage_pmd_size);
923 }
924 
925 static void collapse_max_ptes_shared()
926 {
927         int max_ptes_shared = read_num("khugepaged/max_ptes_shared");
928         int wstatus;
929         void *p;
930 
931         p = alloc_mapping();
932 
933         printf("Allocate huge page...");
934         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
935         fill_memory(p, 0, hpage_pmd_size);
936         if (check_huge(p))
937                 success("OK");
938         else
939                 fail("Fail");
940 
941         printf("Share huge page over fork()...");
942         if (!fork()) {
943                 /* Do not touch settings on child exit */
944                 skip_settings_restore = true;
945                 exit_status = 0;
946 
947                 if (check_huge(p))
948                         success("OK");
949                 else
950                         fail("Fail");
951 
952                 printf("Trigger CoW on page %d of %d...",
953                                 hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr);
954                 fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
955                 if (!check_huge(p))
956                         success("OK");
957                 else
958                         fail("Fail");
959 
960                 if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p))
961                         fail("Timeout");
962                 else if (!check_huge(p))
963                         success("OK");
964                 else
965                         fail("Fail");
966 
967                 printf("Trigger CoW on page %d of %d...",
968                                 hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
969                 fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size);
970                 if (!check_huge(p))
971                         success("OK");
972                 else
973                         fail("Fail");
974 
975 
976                 if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p))
977                         fail("Timeout");
978                 else if (check_huge(p))
979                         success("OK");
980                 else
981                         fail("Fail");
982 
983                 validate_memory(p, 0, hpage_pmd_size);
984                 munmap(p, hpage_pmd_size);
985                 exit(exit_status);
986         }
987 
988         wait(&wstatus);
989         exit_status += WEXITSTATUS(wstatus);
990 
991         printf("Check if parent still has huge page...");
992         if (check_huge(p))
993                 success("OK");
994         else
995                 fail("Fail");
996         validate_memory(p, 0, hpage_pmd_size);
997         munmap(p, hpage_pmd_size);
998 }
999 
1000 int main(void)
1001 {
1002         setbuf(stdout, NULL);
1003 
1004         page_size = getpagesize();
1005         hpage_pmd_size = read_num("hpage_pmd_size");
1006         hpage_pmd_nr = hpage_pmd_size / page_size;
1007 
1008         default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1;
1009         default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8;
1010         default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2;
1011         default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8;
1012 
1013         save_settings();
1014         adjust_settings();
1015 
1016         alloc_at_fault();
1017         collapse_full();
1018         collapse_empty();
1019         collapse_single_pte_entry();
1020         collapse_max_ptes_none();
1021         collapse_swapin_single_pte();
1022         collapse_max_ptes_swap();
1023         collapse_single_pte_entry_compound();
1024         collapse_full_of_compound();
1025         collapse_compound_extreme();
1026         collapse_fork();
1027         collapse_fork_compound();
1028         collapse_max_ptes_shared();
1029 
1030         restore_settings(0);
1031 }
1032 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp