~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/tools/testing/selftests/vm/khugepaged.c

Version: ~ [ linux-6.6-rc1 ] ~ [ linux-6.5.2 ] ~ [ linux-6.4.15 ] ~ [ linux-6.3.13 ] ~ [ linux-6.2.16 ] ~ [ linux-6.1.52 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.131 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.194 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.256 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.294 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.325 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.4.302 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #define _GNU_SOURCE
  2 #include <fcntl.h>
  3 #include <limits.h>
  4 #include <signal.h>
  5 #include <stdio.h>
  6 #include <stdlib.h>
  7 #include <stdbool.h>
  8 #include <string.h>
  9 #include <unistd.h>
 10 
 11 #include <sys/mman.h>
 12 #include <sys/wait.h>
 13 
 14 #ifndef MADV_PAGEOUT
 15 #define MADV_PAGEOUT 21
 16 #endif
 17 
 18 #define BASE_ADDR ((void *)(1UL << 30))
 19 static unsigned long hpage_pmd_size;
 20 static unsigned long page_size;
 21 static int hpage_pmd_nr;
 22 
 23 #define THP_SYSFS "/sys/kernel/mm/transparent_hugepage/"
 24 #define PID_SMAPS "/proc/self/smaps"
 25 
 26 enum thp_enabled {
 27         THP_ALWAYS,
 28         THP_MADVISE,
 29         THP_NEVER,
 30 };
 31 
 32 static const char *thp_enabled_strings[] = {
 33         "always",
 34         "madvise",
 35         "never",
 36         NULL
 37 };
 38 
 39 enum thp_defrag {
 40         THP_DEFRAG_ALWAYS,
 41         THP_DEFRAG_DEFER,
 42         THP_DEFRAG_DEFER_MADVISE,
 43         THP_DEFRAG_MADVISE,
 44         THP_DEFRAG_NEVER,
 45 };
 46 
 47 static const char *thp_defrag_strings[] = {
 48         "always",
 49         "defer",
 50         "defer+madvise",
 51         "madvise",
 52         "never",
 53         NULL
 54 };
 55 
 56 enum shmem_enabled {
 57         SHMEM_ALWAYS,
 58         SHMEM_WITHIN_SIZE,
 59         SHMEM_ADVISE,
 60         SHMEM_NEVER,
 61         SHMEM_DENY,
 62         SHMEM_FORCE,
 63 };
 64 
 65 static const char *shmem_enabled_strings[] = {
 66         "always",
 67         "within_size",
 68         "advise",
 69         "never",
 70         "deny",
 71         "force",
 72         NULL
 73 };
 74 
 75 struct khugepaged_settings {
 76         bool defrag;
 77         unsigned int alloc_sleep_millisecs;
 78         unsigned int scan_sleep_millisecs;
 79         unsigned int max_ptes_none;
 80         unsigned int max_ptes_swap;
 81         unsigned int max_ptes_shared;
 82         unsigned long pages_to_scan;
 83 };
 84 
 85 struct settings {
 86         enum thp_enabled thp_enabled;
 87         enum thp_defrag thp_defrag;
 88         enum shmem_enabled shmem_enabled;
 89         bool debug_cow;
 90         bool use_zero_page;
 91         struct khugepaged_settings khugepaged;
 92 };
 93 
 94 static struct settings default_settings = {
 95         .thp_enabled = THP_MADVISE,
 96         .thp_defrag = THP_DEFRAG_ALWAYS,
 97         .shmem_enabled = SHMEM_NEVER,
 98         .debug_cow = 0,
 99         .use_zero_page = 0,
100         .khugepaged = {
101                 .defrag = 1,
102                 .alloc_sleep_millisecs = 10,
103                 .scan_sleep_millisecs = 10,
104         },
105 };
106 
107 static struct settings saved_settings;
108 static bool skip_settings_restore;
109 
110 static int exit_status;
111 
112 static void success(const char *msg)
113 {
114         printf(" \e[32m%s\e[0m\n", msg);
115 }
116 
117 static void fail(const char *msg)
118 {
119         printf(" \e[31m%s\e[0m\n", msg);
120         exit_status++;
121 }
122 
123 static int read_file(const char *path, char *buf, size_t buflen)
124 {
125         int fd;
126         ssize_t numread;
127 
128         fd = open(path, O_RDONLY);
129         if (fd == -1)
130                 return 0;
131 
132         numread = read(fd, buf, buflen - 1);
133         if (numread < 1) {
134                 close(fd);
135                 return 0;
136         }
137 
138         buf[numread] = '\0';
139         close(fd);
140 
141         return (unsigned int) numread;
142 }
143 
144 static int write_file(const char *path, const char *buf, size_t buflen)
145 {
146         int fd;
147         ssize_t numwritten;
148 
149         fd = open(path, O_WRONLY);
150         if (fd == -1)
151                 return 0;
152 
153         numwritten = write(fd, buf, buflen - 1);
154         close(fd);
155         if (numwritten < 1)
156                 return 0;
157 
158         return (unsigned int) numwritten;
159 }
160 
161 static int read_string(const char *name, const char *strings[])
162 {
163         char path[PATH_MAX];
164         char buf[256];
165         char *c;
166         int ret;
167 
168         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
169         if (ret >= PATH_MAX) {
170                 printf("%s: Pathname is too long\n", __func__);
171                 exit(EXIT_FAILURE);
172         }
173 
174         if (!read_file(path, buf, sizeof(buf))) {
175                 perror(path);
176                 exit(EXIT_FAILURE);
177         }
178 
179         c = strchr(buf, '[');
180         if (!c) {
181                 printf("%s: Parse failure\n", __func__);
182                 exit(EXIT_FAILURE);
183         }
184 
185         c++;
186         memmove(buf, c, sizeof(buf) - (c - buf));
187 
188         c = strchr(buf, ']');
189         if (!c) {
190                 printf("%s: Parse failure\n", __func__);
191                 exit(EXIT_FAILURE);
192         }
193         *c = '\0';
194 
195         ret = 0;
196         while (strings[ret]) {
197                 if (!strcmp(strings[ret], buf))
198                         return ret;
199                 ret++;
200         }
201 
202         printf("Failed to parse %s\n", name);
203         exit(EXIT_FAILURE);
204 }
205 
206 static void write_string(const char *name, const char *val)
207 {
208         char path[PATH_MAX];
209         int ret;
210 
211         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
212         if (ret >= PATH_MAX) {
213                 printf("%s: Pathname is too long\n", __func__);
214                 exit(EXIT_FAILURE);
215         }
216 
217         if (!write_file(path, val, strlen(val) + 1)) {
218                 perror(path);
219                 exit(EXIT_FAILURE);
220         }
221 }
222 
223 static const unsigned long read_num(const char *name)
224 {
225         char path[PATH_MAX];
226         char buf[21];
227         int ret;
228 
229         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
230         if (ret >= PATH_MAX) {
231                 printf("%s: Pathname is too long\n", __func__);
232                 exit(EXIT_FAILURE);
233         }
234 
235         ret = read_file(path, buf, sizeof(buf));
236         if (ret < 0) {
237                 perror("read_file(read_num)");
238                 exit(EXIT_FAILURE);
239         }
240 
241         return strtoul(buf, NULL, 10);
242 }
243 
244 static void write_num(const char *name, unsigned long num)
245 {
246         char path[PATH_MAX];
247         char buf[21];
248         int ret;
249 
250         ret = snprintf(path, PATH_MAX, THP_SYSFS "%s", name);
251         if (ret >= PATH_MAX) {
252                 printf("%s: Pathname is too long\n", __func__);
253                 exit(EXIT_FAILURE);
254         }
255 
256         sprintf(buf, "%ld", num);
257         if (!write_file(path, buf, strlen(buf) + 1)) {
258                 perror(path);
259                 exit(EXIT_FAILURE);
260         }
261 }
262 
263 static void write_settings(struct settings *settings)
264 {
265         struct khugepaged_settings *khugepaged = &settings->khugepaged;
266 
267         write_string("enabled", thp_enabled_strings[settings->thp_enabled]);
268         write_string("defrag", thp_defrag_strings[settings->thp_defrag]);
269         write_string("shmem_enabled",
270                         shmem_enabled_strings[settings->shmem_enabled]);
271         write_num("debug_cow", settings->debug_cow);
272         write_num("use_zero_page", settings->use_zero_page);
273 
274         write_num("khugepaged/defrag", khugepaged->defrag);
275         write_num("khugepaged/alloc_sleep_millisecs",
276                         khugepaged->alloc_sleep_millisecs);
277         write_num("khugepaged/scan_sleep_millisecs",
278                         khugepaged->scan_sleep_millisecs);
279         write_num("khugepaged/max_ptes_none", khugepaged->max_ptes_none);
280         write_num("khugepaged/max_ptes_swap", khugepaged->max_ptes_swap);
281         write_num("khugepaged/max_ptes_shared", khugepaged->max_ptes_shared);
282         write_num("khugepaged/pages_to_scan", khugepaged->pages_to_scan);
283 }
284 
285 static void restore_settings(int sig)
286 {
287         if (skip_settings_restore)
288                 goto out;
289 
290         printf("Restore THP and khugepaged settings...");
291         write_settings(&saved_settings);
292         success("OK");
293         if (sig)
294                 exit(EXIT_FAILURE);
295 out:
296         exit(exit_status);
297 }
298 
299 static void save_settings(void)
300 {
301         printf("Save THP and khugepaged settings...");
302         saved_settings = (struct settings) {
303                 .thp_enabled = read_string("enabled", thp_enabled_strings),
304                 .thp_defrag = read_string("defrag", thp_defrag_strings),
305                 .shmem_enabled =
306                         read_string("shmem_enabled", shmem_enabled_strings),
307                 .debug_cow = read_num("debug_cow"),
308                 .use_zero_page = read_num("use_zero_page"),
309         };
310         saved_settings.khugepaged = (struct khugepaged_settings) {
311                 .defrag = read_num("khugepaged/defrag"),
312                 .alloc_sleep_millisecs =
313                         read_num("khugepaged/alloc_sleep_millisecs"),
314                 .scan_sleep_millisecs =
315                         read_num("khugepaged/scan_sleep_millisecs"),
316                 .max_ptes_none = read_num("khugepaged/max_ptes_none"),
317                 .max_ptes_swap = read_num("khugepaged/max_ptes_swap"),
318                 .max_ptes_shared = read_num("khugepaged/max_ptes_shared"),
319                 .pages_to_scan = read_num("khugepaged/pages_to_scan"),
320         };
321         success("OK");
322 
323         signal(SIGTERM, restore_settings);
324         signal(SIGINT, restore_settings);
325         signal(SIGHUP, restore_settings);
326         signal(SIGQUIT, restore_settings);
327 }
328 
329 static void adjust_settings(void)
330 {
331 
332         printf("Adjust settings...");
333         write_settings(&default_settings);
334         success("OK");
335 }
336 
337 #define MAX_LINE_LENGTH 500
338 
339 static bool check_for_pattern(FILE *fp, char *pattern, char *buf)
340 {
341         while (fgets(buf, MAX_LINE_LENGTH, fp) != NULL) {
342                 if (!strncmp(buf, pattern, strlen(pattern)))
343                         return true;
344         }
345         return false;
346 }
347 
348 static bool check_huge(void *addr)
349 {
350         bool thp = false;
351         int ret;
352         FILE *fp;
353         char buffer[MAX_LINE_LENGTH];
354         char addr_pattern[MAX_LINE_LENGTH];
355 
356         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
357                        (unsigned long) addr);
358         if (ret >= MAX_LINE_LENGTH) {
359                 printf("%s: Pattern is too long\n", __func__);
360                 exit(EXIT_FAILURE);
361         }
362 
363 
364         fp = fopen(PID_SMAPS, "r");
365         if (!fp) {
366                 printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
367                 exit(EXIT_FAILURE);
368         }
369         if (!check_for_pattern(fp, addr_pattern, buffer))
370                 goto err_out;
371 
372         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "AnonHugePages:%10ld kB",
373                        hpage_pmd_size >> 10);
374         if (ret >= MAX_LINE_LENGTH) {
375                 printf("%s: Pattern is too long\n", __func__);
376                 exit(EXIT_FAILURE);
377         }
378         /*
379          * Fetch the AnonHugePages: in the same block and check whether it got
380          * the expected number of hugeepages next.
381          */
382         if (!check_for_pattern(fp, "AnonHugePages:", buffer))
383                 goto err_out;
384 
385         if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
386                 goto err_out;
387 
388         thp = true;
389 err_out:
390         fclose(fp);
391         return thp;
392 }
393 
394 
395 static bool check_swap(void *addr, unsigned long size)
396 {
397         bool swap = false;
398         int ret;
399         FILE *fp;
400         char buffer[MAX_LINE_LENGTH];
401         char addr_pattern[MAX_LINE_LENGTH];
402 
403         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "%08lx-",
404                        (unsigned long) addr);
405         if (ret >= MAX_LINE_LENGTH) {
406                 printf("%s: Pattern is too long\n", __func__);
407                 exit(EXIT_FAILURE);
408         }
409 
410 
411         fp = fopen(PID_SMAPS, "r");
412         if (!fp) {
413                 printf("%s: Failed to open file %s\n", __func__, PID_SMAPS);
414                 exit(EXIT_FAILURE);
415         }
416         if (!check_for_pattern(fp, addr_pattern, buffer))
417                 goto err_out;
418 
419         ret = snprintf(addr_pattern, MAX_LINE_LENGTH, "Swap:%19ld kB",
420                        size >> 10);
421         if (ret >= MAX_LINE_LENGTH) {
422                 printf("%s: Pattern is too long\n", __func__);
423                 exit(EXIT_FAILURE);
424         }
425         /*
426          * Fetch the Swap: in the same block and check whether it got
427          * the expected number of hugeepages next.
428          */
429         if (!check_for_pattern(fp, "Swap:", buffer))
430                 goto err_out;
431 
432         if (strncmp(buffer, addr_pattern, strlen(addr_pattern)))
433                 goto err_out;
434 
435         swap = true;
436 err_out:
437         fclose(fp);
438         return swap;
439 }
440 
441 static void *alloc_mapping(void)
442 {
443         void *p;
444 
445         p = mmap(BASE_ADDR, hpage_pmd_size, PROT_READ | PROT_WRITE,
446                         MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
447         if (p != BASE_ADDR) {
448                 printf("Failed to allocate VMA at %p\n", BASE_ADDR);
449                 exit(EXIT_FAILURE);
450         }
451 
452         return p;
453 }
454 
455 static void fill_memory(int *p, unsigned long start, unsigned long end)
456 {
457         int i;
458 
459         for (i = start / page_size; i < end / page_size; i++)
460                 p[i * page_size / sizeof(*p)] = i + 0xdead0000;
461 }
462 
463 static void validate_memory(int *p, unsigned long start, unsigned long end)
464 {
465         int i;
466 
467         for (i = start / page_size; i < end / page_size; i++) {
468                 if (p[i * page_size / sizeof(*p)] != i + 0xdead0000) {
469                         printf("Page %d is corrupted: %#x\n",
470                                         i, p[i * page_size / sizeof(*p)]);
471                         exit(EXIT_FAILURE);
472                 }
473         }
474 }
475 
476 #define TICK 500000
477 static bool wait_for_scan(const char *msg, char *p)
478 {
479         int full_scans;
480         int timeout = 6; /* 3 seconds */
481 
482         /* Sanity check */
483         if (check_huge(p)) {
484                 printf("Unexpected huge page\n");
485                 exit(EXIT_FAILURE);
486         }
487 
488         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
489 
490         /* Wait until the second full_scan completed */
491         full_scans = read_num("khugepaged/full_scans") + 2;
492 
493         printf("%s...", msg);
494         while (timeout--) {
495                 if (check_huge(p))
496                         break;
497                 if (read_num("khugepaged/full_scans") >= full_scans)
498                         break;
499                 printf(".");
500                 usleep(TICK);
501         }
502 
503         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
504 
505         return timeout == -1;
506 }
507 
508 static void alloc_at_fault(void)
509 {
510         struct settings settings = default_settings;
511         char *p;
512 
513         settings.thp_enabled = THP_ALWAYS;
514         write_settings(&settings);
515 
516         p = alloc_mapping();
517         *p = 1;
518         printf("Allocate huge page on fault...");
519         if (check_huge(p))
520                 success("OK");
521         else
522                 fail("Fail");
523 
524         write_settings(&default_settings);
525 
526         madvise(p, page_size, MADV_DONTNEED);
527         printf("Split huge PMD on MADV_DONTNEED...");
528         if (!check_huge(p))
529                 success("OK");
530         else
531                 fail("Fail");
532         munmap(p, hpage_pmd_size);
533 }
534 
535 static void collapse_full(void)
536 {
537         void *p;
538 
539         p = alloc_mapping();
540         fill_memory(p, 0, hpage_pmd_size);
541         if (wait_for_scan("Collapse fully populated PTE table", p))
542                 fail("Timeout");
543         else if (check_huge(p))
544                 success("OK");
545         else
546                 fail("Fail");
547         validate_memory(p, 0, hpage_pmd_size);
548         munmap(p, hpage_pmd_size);
549 }
550 
551 static void collapse_empty(void)
552 {
553         void *p;
554 
555         p = alloc_mapping();
556         if (wait_for_scan("Do not collapse empty PTE table", p))
557                 fail("Timeout");
558         else if (check_huge(p))
559                 fail("Fail");
560         else
561                 success("OK");
562         munmap(p, hpage_pmd_size);
563 }
564 
565 static void collapse_single_pte_entry(void)
566 {
567         void *p;
568 
569         p = alloc_mapping();
570         fill_memory(p, 0, page_size);
571         if (wait_for_scan("Collapse PTE table with single PTE entry present", p))
572                 fail("Timeout");
573         else if (check_huge(p))
574                 success("OK");
575         else
576                 fail("Fail");
577         validate_memory(p, 0, page_size);
578         munmap(p, hpage_pmd_size);
579 }
580 
581 static void collapse_max_ptes_none(void)
582 {
583         int max_ptes_none = hpage_pmd_nr / 2;
584         struct settings settings = default_settings;
585         void *p;
586 
587         settings.khugepaged.max_ptes_none = max_ptes_none;
588         write_settings(&settings);
589 
590         p = alloc_mapping();
591 
592         fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
593         if (wait_for_scan("Do not collapse with max_ptes_none exceeded", p))
594                 fail("Timeout");
595         else if (check_huge(p))
596                 fail("Fail");
597         else
598                 success("OK");
599         validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
600 
601         fill_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
602         if (wait_for_scan("Collapse with max_ptes_none PTEs empty", p))
603                 fail("Timeout");
604         else if (check_huge(p))
605                 success("OK");
606         else
607                 fail("Fail");
608         validate_memory(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
609 
610         munmap(p, hpage_pmd_size);
611         write_settings(&default_settings);
612 }
613 
614 static void collapse_swapin_single_pte(void)
615 {
616         void *p;
617         p = alloc_mapping();
618         fill_memory(p, 0, hpage_pmd_size);
619 
620         printf("Swapout one page...");
621         if (madvise(p, page_size, MADV_PAGEOUT)) {
622                 perror("madvise(MADV_PAGEOUT)");
623                 exit(EXIT_FAILURE);
624         }
625         if (check_swap(p, page_size)) {
626                 success("OK");
627         } else {
628                 fail("Fail");
629                 goto out;
630         }
631 
632         if (wait_for_scan("Collapse with swapping in single PTE entry", p))
633                 fail("Timeout");
634         else if (check_huge(p))
635                 success("OK");
636         else
637                 fail("Fail");
638         validate_memory(p, 0, hpage_pmd_size);
639 out:
640         munmap(p, hpage_pmd_size);
641 }
642 
643 static void collapse_max_ptes_swap(void)
644 {
645         int max_ptes_swap = read_num("khugepaged/max_ptes_swap");
646         void *p;
647 
648         p = alloc_mapping();
649 
650         fill_memory(p, 0, hpage_pmd_size);
651         printf("Swapout %d of %d pages...", max_ptes_swap + 1, hpage_pmd_nr);
652         if (madvise(p, (max_ptes_swap + 1) * page_size, MADV_PAGEOUT)) {
653                 perror("madvise(MADV_PAGEOUT)");
654                 exit(EXIT_FAILURE);
655         }
656         if (check_swap(p, (max_ptes_swap + 1) * page_size)) {
657                 success("OK");
658         } else {
659                 fail("Fail");
660                 goto out;
661         }
662 
663         if (wait_for_scan("Do not collapse with max_ptes_swap exceeded", p))
664                 fail("Timeout");
665         else if (check_huge(p))
666                 fail("Fail");
667         else
668                 success("OK");
669         validate_memory(p, 0, hpage_pmd_size);
670 
671         fill_memory(p, 0, hpage_pmd_size);
672         printf("Swapout %d of %d pages...", max_ptes_swap, hpage_pmd_nr);
673         if (madvise(p, max_ptes_swap * page_size, MADV_PAGEOUT)) {
674                 perror("madvise(MADV_PAGEOUT)");
675                 exit(EXIT_FAILURE);
676         }
677         if (check_swap(p, max_ptes_swap * page_size)) {
678                 success("OK");
679         } else {
680                 fail("Fail");
681                 goto out;
682         }
683 
684         if (wait_for_scan("Collapse with max_ptes_swap pages swapped out", p))
685                 fail("Timeout");
686         else if (check_huge(p))
687                 success("OK");
688         else
689                 fail("Fail");
690         validate_memory(p, 0, hpage_pmd_size);
691 out:
692         munmap(p, hpage_pmd_size);
693 }
694 
695 static void collapse_single_pte_entry_compound(void)
696 {
697         void *p;
698 
699         p = alloc_mapping();
700 
701         printf("Allocate huge page...");
702         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
703         fill_memory(p, 0, hpage_pmd_size);
704         if (check_huge(p))
705                 success("OK");
706         else
707                 fail("Fail");
708         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
709 
710         printf("Split huge page leaving single PTE mapping compound page...");
711         madvise(p + page_size, hpage_pmd_size - page_size, MADV_DONTNEED);
712         if (!check_huge(p))
713                 success("OK");
714         else
715                 fail("Fail");
716 
717         if (wait_for_scan("Collapse PTE table with single PTE mapping compound page", p))
718                 fail("Timeout");
719         else if (check_huge(p))
720                 success("OK");
721         else
722                 fail("Fail");
723         validate_memory(p, 0, page_size);
724         munmap(p, hpage_pmd_size);
725 }
726 
727 static void collapse_full_of_compound(void)
728 {
729         void *p;
730 
731         p = alloc_mapping();
732 
733         printf("Allocate huge page...");
734         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
735         fill_memory(p, 0, hpage_pmd_size);
736         if (check_huge(p))
737                 success("OK");
738         else
739                 fail("Fail");
740 
741         printf("Split huge page leaving single PTE page table full of compound pages...");
742         madvise(p, page_size, MADV_NOHUGEPAGE);
743         madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
744         if (!check_huge(p))
745                 success("OK");
746         else
747                 fail("Fail");
748 
749         if (wait_for_scan("Collapse PTE table full of compound pages", p))
750                 fail("Timeout");
751         else if (check_huge(p))
752                 success("OK");
753         else
754                 fail("Fail");
755         validate_memory(p, 0, hpage_pmd_size);
756         munmap(p, hpage_pmd_size);
757 }
758 
759 static void collapse_compound_extreme(void)
760 {
761         void *p;
762         int i;
763 
764         p = alloc_mapping();
765         for (i = 0; i < hpage_pmd_nr; i++) {
766                 printf("\rConstruct PTE page table full of different PTE-mapped compound pages %3d/%d...",
767                                 i + 1, hpage_pmd_nr);
768 
769                 madvise(BASE_ADDR, hpage_pmd_size, MADV_HUGEPAGE);
770                 fill_memory(BASE_ADDR, 0, hpage_pmd_size);
771                 if (!check_huge(BASE_ADDR)) {
772                         printf("Failed to allocate huge page\n");
773                         exit(EXIT_FAILURE);
774                 }
775                 madvise(BASE_ADDR, hpage_pmd_size, MADV_NOHUGEPAGE);
776 
777                 p = mremap(BASE_ADDR - i * page_size,
778                                 i * page_size + hpage_pmd_size,
779                                 (i + 1) * page_size,
780                                 MREMAP_MAYMOVE | MREMAP_FIXED,
781                                 BASE_ADDR + 2 * hpage_pmd_size);
782                 if (p == MAP_FAILED) {
783                         perror("mremap+unmap");
784                         exit(EXIT_FAILURE);
785                 }
786 
787                 p = mremap(BASE_ADDR + 2 * hpage_pmd_size,
788                                 (i + 1) * page_size,
789                                 (i + 1) * page_size + hpage_pmd_size,
790                                 MREMAP_MAYMOVE | MREMAP_FIXED,
791                                 BASE_ADDR - (i + 1) * page_size);
792                 if (p == MAP_FAILED) {
793                         perror("mremap+alloc");
794                         exit(EXIT_FAILURE);
795                 }
796         }
797 
798         munmap(BASE_ADDR, hpage_pmd_size);
799         fill_memory(p, 0, hpage_pmd_size);
800         if (!check_huge(p))
801                 success("OK");
802         else
803                 fail("Fail");
804 
805         if (wait_for_scan("Collapse PTE table full of different compound pages", p))
806                 fail("Timeout");
807         else if (check_huge(p))
808                 success("OK");
809         else
810                 fail("Fail");
811 
812         validate_memory(p, 0, hpage_pmd_size);
813         munmap(p, hpage_pmd_size);
814 }
815 
816 static void collapse_fork(void)
817 {
818         int wstatus;
819         void *p;
820 
821         p = alloc_mapping();
822 
823         printf("Allocate small page...");
824         fill_memory(p, 0, page_size);
825         if (!check_huge(p))
826                 success("OK");
827         else
828                 fail("Fail");
829 
830         printf("Share small page over fork()...");
831         if (!fork()) {
832                 /* Do not touch settings on child exit */
833                 skip_settings_restore = true;
834                 exit_status = 0;
835 
836                 if (!check_huge(p))
837                         success("OK");
838                 else
839                         fail("Fail");
840 
841                 fill_memory(p, page_size, 2 * page_size);
842 
843                 if (wait_for_scan("Collapse PTE table with single page shared with parent process", p))
844                         fail("Timeout");
845                 else if (check_huge(p))
846                         success("OK");
847                 else
848                         fail("Fail");
849 
850                 validate_memory(p, 0, page_size);
851                 munmap(p, hpage_pmd_size);
852                 exit(exit_status);
853         }
854 
855         wait(&wstatus);
856         exit_status += WEXITSTATUS(wstatus);
857 
858         printf("Check if parent still has small page...");
859         if (!check_huge(p))
860                 success("OK");
861         else
862                 fail("Fail");
863         validate_memory(p, 0, page_size);
864         munmap(p, hpage_pmd_size);
865 }
866 
867 static void collapse_fork_compound(void)
868 {
869         int wstatus;
870         void *p;
871 
872         p = alloc_mapping();
873 
874         printf("Allocate huge page...");
875         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
876         fill_memory(p, 0, hpage_pmd_size);
877         if (check_huge(p))
878                 success("OK");
879         else
880                 fail("Fail");
881 
882         printf("Share huge page over fork()...");
883         if (!fork()) {
884                 /* Do not touch settings on child exit */
885                 skip_settings_restore = true;
886                 exit_status = 0;
887 
888                 if (check_huge(p))
889                         success("OK");
890                 else
891                         fail("Fail");
892 
893                 printf("Split huge page PMD in child process...");
894                 madvise(p, page_size, MADV_NOHUGEPAGE);
895                 madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE);
896                 if (!check_huge(p))
897                         success("OK");
898                 else
899                         fail("Fail");
900                 fill_memory(p, 0, page_size);
901 
902                 write_num("khugepaged/max_ptes_shared", hpage_pmd_nr - 1);
903                 if (wait_for_scan("Collapse PTE table full of compound pages in child", p))
904                         fail("Timeout");
905                 else if (check_huge(p))
906                         success("OK");
907                 else
908                         fail("Fail");
909                 write_num("khugepaged/max_ptes_shared",
910                                 default_settings.khugepaged.max_ptes_shared);
911 
912                 validate_memory(p, 0, hpage_pmd_size);
913                 munmap(p, hpage_pmd_size);
914                 exit(exit_status);
915         }
916 
917         wait(&wstatus);
918         exit_status += WEXITSTATUS(wstatus);
919 
920         printf("Check if parent still has huge page...");
921         if (check_huge(p))
922                 success("OK");
923         else
924                 fail("Fail");
925         validate_memory(p, 0, hpage_pmd_size);
926         munmap(p, hpage_pmd_size);
927 }
928 
929 static void collapse_max_ptes_shared()
930 {
931         int max_ptes_shared = read_num("khugepaged/max_ptes_shared");
932         int wstatus;
933         void *p;
934 
935         p = alloc_mapping();
936 
937         printf("Allocate huge page...");
938         madvise(p, hpage_pmd_size, MADV_HUGEPAGE);
939         fill_memory(p, 0, hpage_pmd_size);
940         if (check_huge(p))
941                 success("OK");
942         else
943                 fail("Fail");
944 
945         printf("Share huge page over fork()...");
946         if (!fork()) {
947                 /* Do not touch settings on child exit */
948                 skip_settings_restore = true;
949                 exit_status = 0;
950 
951                 if (check_huge(p))
952                         success("OK");
953                 else
954                         fail("Fail");
955 
956                 printf("Trigger CoW on page %d of %d...",
957                                 hpage_pmd_nr - max_ptes_shared - 1, hpage_pmd_nr);
958                 fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
959                 if (!check_huge(p))
960                         success("OK");
961                 else
962                         fail("Fail");
963 
964                 if (wait_for_scan("Do not collapse with max_ptes_shared exceeded", p))
965                         fail("Timeout");
966                 else if (!check_huge(p))
967                         success("OK");
968                 else
969                         fail("Fail");
970 
971                 printf("Trigger CoW on page %d of %d...",
972                                 hpage_pmd_nr - max_ptes_shared, hpage_pmd_nr);
973                 fill_memory(p, 0, (hpage_pmd_nr - max_ptes_shared) * page_size);
974                 if (!check_huge(p))
975                         success("OK");
976                 else
977                         fail("Fail");
978 
979 
980                 if (wait_for_scan("Collapse with max_ptes_shared PTEs shared", p))
981                         fail("Timeout");
982                 else if (check_huge(p))
983                         success("OK");
984                 else
985                         fail("Fail");
986 
987                 validate_memory(p, 0, hpage_pmd_size);
988                 munmap(p, hpage_pmd_size);
989                 exit(exit_status);
990         }
991 
992         wait(&wstatus);
993         exit_status += WEXITSTATUS(wstatus);
994 
995         printf("Check if parent still has huge page...");
996         if (check_huge(p))
997                 success("OK");
998         else
999                 fail("Fail");
1000         validate_memory(p, 0, hpage_pmd_size);
1001         munmap(p, hpage_pmd_size);
1002 }
1003 
1004 int main(void)
1005 {
1006         setbuf(stdout, NULL);
1007 
1008         page_size = getpagesize();
1009         hpage_pmd_size = read_num("hpage_pmd_size");
1010         hpage_pmd_nr = hpage_pmd_size / page_size;
1011 
1012         default_settings.khugepaged.max_ptes_none = hpage_pmd_nr - 1;
1013         default_settings.khugepaged.max_ptes_swap = hpage_pmd_nr / 8;
1014         default_settings.khugepaged.max_ptes_shared = hpage_pmd_nr / 2;
1015         default_settings.khugepaged.pages_to_scan = hpage_pmd_nr * 8;
1016 
1017         save_settings();
1018         adjust_settings();
1019 
1020         alloc_at_fault();
1021         collapse_full();
1022         collapse_empty();
1023         collapse_single_pte_entry();
1024         collapse_max_ptes_none();
1025         collapse_swapin_single_pte();
1026         collapse_max_ptes_swap();
1027         collapse_single_pte_entry_compound();
1028         collapse_full_of_compound();
1029         collapse_compound_extreme();
1030         collapse_fork();
1031         collapse_fork_compound();
1032         collapse_max_ptes_shared();
1033 
1034         restore_settings(0);
1035 }
1036 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp