~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/swapfile.c

Version: ~ [ linux-5.19-rc3 ] ~ [ linux-5.18.5 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.48 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.123 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.199 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.248 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.284 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.319 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  *  linux/mm/swapfile.c
  3  *
  4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
  5  *  Swap reorganised 29.12.95, Stephen Tweedie
  6  */
  7 
  8 #include <linux/mm.h>
  9 #include <linux/hugetlb.h>
 10 #include <linux/mman.h>
 11 #include <linux/slab.h>
 12 #include <linux/kernel_stat.h>
 13 #include <linux/swap.h>
 14 #include <linux/vmalloc.h>
 15 #include <linux/pagemap.h>
 16 #include <linux/namei.h>
 17 #include <linux/shmem_fs.h>
 18 #include <linux/blkdev.h>
 19 #include <linux/random.h>
 20 #include <linux/writeback.h>
 21 #include <linux/proc_fs.h>
 22 #include <linux/seq_file.h>
 23 #include <linux/init.h>
 24 #include <linux/ksm.h>
 25 #include <linux/rmap.h>
 26 #include <linux/security.h>
 27 #include <linux/backing-dev.h>
 28 #include <linux/mutex.h>
 29 #include <linux/capability.h>
 30 #include <linux/syscalls.h>
 31 #include <linux/memcontrol.h>
 32 #include <linux/poll.h>
 33 #include <linux/oom.h>
 34 #include <linux/frontswap.h>
 35 #include <linux/swapfile.h>
 36 #include <linux/export.h>
 37 
 38 #include <asm/pgtable.h>
 39 #include <asm/tlbflush.h>
 40 #include <linux/swapops.h>
 41 #include <linux/swap_cgroup.h>
 42 
 43 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
 44                                  unsigned char);
 45 static void free_swap_count_continuations(struct swap_info_struct *);
 46 static sector_t map_swap_entry(swp_entry_t, struct block_device**);
 47 
 48 DEFINE_SPINLOCK(swap_lock);
 49 static unsigned int nr_swapfiles;
 50 atomic_long_t nr_swap_pages;
 51 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
 52 long total_swap_pages;
 53 static int least_priority;
 54 
 55 static const char Bad_file[] = "Bad swap file entry ";
 56 static const char Unused_file[] = "Unused swap file entry ";
 57 static const char Bad_offset[] = "Bad swap offset entry ";
 58 static const char Unused_offset[] = "Unused swap offset entry ";
 59 
 60 /*
 61  * all active swap_info_structs
 62  * protected with swap_lock, and ordered by priority.
 63  */
 64 PLIST_HEAD(swap_active_head);
 65 
 66 /*
 67  * all available (active, not full) swap_info_structs
 68  * protected with swap_avail_lock, ordered by priority.
 69  * This is used by get_swap_page() instead of swap_active_head
 70  * because swap_active_head includes all swap_info_structs,
 71  * but get_swap_page() doesn't need to look at full ones.
 72  * This uses its own lock instead of swap_lock because when a
 73  * swap_info_struct changes between not-full/full, it needs to
 74  * add/remove itself to/from this list, but the swap_info_struct->lock
 75  * is held and the locking order requires swap_lock to be taken
 76  * before any swap_info_struct->lock.
 77  */
 78 static PLIST_HEAD(swap_avail_head);
 79 static DEFINE_SPINLOCK(swap_avail_lock);
 80 
 81 struct swap_info_struct *swap_info[MAX_SWAPFILES];
 82 
 83 static DEFINE_MUTEX(swapon_mutex);
 84 
 85 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
 86 /* Activity counter to indicate that a swapon or swapoff has occurred */
 87 static atomic_t proc_poll_event = ATOMIC_INIT(0);
 88 
 89 static inline unsigned char swap_count(unsigned char ent)
 90 {
 91         return ent & ~SWAP_HAS_CACHE;   /* may include SWAP_HAS_CONT flag */
 92 }
 93 
 94 /* returns 1 if swap entry is freed */
 95 static int
 96 __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
 97 {
 98         swp_entry_t entry = swp_entry(si->type, offset);
 99         struct page *page;
100         int ret = 0;
101 
102         page = find_get_page(swap_address_space(entry), entry.val);
103         if (!page)
104                 return 0;
105         /*
106          * This function is called from scan_swap_map() and it's called
107          * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
108          * We have to use trylock for avoiding deadlock. This is a special
109          * case and you should use try_to_free_swap() with explicit lock_page()
110          * in usual operations.
111          */
112         if (trylock_page(page)) {
113                 ret = try_to_free_swap(page);
114                 unlock_page(page);
115         }
116         page_cache_release(page);
117         return ret;
118 }
119 
120 /*
121  * swapon tell device that all the old swap contents can be discarded,
122  * to allow the swap device to optimize its wear-levelling.
123  */
124 static int discard_swap(struct swap_info_struct *si)
125 {
126         struct swap_extent *se;
127         sector_t start_block;
128         sector_t nr_blocks;
129         int err = 0;
130 
131         /* Do not discard the swap header page! */
132         se = &si->first_swap_extent;
133         start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
134         nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
135         if (nr_blocks) {
136                 err = blkdev_issue_discard(si->bdev, start_block,
137                                 nr_blocks, GFP_KERNEL, 0);
138                 if (err)
139                         return err;
140                 cond_resched();
141         }
142 
143         list_for_each_entry(se, &si->first_swap_extent.list, list) {
144                 start_block = se->start_block << (PAGE_SHIFT - 9);
145                 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
146 
147                 err = blkdev_issue_discard(si->bdev, start_block,
148                                 nr_blocks, GFP_KERNEL, 0);
149                 if (err)
150                         break;
151 
152                 cond_resched();
153         }
154         return err;             /* That will often be -EOPNOTSUPP */
155 }
156 
157 /*
158  * swap allocation tell device that a cluster of swap can now be discarded,
159  * to allow the swap device to optimize its wear-levelling.
160  */
161 static void discard_swap_cluster(struct swap_info_struct *si,
162                                  pgoff_t start_page, pgoff_t nr_pages)
163 {
164         struct swap_extent *se = si->curr_swap_extent;
165         int found_extent = 0;
166 
167         while (nr_pages) {
168                 struct list_head *lh;
169 
170                 if (se->start_page <= start_page &&
171                     start_page < se->start_page + se->nr_pages) {
172                         pgoff_t offset = start_page - se->start_page;
173                         sector_t start_block = se->start_block + offset;
174                         sector_t nr_blocks = se->nr_pages - offset;
175 
176                         if (nr_blocks > nr_pages)
177                                 nr_blocks = nr_pages;
178                         start_page += nr_blocks;
179                         nr_pages -= nr_blocks;
180 
181                         if (!found_extent++)
182                                 si->curr_swap_extent = se;
183 
184                         start_block <<= PAGE_SHIFT - 9;
185                         nr_blocks <<= PAGE_SHIFT - 9;
186                         if (blkdev_issue_discard(si->bdev, start_block,
187                                     nr_blocks, GFP_NOIO, 0))
188                                 break;
189                 }
190 
191                 lh = se->list.next;
192                 se = list_entry(lh, struct swap_extent, list);
193         }
194 }
195 
196 #define SWAPFILE_CLUSTER        256
197 #define LATENCY_LIMIT           256
198 
199 static inline void cluster_set_flag(struct swap_cluster_info *info,
200         unsigned int flag)
201 {
202         info->flags = flag;
203 }
204 
205 static inline unsigned int cluster_count(struct swap_cluster_info *info)
206 {
207         return info->data;
208 }
209 
210 static inline void cluster_set_count(struct swap_cluster_info *info,
211                                      unsigned int c)
212 {
213         info->data = c;
214 }
215 
216 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
217                                          unsigned int c, unsigned int f)
218 {
219         info->flags = f;
220         info->data = c;
221 }
222 
223 static inline unsigned int cluster_next(struct swap_cluster_info *info)
224 {
225         return info->data;
226 }
227 
228 static inline void cluster_set_next(struct swap_cluster_info *info,
229                                     unsigned int n)
230 {
231         info->data = n;
232 }
233 
234 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
235                                          unsigned int n, unsigned int f)
236 {
237         info->flags = f;
238         info->data = n;
239 }
240 
241 static inline bool cluster_is_free(struct swap_cluster_info *info)
242 {
243         return info->flags & CLUSTER_FLAG_FREE;
244 }
245 
246 static inline bool cluster_is_null(struct swap_cluster_info *info)
247 {
248         return info->flags & CLUSTER_FLAG_NEXT_NULL;
249 }
250 
251 static inline void cluster_set_null(struct swap_cluster_info *info)
252 {
253         info->flags = CLUSTER_FLAG_NEXT_NULL;
254         info->data = 0;
255 }
256 
257 /* Add a cluster to discard list and schedule it to do discard */
258 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
259                 unsigned int idx)
260 {
261         /*
262          * If scan_swap_map() can't find a free cluster, it will check
263          * si->swap_map directly. To make sure the discarding cluster isn't
264          * taken by scan_swap_map(), mark the swap entries bad (occupied). It
265          * will be cleared after discard
266          */
267         memset(si->swap_map + idx * SWAPFILE_CLUSTER,
268                         SWAP_MAP_BAD, SWAPFILE_CLUSTER);
269 
270         if (cluster_is_null(&si->discard_cluster_head)) {
271                 cluster_set_next_flag(&si->discard_cluster_head,
272                                                 idx, 0);
273                 cluster_set_next_flag(&si->discard_cluster_tail,
274                                                 idx, 0);
275         } else {
276                 unsigned int tail = cluster_next(&si->discard_cluster_tail);
277                 cluster_set_next(&si->cluster_info[tail], idx);
278                 cluster_set_next_flag(&si->discard_cluster_tail,
279                                                 idx, 0);
280         }
281 
282         schedule_work(&si->discard_work);
283 }
284 
285 /*
286  * Doing discard actually. After a cluster discard is finished, the cluster
287  * will be added to free cluster list. caller should hold si->lock.
288 */
289 static void swap_do_scheduled_discard(struct swap_info_struct *si)
290 {
291         struct swap_cluster_info *info;
292         unsigned int idx;
293 
294         info = si->cluster_info;
295 
296         while (!cluster_is_null(&si->discard_cluster_head)) {
297                 idx = cluster_next(&si->discard_cluster_head);
298 
299                 cluster_set_next_flag(&si->discard_cluster_head,
300                                                 cluster_next(&info[idx]), 0);
301                 if (cluster_next(&si->discard_cluster_tail) == idx) {
302                         cluster_set_null(&si->discard_cluster_head);
303                         cluster_set_null(&si->discard_cluster_tail);
304                 }
305                 spin_unlock(&si->lock);
306 
307                 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
308                                 SWAPFILE_CLUSTER);
309 
310                 spin_lock(&si->lock);
311                 cluster_set_flag(&info[idx], CLUSTER_FLAG_FREE);
312                 if (cluster_is_null(&si->free_cluster_head)) {
313                         cluster_set_next_flag(&si->free_cluster_head,
314                                                 idx, 0);
315                         cluster_set_next_flag(&si->free_cluster_tail,
316                                                 idx, 0);
317                 } else {
318                         unsigned int tail;
319 
320                         tail = cluster_next(&si->free_cluster_tail);
321                         cluster_set_next(&info[tail], idx);
322                         cluster_set_next_flag(&si->free_cluster_tail,
323                                                 idx, 0);
324                 }
325                 memset(si->swap_map + idx * SWAPFILE_CLUSTER,
326                                 0, SWAPFILE_CLUSTER);
327         }
328 }
329 
330 static void swap_discard_work(struct work_struct *work)
331 {
332         struct swap_info_struct *si;
333 
334         si = container_of(work, struct swap_info_struct, discard_work);
335 
336         spin_lock(&si->lock);
337         swap_do_scheduled_discard(si);
338         spin_unlock(&si->lock);
339 }
340 
341 /*
342  * The cluster corresponding to page_nr will be used. The cluster will be
343  * removed from free cluster list and its usage counter will be increased.
344  */
345 static void inc_cluster_info_page(struct swap_info_struct *p,
346         struct swap_cluster_info *cluster_info, unsigned long page_nr)
347 {
348         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
349 
350         if (!cluster_info)
351                 return;
352         if (cluster_is_free(&cluster_info[idx])) {
353                 VM_BUG_ON(cluster_next(&p->free_cluster_head) != idx);
354                 cluster_set_next_flag(&p->free_cluster_head,
355                         cluster_next(&cluster_info[idx]), 0);
356                 if (cluster_next(&p->free_cluster_tail) == idx) {
357                         cluster_set_null(&p->free_cluster_tail);
358                         cluster_set_null(&p->free_cluster_head);
359                 }
360                 cluster_set_count_flag(&cluster_info[idx], 0, 0);
361         }
362 
363         VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
364         cluster_set_count(&cluster_info[idx],
365                 cluster_count(&cluster_info[idx]) + 1);
366 }
367 
368 /*
369  * The cluster corresponding to page_nr decreases one usage. If the usage
370  * counter becomes 0, which means no page in the cluster is in using, we can
371  * optionally discard the cluster and add it to free cluster list.
372  */
373 static void dec_cluster_info_page(struct swap_info_struct *p,
374         struct swap_cluster_info *cluster_info, unsigned long page_nr)
375 {
376         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
377 
378         if (!cluster_info)
379                 return;
380 
381         VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
382         cluster_set_count(&cluster_info[idx],
383                 cluster_count(&cluster_info[idx]) - 1);
384 
385         if (cluster_count(&cluster_info[idx]) == 0) {
386                 /*
387                  * If the swap is discardable, prepare discard the cluster
388                  * instead of free it immediately. The cluster will be freed
389                  * after discard.
390                  */
391                 if ((p->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
392                                  (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
393                         swap_cluster_schedule_discard(p, idx);
394                         return;
395                 }
396 
397                 cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
398                 if (cluster_is_null(&p->free_cluster_head)) {
399                         cluster_set_next_flag(&p->free_cluster_head, idx, 0);
400                         cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
401                 } else {
402                         unsigned int tail = cluster_next(&p->free_cluster_tail);
403                         cluster_set_next(&cluster_info[tail], idx);
404                         cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
405                 }
406         }
407 }
408 
409 /*
410  * It's possible scan_swap_map() uses a free cluster in the middle of free
411  * cluster list. Avoiding such abuse to avoid list corruption.
412  */
413 static bool
414 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
415         unsigned long offset)
416 {
417         struct percpu_cluster *percpu_cluster;
418         bool conflict;
419 
420         offset /= SWAPFILE_CLUSTER;
421         conflict = !cluster_is_null(&si->free_cluster_head) &&
422                 offset != cluster_next(&si->free_cluster_head) &&
423                 cluster_is_free(&si->cluster_info[offset]);
424 
425         if (!conflict)
426                 return false;
427 
428         percpu_cluster = this_cpu_ptr(si->percpu_cluster);
429         cluster_set_null(&percpu_cluster->index);
430         return true;
431 }
432 
433 /*
434  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
435  * might involve allocating a new cluster for current CPU too.
436  */
437 static void scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
438         unsigned long *offset, unsigned long *scan_base)
439 {
440         struct percpu_cluster *cluster;
441         bool found_free;
442         unsigned long tmp;
443 
444 new_cluster:
445         cluster = this_cpu_ptr(si->percpu_cluster);
446         if (cluster_is_null(&cluster->index)) {
447                 if (!cluster_is_null(&si->free_cluster_head)) {
448                         cluster->index = si->free_cluster_head;
449                         cluster->next = cluster_next(&cluster->index) *
450                                         SWAPFILE_CLUSTER;
451                 } else if (!cluster_is_null(&si->discard_cluster_head)) {
452                         /*
453                          * we don't have free cluster but have some clusters in
454                          * discarding, do discard now and reclaim them
455                          */
456                         swap_do_scheduled_discard(si);
457                         *scan_base = *offset = si->cluster_next;
458                         goto new_cluster;
459                 } else
460                         return;
461         }
462 
463         found_free = false;
464 
465         /*
466          * Other CPUs can use our cluster if they can't find a free cluster,
467          * check if there is still free entry in the cluster
468          */
469         tmp = cluster->next;
470         while (tmp < si->max && tmp < (cluster_next(&cluster->index) + 1) *
471                SWAPFILE_CLUSTER) {
472                 if (!si->swap_map[tmp]) {
473                         found_free = true;
474                         break;
475                 }
476                 tmp++;
477         }
478         if (!found_free) {
479                 cluster_set_null(&cluster->index);
480                 goto new_cluster;
481         }
482         cluster->next = tmp + 1;
483         *offset = tmp;
484         *scan_base = tmp;
485 }
486 
487 static unsigned long scan_swap_map(struct swap_info_struct *si,
488                                    unsigned char usage)
489 {
490         unsigned long offset;
491         unsigned long scan_base;
492         unsigned long last_in_cluster = 0;
493         int latency_ration = LATENCY_LIMIT;
494 
495         /*
496          * We try to cluster swap pages by allocating them sequentially
497          * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
498          * way, however, we resort to first-free allocation, starting
499          * a new cluster.  This prevents us from scattering swap pages
500          * all over the entire swap partition, so that we reduce
501          * overall disk seek times between swap pages.  -- sct
502          * But we do now try to find an empty cluster.  -Andrea
503          * And we let swap pages go all over an SSD partition.  Hugh
504          */
505 
506         si->flags += SWP_SCANNING;
507         scan_base = offset = si->cluster_next;
508 
509         /* SSD algorithm */
510         if (si->cluster_info) {
511                 scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
512                 goto checks;
513         }
514 
515         if (unlikely(!si->cluster_nr--)) {
516                 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
517                         si->cluster_nr = SWAPFILE_CLUSTER - 1;
518                         goto checks;
519                 }
520 
521                 spin_unlock(&si->lock);
522 
523                 /*
524                  * If seek is expensive, start searching for new cluster from
525                  * start of partition, to minimize the span of allocated swap.
526                  * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
527                  * case, just handled by scan_swap_map_try_ssd_cluster() above.
528                  */
529                 scan_base = offset = si->lowest_bit;
530                 last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
531 
532                 /* Locate the first empty (unaligned) cluster */
533                 for (; last_in_cluster <= si->highest_bit; offset++) {
534                         if (si->swap_map[offset])
535                                 last_in_cluster = offset + SWAPFILE_CLUSTER;
536                         else if (offset == last_in_cluster) {
537                                 spin_lock(&si->lock);
538                                 offset -= SWAPFILE_CLUSTER - 1;
539                                 si->cluster_next = offset;
540                                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
541                                 goto checks;
542                         }
543                         if (unlikely(--latency_ration < 0)) {
544                                 cond_resched();
545                                 latency_ration = LATENCY_LIMIT;
546                         }
547                 }
548 
549                 offset = scan_base;
550                 spin_lock(&si->lock);
551                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
552         }
553 
554 checks:
555         if (si->cluster_info) {
556                 while (scan_swap_map_ssd_cluster_conflict(si, offset))
557                         scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
558         }
559         if (!(si->flags & SWP_WRITEOK))
560                 goto no_page;
561         if (!si->highest_bit)
562                 goto no_page;
563         if (offset > si->highest_bit)
564                 scan_base = offset = si->lowest_bit;
565 
566         /* reuse swap entry of cache-only swap if not busy. */
567         if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
568                 int swap_was_freed;
569                 spin_unlock(&si->lock);
570                 swap_was_freed = __try_to_reclaim_swap(si, offset);
571                 spin_lock(&si->lock);
572                 /* entry was freed successfully, try to use this again */
573                 if (swap_was_freed)
574                         goto checks;
575                 goto scan; /* check next one */
576         }
577 
578         if (si->swap_map[offset])
579                 goto scan;
580 
581         if (offset == si->lowest_bit)
582                 si->lowest_bit++;
583         if (offset == si->highest_bit)
584                 si->highest_bit--;
585         si->inuse_pages++;
586         if (si->inuse_pages == si->pages) {
587                 si->lowest_bit = si->max;
588                 si->highest_bit = 0;
589                 spin_lock(&swap_avail_lock);
590                 plist_del(&si->avail_list, &swap_avail_head);
591                 spin_unlock(&swap_avail_lock);
592         }
593         si->swap_map[offset] = usage;
594         inc_cluster_info_page(si, si->cluster_info, offset);
595         si->cluster_next = offset + 1;
596         si->flags -= SWP_SCANNING;
597 
598         return offset;
599 
600 scan:
601         spin_unlock(&si->lock);
602         while (++offset <= si->highest_bit) {
603                 if (!si->swap_map[offset]) {
604                         spin_lock(&si->lock);
605                         goto checks;
606                 }
607                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
608                         spin_lock(&si->lock);
609                         goto checks;
610                 }
611                 if (unlikely(--latency_ration < 0)) {
612                         cond_resched();
613                         latency_ration = LATENCY_LIMIT;
614                 }
615         }
616         offset = si->lowest_bit;
617         while (offset < scan_base) {
618                 if (!si->swap_map[offset]) {
619                         spin_lock(&si->lock);
620                         goto checks;
621                 }
622                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
623                         spin_lock(&si->lock);
624                         goto checks;
625                 }
626                 if (unlikely(--latency_ration < 0)) {
627                         cond_resched();
628                         latency_ration = LATENCY_LIMIT;
629                 }
630                 offset++;
631         }
632         spin_lock(&si->lock);
633 
634 no_page:
635         si->flags -= SWP_SCANNING;
636         return 0;
637 }
638 
639 swp_entry_t get_swap_page(void)
640 {
641         struct swap_info_struct *si, *next;
642         pgoff_t offset;
643 
644         if (atomic_long_read(&nr_swap_pages) <= 0)
645                 goto noswap;
646         atomic_long_dec(&nr_swap_pages);
647 
648         spin_lock(&swap_avail_lock);
649 
650 start_over:
651         plist_for_each_entry_safe(si, next, &swap_avail_head, avail_list) {
652                 /* requeue si to after same-priority siblings */
653                 plist_requeue(&si->avail_list, &swap_avail_head);
654                 spin_unlock(&swap_avail_lock);
655                 spin_lock(&si->lock);
656                 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
657                         spin_lock(&swap_avail_lock);
658                         if (plist_node_empty(&si->avail_list)) {
659                                 spin_unlock(&si->lock);
660                                 goto nextsi;
661                         }
662                         WARN(!si->highest_bit,
663                              "swap_info %d in list but !highest_bit\n",
664                              si->type);
665                         WARN(!(si->flags & SWP_WRITEOK),
666                              "swap_info %d in list but !SWP_WRITEOK\n",
667                              si->type);
668                         plist_del(&si->avail_list, &swap_avail_head);
669                         spin_unlock(&si->lock);
670                         goto nextsi;
671                 }
672 
673                 /* This is called for allocating swap entry for cache */
674                 offset = scan_swap_map(si, SWAP_HAS_CACHE);
675                 spin_unlock(&si->lock);
676                 if (offset)
677                         return swp_entry(si->type, offset);
678                 pr_debug("scan_swap_map of si %d failed to find offset\n",
679                        si->type);
680                 spin_lock(&swap_avail_lock);
681 nextsi:
682                 /*
683                  * if we got here, it's likely that si was almost full before,
684                  * and since scan_swap_map() can drop the si->lock, multiple
685                  * callers probably all tried to get a page from the same si
686                  * and it filled up before we could get one; or, the si filled
687                  * up between us dropping swap_avail_lock and taking si->lock.
688                  * Since we dropped the swap_avail_lock, the swap_avail_head
689                  * list may have been modified; so if next is still in the
690                  * swap_avail_head list then try it, otherwise start over.
691                  */
692                 if (plist_node_empty(&next->avail_list))
693                         goto start_over;
694         }
695 
696         spin_unlock(&swap_avail_lock);
697 
698         atomic_long_inc(&nr_swap_pages);
699 noswap:
700         return (swp_entry_t) {0};
701 }
702 
703 /* The only caller of this function is now suspend routine */
704 swp_entry_t get_swap_page_of_type(int type)
705 {
706         struct swap_info_struct *si;
707         pgoff_t offset;
708 
709         si = swap_info[type];
710         spin_lock(&si->lock);
711         if (si && (si->flags & SWP_WRITEOK)) {
712                 atomic_long_dec(&nr_swap_pages);
713                 /* This is called for allocating swap entry, not cache */
714                 offset = scan_swap_map(si, 1);
715                 if (offset) {
716                         spin_unlock(&si->lock);
717                         return swp_entry(type, offset);
718                 }
719                 atomic_long_inc(&nr_swap_pages);
720         }
721         spin_unlock(&si->lock);
722         return (swp_entry_t) {0};
723 }
724 
725 static struct swap_info_struct *swap_info_get(swp_entry_t entry)
726 {
727         struct swap_info_struct *p;
728         unsigned long offset, type;
729 
730         if (!entry.val)
731                 goto out;
732         type = swp_type(entry);
733         if (type >= nr_swapfiles)
734                 goto bad_nofile;
735         p = swap_info[type];
736         if (!(p->flags & SWP_USED))
737                 goto bad_device;
738         offset = swp_offset(entry);
739         if (offset >= p->max)
740                 goto bad_offset;
741         if (!p->swap_map[offset])
742                 goto bad_free;
743         spin_lock(&p->lock);
744         return p;
745 
746 bad_free:
747         pr_err("swap_free: %s%08lx\n", Unused_offset, entry.val);
748         goto out;
749 bad_offset:
750         pr_err("swap_free: %s%08lx\n", Bad_offset, entry.val);
751         goto out;
752 bad_device:
753         pr_err("swap_free: %s%08lx\n", Unused_file, entry.val);
754         goto out;
755 bad_nofile:
756         pr_err("swap_free: %s%08lx\n", Bad_file, entry.val);
757 out:
758         return NULL;
759 }
760 
761 static unsigned char swap_entry_free(struct swap_info_struct *p,
762                                      swp_entry_t entry, unsigned char usage)
763 {
764         unsigned long offset = swp_offset(entry);
765         unsigned char count;
766         unsigned char has_cache;
767 
768         count = p->swap_map[offset];
769         has_cache = count & SWAP_HAS_CACHE;
770         count &= ~SWAP_HAS_CACHE;
771 
772         if (usage == SWAP_HAS_CACHE) {
773                 VM_BUG_ON(!has_cache);
774                 has_cache = 0;
775         } else if (count == SWAP_MAP_SHMEM) {
776                 /*
777                  * Or we could insist on shmem.c using a special
778                  * swap_shmem_free() and free_shmem_swap_and_cache()...
779                  */
780                 count = 0;
781         } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
782                 if (count == COUNT_CONTINUED) {
783                         if (swap_count_continued(p, offset, count))
784                                 count = SWAP_MAP_MAX | COUNT_CONTINUED;
785                         else
786                                 count = SWAP_MAP_MAX;
787                 } else
788                         count--;
789         }
790 
791         if (!count)
792                 mem_cgroup_uncharge_swap(entry);
793 
794         usage = count | has_cache;
795         p->swap_map[offset] = usage;
796 
797         /* free if no reference */
798         if (!usage) {
799                 dec_cluster_info_page(p, p->cluster_info, offset);
800                 if (offset < p->lowest_bit)
801                         p->lowest_bit = offset;
802                 if (offset > p->highest_bit) {
803                         bool was_full = !p->highest_bit;
804                         p->highest_bit = offset;
805                         if (was_full && (p->flags & SWP_WRITEOK)) {
806                                 spin_lock(&swap_avail_lock);
807                                 WARN_ON(!plist_node_empty(&p->avail_list));
808                                 if (plist_node_empty(&p->avail_list))
809                                         plist_add(&p->avail_list,
810                                                   &swap_avail_head);
811                                 spin_unlock(&swap_avail_lock);
812                         }
813                 }
814                 atomic_long_inc(&nr_swap_pages);
815                 p->inuse_pages--;
816                 frontswap_invalidate_page(p->type, offset);
817                 if (p->flags & SWP_BLKDEV) {
818                         struct gendisk *disk = p->bdev->bd_disk;
819                         if (disk->fops->swap_slot_free_notify)
820                                 disk->fops->swap_slot_free_notify(p->bdev,
821                                                                   offset);
822                 }
823         }
824 
825         return usage;
826 }
827 
828 /*
829  * Caller has made sure that the swap device corresponding to entry
830  * is still around or has not been recycled.
831  */
832 void swap_free(swp_entry_t entry)
833 {
834         struct swap_info_struct *p;
835 
836         p = swap_info_get(entry);
837         if (p) {
838                 swap_entry_free(p, entry, 1);
839                 spin_unlock(&p->lock);
840         }
841 }
842 
843 /*
844  * Called after dropping swapcache to decrease refcnt to swap entries.
845  */
846 void swapcache_free(swp_entry_t entry)
847 {
848         struct swap_info_struct *p;
849 
850         p = swap_info_get(entry);
851         if (p) {
852                 swap_entry_free(p, entry, SWAP_HAS_CACHE);
853                 spin_unlock(&p->lock);
854         }
855 }
856 
857 /*
858  * How many references to page are currently swapped out?
859  * This does not give an exact answer when swap count is continued,
860  * but does include the high COUNT_CONTINUED flag to allow for that.
861  */
862 int page_swapcount(struct page *page)
863 {
864         int count = 0;
865         struct swap_info_struct *p;
866         swp_entry_t entry;
867 
868         entry.val = page_private(page);
869         p = swap_info_get(entry);
870         if (p) {
871                 count = swap_count(p->swap_map[swp_offset(entry)]);
872                 spin_unlock(&p->lock);
873         }
874         return count;
875 }
876 
877 /*
878  * How many references to @entry are currently swapped out?
879  * This considers COUNT_CONTINUED so it returns exact answer.
880  */
881 int swp_swapcount(swp_entry_t entry)
882 {
883         int count, tmp_count, n;
884         struct swap_info_struct *p;
885         struct page *page;
886         pgoff_t offset;
887         unsigned char *map;
888 
889         p = swap_info_get(entry);
890         if (!p)
891                 return 0;
892 
893         count = swap_count(p->swap_map[swp_offset(entry)]);
894         if (!(count & COUNT_CONTINUED))
895                 goto out;
896 
897         count &= ~COUNT_CONTINUED;
898         n = SWAP_MAP_MAX + 1;
899 
900         offset = swp_offset(entry);
901         page = vmalloc_to_page(p->swap_map + offset);
902         offset &= ~PAGE_MASK;
903         VM_BUG_ON(page_private(page) != SWP_CONTINUED);
904 
905         do {
906                 page = list_entry(page->lru.next, struct page, lru);
907                 map = kmap_atomic(page);
908                 tmp_count = map[offset];
909                 kunmap_atomic(map);
910 
911                 count += (tmp_count & ~COUNT_CONTINUED) * n;
912                 n *= (SWAP_CONT_MAX + 1);
913         } while (tmp_count & COUNT_CONTINUED);
914 out:
915         spin_unlock(&p->lock);
916         return count;
917 }
918 
919 /*
920  * We can write to an anon page without COW if there are no other references
921  * to it.  And as a side-effect, free up its swap: because the old content
922  * on disk will never be read, and seeking back there to write new content
923  * later would only waste time away from clustering.
924  */
925 int reuse_swap_page(struct page *page)
926 {
927         int count;
928 
929         VM_BUG_ON_PAGE(!PageLocked(page), page);
930         if (unlikely(PageKsm(page)))
931                 return 0;
932         count = page_mapcount(page);
933         if (count <= 1 && PageSwapCache(page)) {
934                 count += page_swapcount(page);
935                 if (count == 1 && !PageWriteback(page)) {
936                         delete_from_swap_cache(page);
937                         SetPageDirty(page);
938                 }
939         }
940         return count <= 1;
941 }
942 
943 /*
944  * If swap is getting full, or if there are no more mappings of this page,
945  * then try_to_free_swap is called to free its swap space.
946  */
947 int try_to_free_swap(struct page *page)
948 {
949         VM_BUG_ON_PAGE(!PageLocked(page), page);
950 
951         if (!PageSwapCache(page))
952                 return 0;
953         if (PageWriteback(page))
954                 return 0;
955         if (page_swapcount(page))
956                 return 0;
957 
958         /*
959          * Once hibernation has begun to create its image of memory,
960          * there's a danger that one of the calls to try_to_free_swap()
961          * - most probably a call from __try_to_reclaim_swap() while
962          * hibernation is allocating its own swap pages for the image,
963          * but conceivably even a call from memory reclaim - will free
964          * the swap from a page which has already been recorded in the
965          * image as a clean swapcache page, and then reuse its swap for
966          * another page of the image.  On waking from hibernation, the
967          * original page might be freed under memory pressure, then
968          * later read back in from swap, now with the wrong data.
969          *
970          * Hibernation suspends storage while it is writing the image
971          * to disk so check that here.
972          */
973         if (pm_suspended_storage())
974                 return 0;
975 
976         delete_from_swap_cache(page);
977         SetPageDirty(page);
978         return 1;
979 }
980 
981 /*
982  * Free the swap entry like above, but also try to
983  * free the page cache entry if it is the last user.
984  */
985 int free_swap_and_cache(swp_entry_t entry)
986 {
987         struct swap_info_struct *p;
988         struct page *page = NULL;
989 
990         if (non_swap_entry(entry))
991                 return 1;
992 
993         p = swap_info_get(entry);
994         if (p) {
995                 if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
996                         page = find_get_page(swap_address_space(entry),
997                                                 entry.val);
998                         if (page && !trylock_page(page)) {
999                                 page_cache_release(page);
1000                                 page = NULL;
1001                         }
1002                 }
1003                 spin_unlock(&p->lock);
1004         }
1005         if (page) {
1006                 /*
1007                  * Not mapped elsewhere, or swap space full? Free it!
1008                  * Also recheck PageSwapCache now page is locked (above).
1009                  */
1010                 if (PageSwapCache(page) && !PageWriteback(page) &&
1011                                 (!page_mapped(page) || vm_swap_full())) {
1012                         delete_from_swap_cache(page);
1013                         SetPageDirty(page);
1014                 }
1015                 unlock_page(page);
1016                 page_cache_release(page);
1017         }
1018         return p != NULL;
1019 }
1020 
1021 #ifdef CONFIG_HIBERNATION
1022 /*
1023  * Find the swap type that corresponds to given device (if any).
1024  *
1025  * @offset - number of the PAGE_SIZE-sized block of the device, starting
1026  * from 0, in which the swap header is expected to be located.
1027  *
1028  * This is needed for the suspend to disk (aka swsusp).
1029  */
1030 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1031 {
1032         struct block_device *bdev = NULL;
1033         int type;
1034 
1035         if (device)
1036                 bdev = bdget(device);
1037 
1038         spin_lock(&swap_lock);
1039         for (type = 0; type < nr_swapfiles; type++) {
1040                 struct swap_info_struct *sis = swap_info[type];
1041 
1042                 if (!(sis->flags & SWP_WRITEOK))
1043                         continue;
1044 
1045                 if (!bdev) {
1046                         if (bdev_p)
1047                                 *bdev_p = bdgrab(sis->bdev);
1048 
1049                         spin_unlock(&swap_lock);
1050                         return type;
1051                 }
1052                 if (bdev == sis->bdev) {
1053                         struct swap_extent *se = &sis->first_swap_extent;
1054 
1055                         if (se->start_block == offset) {
1056                                 if (bdev_p)
1057                                         *bdev_p = bdgrab(sis->bdev);
1058 
1059                                 spin_unlock(&swap_lock);
1060                                 bdput(bdev);
1061                                 return type;
1062                         }
1063                 }
1064         }
1065         spin_unlock(&swap_lock);
1066         if (bdev)
1067                 bdput(bdev);
1068 
1069         return -ENODEV;
1070 }
1071 
1072 /*
1073  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1074  * corresponding to given index in swap_info (swap type).
1075  */
1076 sector_t swapdev_block(int type, pgoff_t offset)
1077 {
1078         struct block_device *bdev;
1079 
1080         if ((unsigned int)type >= nr_swapfiles)
1081                 return 0;
1082         if (!(swap_info[type]->flags & SWP_WRITEOK))
1083                 return 0;
1084         return map_swap_entry(swp_entry(type, offset), &bdev);
1085 }
1086 
1087 /*
1088  * Return either the total number of swap pages of given type, or the number
1089  * of free pages of that type (depending on @free)
1090  *
1091  * This is needed for software suspend
1092  */
1093 unsigned int count_swap_pages(int type, int free)
1094 {
1095         unsigned int n = 0;
1096 
1097         spin_lock(&swap_lock);
1098         if ((unsigned int)type < nr_swapfiles) {
1099                 struct swap_info_struct *sis = swap_info[type];
1100 
1101                 spin_lock(&sis->lock);
1102                 if (sis->flags & SWP_WRITEOK) {
1103                         n = sis->pages;
1104                         if (free)
1105                                 n -= sis->inuse_pages;
1106                 }
1107                 spin_unlock(&sis->lock);
1108         }
1109         spin_unlock(&swap_lock);
1110         return n;
1111 }
1112 #endif /* CONFIG_HIBERNATION */
1113 
1114 static inline int maybe_same_pte(pte_t pte, pte_t swp_pte)
1115 {
1116 #ifdef CONFIG_MEM_SOFT_DIRTY
1117         /*
1118          * When pte keeps soft dirty bit the pte generated
1119          * from swap entry does not has it, still it's same
1120          * pte from logical point of view.
1121          */
1122         pte_t swp_pte_dirty = pte_swp_mksoft_dirty(swp_pte);
1123         return pte_same(pte, swp_pte) || pte_same(pte, swp_pte_dirty);
1124 #else
1125         return pte_same(pte, swp_pte);
1126 #endif
1127 }
1128 
1129 /*
1130  * No need to decide whether this PTE shares the swap entry with others,
1131  * just let do_wp_page work it out if a write is requested later - to
1132  * force COW, vm_page_prot omits write permission from any private vma.
1133  */
1134 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1135                 unsigned long addr, swp_entry_t entry, struct page *page)
1136 {
1137         struct page *swapcache;
1138         struct mem_cgroup *memcg;
1139         spinlock_t *ptl;
1140         pte_t *pte;
1141         int ret = 1;
1142 
1143         swapcache = page;
1144         page = ksm_might_need_to_copy(page, vma, addr);
1145         if (unlikely(!page))
1146                 return -ENOMEM;
1147 
1148         if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg)) {
1149                 ret = -ENOMEM;
1150                 goto out_nolock;
1151         }
1152 
1153         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1154         if (unlikely(!maybe_same_pte(*pte, swp_entry_to_pte(entry)))) {
1155                 mem_cgroup_cancel_charge(page, memcg);
1156                 ret = 0;
1157                 goto out;
1158         }
1159 
1160         dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1161         inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1162         get_page(page);
1163         set_pte_at(vma->vm_mm, addr, pte,
1164                    pte_mkold(mk_pte(page, vma->vm_page_prot)));
1165         if (page == swapcache) {
1166                 page_add_anon_rmap(page, vma, addr);
1167                 mem_cgroup_commit_charge(page, memcg, true);
1168         } else { /* ksm created a completely new copy */
1169                 page_add_new_anon_rmap(page, vma, addr);
1170                 mem_cgroup_commit_charge(page, memcg, false);
1171                 lru_cache_add_active_or_unevictable(page, vma);
1172         }
1173         swap_free(entry);
1174         /*
1175          * Move the page to the active list so it is not
1176          * immediately swapped out again after swapon.
1177          */
1178         activate_page(page);
1179 out:
1180         pte_unmap_unlock(pte, ptl);
1181 out_nolock:
1182         if (page != swapcache) {
1183                 unlock_page(page);
1184                 put_page(page);
1185         }
1186         return ret;
1187 }
1188 
1189 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1190                                 unsigned long addr, unsigned long end,
1191                                 swp_entry_t entry, struct page *page)
1192 {
1193         pte_t swp_pte = swp_entry_to_pte(entry);
1194         pte_t *pte;
1195         int ret = 0;
1196 
1197         /*
1198          * We don't actually need pte lock while scanning for swp_pte: since
1199          * we hold page lock and mmap_sem, swp_pte cannot be inserted into the
1200          * page table while we're scanning; though it could get zapped, and on
1201          * some architectures (e.g. x86_32 with PAE) we might catch a glimpse
1202          * of unmatched parts which look like swp_pte, so unuse_pte must
1203          * recheck under pte lock.  Scanning without pte lock lets it be
1204          * preemptable whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
1205          */
1206         pte = pte_offset_map(pmd, addr);
1207         do {
1208                 /*
1209                  * swapoff spends a _lot_ of time in this loop!
1210                  * Test inline before going to call unuse_pte.
1211                  */
1212                 if (unlikely(maybe_same_pte(*pte, swp_pte))) {
1213                         pte_unmap(pte);
1214                         ret = unuse_pte(vma, pmd, addr, entry, page);
1215                         if (ret)
1216                                 goto out;
1217                         pte = pte_offset_map(pmd, addr);
1218                 }
1219         } while (pte++, addr += PAGE_SIZE, addr != end);
1220         pte_unmap(pte - 1);
1221 out:
1222         return ret;
1223 }
1224 
1225 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1226                                 unsigned long addr, unsigned long end,
1227                                 swp_entry_t entry, struct page *page)
1228 {
1229         pmd_t *pmd;
1230         unsigned long next;
1231         int ret;
1232 
1233         pmd = pmd_offset(pud, addr);
1234         do {
1235                 next = pmd_addr_end(addr, end);
1236                 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1237                         continue;
1238                 ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
1239                 if (ret)
1240                         return ret;
1241         } while (pmd++, addr = next, addr != end);
1242         return 0;
1243 }
1244 
1245 static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1246                                 unsigned long addr, unsigned long end,
1247                                 swp_entry_t entry, struct page *page)
1248 {
1249         pud_t *pud;
1250         unsigned long next;
1251         int ret;
1252 
1253         pud = pud_offset(pgd, addr);
1254         do {
1255                 next = pud_addr_end(addr, end);
1256                 if (pud_none_or_clear_bad(pud))
1257                         continue;
1258                 ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
1259                 if (ret)
1260                         return ret;
1261         } while (pud++, addr = next, addr != end);
1262         return 0;
1263 }
1264 
1265 static int unuse_vma(struct vm_area_struct *vma,
1266                                 swp_entry_t entry, struct page *page)
1267 {
1268         pgd_t *pgd;
1269         unsigned long addr, end, next;
1270         int ret;
1271 
1272         if (page_anon_vma(page)) {
1273                 addr = page_address_in_vma(page, vma);
1274                 if (addr == -EFAULT)
1275                         return 0;
1276                 else
1277                         end = addr + PAGE_SIZE;
1278         } else {
1279                 addr = vma->vm_start;
1280                 end = vma->vm_end;
1281         }
1282 
1283         pgd = pgd_offset(vma->vm_mm, addr);
1284         do {
1285                 next = pgd_addr_end(addr, end);
1286                 if (pgd_none_or_clear_bad(pgd))
1287                         continue;
1288                 ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1289                 if (ret)
1290                         return ret;
1291         } while (pgd++, addr = next, addr != end);
1292         return 0;
1293 }
1294 
1295 static int unuse_mm(struct mm_struct *mm,
1296                                 swp_entry_t entry, struct page *page)
1297 {
1298         struct vm_area_struct *vma;
1299         int ret = 0;
1300 
1301         if (!down_read_trylock(&mm->mmap_sem)) {
1302                 /*
1303                  * Activate page so shrink_inactive_list is unlikely to unmap
1304                  * its ptes while lock is dropped, so swapoff can make progress.
1305                  */
1306                 activate_page(page);
1307                 unlock_page(page);
1308                 down_read(&mm->mmap_sem);
1309                 lock_page(page);
1310         }
1311         for (vma = mm->mmap; vma; vma = vma->vm_next) {
1312                 if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
1313                         break;
1314         }
1315         up_read(&mm->mmap_sem);
1316         return (ret < 0)? ret: 0;
1317 }
1318 
1319 /*
1320  * Scan swap_map (or frontswap_map if frontswap parameter is true)
1321  * from current position to next entry still in use.
1322  * Recycle to start on reaching the end, returning 0 when empty.
1323  */
1324 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1325                                         unsigned int prev, bool frontswap)
1326 {
1327         unsigned int max = si->max;
1328         unsigned int i = prev;
1329         unsigned char count;
1330 
1331         /*
1332          * No need for swap_lock here: we're just looking
1333          * for whether an entry is in use, not modifying it; false
1334          * hits are okay, and sys_swapoff() has already prevented new
1335          * allocations from this area (while holding swap_lock).
1336          */
1337         for (;;) {
1338                 if (++i >= max) {
1339                         if (!prev) {
1340                                 i = 0;
1341                                 break;
1342                         }
1343                         /*
1344                          * No entries in use at top of swap_map,
1345                          * loop back to start and recheck there.
1346                          */
1347                         max = prev + 1;
1348                         prev = 0;
1349                         i = 1;
1350                 }
1351                 if (frontswap) {
1352                         if (frontswap_test(si, i))
1353                                 break;
1354                         else
1355                                 continue;
1356                 }
1357                 count = READ_ONCE(si->swap_map[i]);
1358                 if (count && swap_count(count) != SWAP_MAP_BAD)
1359                         break;
1360         }
1361         return i;
1362 }
1363 
1364 /*
1365  * We completely avoid races by reading each swap page in advance,
1366  * and then search for the process using it.  All the necessary
1367  * page table adjustments can then be made atomically.
1368  *
1369  * if the boolean frontswap is true, only unuse pages_to_unuse pages;
1370  * pages_to_unuse==0 means all pages; ignored if frontswap is false
1371  */
1372 int try_to_unuse(unsigned int type, bool frontswap,
1373                  unsigned long pages_to_unuse)
1374 {
1375         struct swap_info_struct *si = swap_info[type];
1376         struct mm_struct *start_mm;
1377         volatile unsigned char *swap_map; /* swap_map is accessed without
1378                                            * locking. Mark it as volatile
1379                                            * to prevent compiler doing
1380                                            * something odd.
1381                                            */
1382         unsigned char swcount;
1383         struct page *page;
1384         swp_entry_t entry;
1385         unsigned int i = 0;
1386         int retval = 0;
1387 
1388         /*
1389          * When searching mms for an entry, a good strategy is to
1390          * start at the first mm we freed the previous entry from
1391          * (though actually we don't notice whether we or coincidence
1392          * freed the entry).  Initialize this start_mm with a hold.
1393          *
1394          * A simpler strategy would be to start at the last mm we
1395          * freed the previous entry from; but that would take less
1396          * advantage of mmlist ordering, which clusters forked mms
1397          * together, child after parent.  If we race with dup_mmap(), we
1398          * prefer to resolve parent before child, lest we miss entries
1399          * duplicated after we scanned child: using last mm would invert
1400          * that.
1401          */
1402         start_mm = &init_mm;
1403         atomic_inc(&init_mm.mm_users);
1404 
1405         /*
1406          * Keep on scanning until all entries have gone.  Usually,
1407          * one pass through swap_map is enough, but not necessarily:
1408          * there are races when an instance of an entry might be missed.
1409          */
1410         while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
1411                 if (signal_pending(current)) {
1412                         retval = -EINTR;
1413                         break;
1414                 }
1415 
1416                 /*
1417                  * Get a page for the entry, using the existing swap
1418                  * cache page if there is one.  Otherwise, get a clean
1419                  * page and read the swap into it.
1420                  */
1421                 swap_map = &si->swap_map[i];
1422                 entry = swp_entry(type, i);
1423                 page = read_swap_cache_async(entry,
1424                                         GFP_HIGHUSER_MOVABLE, NULL, 0);
1425                 if (!page) {
1426                         /*
1427                          * Either swap_duplicate() failed because entry
1428                          * has been freed independently, and will not be
1429                          * reused since sys_swapoff() already disabled
1430                          * allocation from here, or alloc_page() failed.
1431                          */
1432                         swcount = *swap_map;
1433                         /*
1434                          * We don't hold lock here, so the swap entry could be
1435                          * SWAP_MAP_BAD (when the cluster is discarding).
1436                          * Instead of fail out, We can just skip the swap
1437                          * entry because swapoff will wait for discarding
1438                          * finish anyway.
1439                          */
1440                         if (!swcount || swcount == SWAP_MAP_BAD)
1441                                 continue;
1442                         retval = -ENOMEM;
1443                         break;
1444                 }
1445 
1446                 /*
1447                  * Don't hold on to start_mm if it looks like exiting.
1448                  */
1449                 if (atomic_read(&start_mm->mm_users) == 1) {
1450                         mmput(start_mm);
1451                         start_mm = &init_mm;
1452                         atomic_inc(&init_mm.mm_users);
1453                 }
1454 
1455                 /*
1456                  * Wait for and lock page.  When do_swap_page races with
1457                  * try_to_unuse, do_swap_page can handle the fault much
1458                  * faster than try_to_unuse can locate the entry.  This
1459                  * apparently redundant "wait_on_page_locked" lets try_to_unuse
1460                  * defer to do_swap_page in such a case - in some tests,
1461                  * do_swap_page and try_to_unuse repeatedly compete.
1462                  */
1463                 wait_on_page_locked(page);
1464                 wait_on_page_writeback(page);
1465                 lock_page(page);
1466                 wait_on_page_writeback(page);
1467 
1468                 /*
1469                  * Remove all references to entry.
1470                  */
1471                 swcount = *swap_map;
1472                 if (swap_count(swcount) == SWAP_MAP_SHMEM) {
1473                         retval = shmem_unuse(entry, page);
1474                         /* page has already been unlocked and released */
1475                         if (retval < 0)
1476                                 break;
1477                         continue;
1478                 }
1479                 if (swap_count(swcount) && start_mm != &init_mm)
1480                         retval = unuse_mm(start_mm, entry, page);
1481 
1482                 if (swap_count(*swap_map)) {
1483                         int set_start_mm = (*swap_map >= swcount);
1484                         struct list_head *p = &start_mm->mmlist;
1485                         struct mm_struct *new_start_mm = start_mm;
1486                         struct mm_struct *prev_mm = start_mm;
1487                         struct mm_struct *mm;
1488 
1489                         atomic_inc(&new_start_mm->mm_users);
1490                         atomic_inc(&prev_mm->mm_users);
1491                         spin_lock(&mmlist_lock);
1492                         while (swap_count(*swap_map) && !retval &&
1493                                         (p = p->next) != &start_mm->mmlist) {
1494                                 mm = list_entry(p, struct mm_struct, mmlist);
1495                                 if (!atomic_inc_not_zero(&mm->mm_users))
1496                                         continue;
1497                                 spin_unlock(&mmlist_lock);
1498                                 mmput(prev_mm);
1499                                 prev_mm = mm;
1500 
1501                                 cond_resched();
1502 
1503                                 swcount = *swap_map;
1504                                 if (!swap_count(swcount)) /* any usage ? */
1505                                         ;
1506                                 else if (mm == &init_mm)
1507                                         set_start_mm = 1;
1508                                 else
1509                                         retval = unuse_mm(mm, entry, page);
1510 
1511                                 if (set_start_mm && *swap_map < swcount) {
1512                                         mmput(new_start_mm);
1513                                         atomic_inc(&mm->mm_users);
1514                                         new_start_mm = mm;
1515                                         set_start_mm = 0;
1516                                 }
1517                                 spin_lock(&mmlist_lock);
1518                         }
1519                         spin_unlock(&mmlist_lock);
1520                         mmput(prev_mm);
1521                         mmput(start_mm);
1522                         start_mm = new_start_mm;
1523                 }
1524                 if (retval) {
1525                         unlock_page(page);
1526                         page_cache_release(page);
1527                         break;
1528                 }
1529 
1530                 /*
1531                  * If a reference remains (rare), we would like to leave
1532                  * the page in the swap cache; but try_to_unmap could
1533                  * then re-duplicate the entry once we drop page lock,
1534                  * so we might loop indefinitely; also, that page could
1535                  * not be swapped out to other storage meanwhile.  So:
1536                  * delete from cache even if there's another reference,
1537                  * after ensuring that the data has been saved to disk -
1538                  * since if the reference remains (rarer), it will be
1539                  * read from disk into another page.  Splitting into two
1540                  * pages would be incorrect if swap supported "shared
1541                  * private" pages, but they are handled by tmpfs files.
1542                  *
1543                  * Given how unuse_vma() targets one particular offset
1544                  * in an anon_vma, once the anon_vma has been determined,
1545                  * this splitting happens to be just what is needed to
1546                  * handle where KSM pages have been swapped out: re-reading
1547                  * is unnecessarily slow, but we can fix that later on.
1548                  */
1549                 if (swap_count(*swap_map) &&
1550                      PageDirty(page) && PageSwapCache(page)) {
1551                         struct writeback_control wbc = {
1552                                 .sync_mode = WB_SYNC_NONE,
1553                         };
1554 
1555                         swap_writepage(page, &wbc);
1556                         lock_page(page);
1557                         wait_on_page_writeback(page);
1558                 }
1559 
1560                 /*
1561                  * It is conceivable that a racing task removed this page from
1562                  * swap cache just before we acquired the page lock at the top,
1563                  * or while we dropped it in unuse_mm().  The page might even
1564                  * be back in swap cache on another swap area: that we must not
1565                  * delete, since it may not have been written out to swap yet.
1566                  */
1567                 if (PageSwapCache(page) &&
1568                     likely(page_private(page) == entry.val))
1569                         delete_from_swap_cache(page);
1570 
1571                 /*
1572                  * So we could skip searching mms once swap count went
1573                  * to 1, we did not mark any present ptes as dirty: must
1574                  * mark page dirty so shrink_page_list will preserve it.
1575                  */
1576                 SetPageDirty(page);
1577                 unlock_page(page);
1578                 page_cache_release(page);
1579 
1580                 /*
1581                  * Make sure that we aren't completely killing
1582                  * interactive performance.
1583                  */
1584                 cond_resched();
1585                 if (frontswap && pages_to_unuse > 0) {
1586                         if (!--pages_to_unuse)
1587                                 break;
1588                 }
1589         }
1590 
1591         mmput(start_mm);
1592         return retval;
1593 }
1594 
1595 /*
1596  * After a successful try_to_unuse, if no swap is now in use, we know
1597  * we can empty the mmlist.  swap_lock must be held on entry and exit.
1598  * Note that mmlist_lock nests inside swap_lock, and an mm must be
1599  * added to the mmlist just after page_duplicate - before would be racy.
1600  */
1601 static void drain_mmlist(void)
1602 {
1603         struct list_head *p, *next;
1604         unsigned int type;
1605 
1606         for (type = 0; type < nr_swapfiles; type++)
1607                 if (swap_info[type]->inuse_pages)
1608                         return;
1609         spin_lock(&mmlist_lock);
1610         list_for_each_safe(p, next, &init_mm.mmlist)
1611                 list_del_init(p);
1612         spin_unlock(&mmlist_lock);
1613 }
1614 
1615 /*
1616  * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
1617  * corresponds to page offset for the specified swap entry.
1618  * Note that the type of this function is sector_t, but it returns page offset
1619  * into the bdev, not sector offset.
1620  */
1621 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
1622 {
1623         struct swap_info_struct *sis;
1624         struct swap_extent *start_se;
1625         struct swap_extent *se;
1626         pgoff_t offset;
1627 
1628         sis = swap_info[swp_type(entry)];
1629         *bdev = sis->bdev;
1630 
1631         offset = swp_offset(entry);
1632         start_se = sis->curr_swap_extent;
1633         se = start_se;
1634 
1635         for ( ; ; ) {
1636                 struct list_head *lh;
1637 
1638                 if (se->start_page <= offset &&
1639                                 offset < (se->start_page + se->nr_pages)) {
1640                         return se->start_block + (offset - se->start_page);
1641                 }
1642                 lh = se->list.next;
1643                 se = list_entry(lh, struct swap_extent, list);
1644                 sis->curr_swap_extent = se;
1645                 BUG_ON(se == start_se);         /* It *must* be present */
1646         }
1647 }
1648 
1649 /*
1650  * Returns the page offset into bdev for the specified page's swap entry.
1651  */
1652 sector_t map_swap_page(struct page *page, struct block_device **bdev)
1653 {
1654         swp_entry_t entry;
1655         entry.val = page_private(page);
1656         return map_swap_entry(entry, bdev);
1657 }
1658 
1659 /*
1660  * Free all of a swapdev's extent information
1661  */
1662 static void destroy_swap_extents(struct swap_info_struct *sis)
1663 {
1664         while (!list_empty(&sis->first_swap_extent.list)) {
1665                 struct swap_extent *se;
1666 
1667                 se = list_entry(sis->first_swap_extent.list.next,
1668                                 struct swap_extent, list);
1669                 list_del(&se->list);
1670                 kfree(se);
1671         }
1672 
1673         if (sis->flags & SWP_FILE) {
1674                 struct file *swap_file = sis->swap_file;
1675                 struct address_space *mapping = swap_file->f_mapping;
1676 
1677                 sis->flags &= ~SWP_FILE;
1678                 mapping->a_ops->swap_deactivate(swap_file);
1679         }
1680 }
1681 
1682 /*
1683  * Add a block range (and the corresponding page range) into this swapdev's
1684  * extent list.  The extent list is kept sorted in page order.
1685  *
1686  * This function rather assumes that it is called in ascending page order.
1687  */
1688 int
1689 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1690                 unsigned long nr_pages, sector_t start_block)
1691 {
1692         struct swap_extent *se;
1693         struct swap_extent *new_se;
1694         struct list_head *lh;
1695 
1696         if (start_page == 0) {
1697                 se = &sis->first_swap_extent;
1698                 sis->curr_swap_extent = se;
1699                 se->start_page = 0;
1700                 se->nr_pages = nr_pages;
1701                 se->start_block = start_block;
1702                 return 1;
1703         } else {
1704                 lh = sis->first_swap_extent.list.prev;  /* Highest extent */
1705                 se = list_entry(lh, struct swap_extent, list);
1706                 BUG_ON(se->start_page + se->nr_pages != start_page);
1707                 if (se->start_block + se->nr_pages == start_block) {
1708                         /* Merge it */
1709                         se->nr_pages += nr_pages;
1710                         return 0;
1711                 }
1712         }
1713 
1714         /*
1715          * No merge.  Insert a new extent, preserving ordering.
1716          */
1717         new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1718         if (new_se == NULL)
1719                 return -ENOMEM;
1720         new_se->start_page = start_page;
1721         new_se->nr_pages = nr_pages;
1722         new_se->start_block = start_block;
1723 
1724         list_add_tail(&new_se->list, &sis->first_swap_extent.list);
1725         return 1;
1726 }
1727 
1728 /*
1729  * A `swap extent' is a simple thing which maps a contiguous range of pages
1730  * onto a contiguous range of disk blocks.  An ordered list of swap extents
1731  * is built at swapon time and is then used at swap_writepage/swap_readpage
1732  * time for locating where on disk a page belongs.
1733  *
1734  * If the swapfile is an S_ISBLK block device, a single extent is installed.
1735  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
1736  * swap files identically.
1737  *
1738  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
1739  * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
1740  * swapfiles are handled *identically* after swapon time.
1741  *
1742  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
1743  * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
1744  * some stray blocks are found which do not fall within the PAGE_SIZE alignment
1745  * requirements, they are simply tossed out - we will never use those blocks
1746  * for swapping.
1747  *
1748  * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
1749  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
1750  * which will scribble on the fs.
1751  *
1752  * The amount of disk space which a single swap extent represents varies.
1753  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
1754  * extents in the list.  To avoid much list walking, we cache the previous
1755  * search location in `curr_swap_extent', and start new searches from there.
1756  * This is extremely effective.  The average number of iterations in
1757  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
1758  */
1759 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1760 {
1761         struct file *swap_file = sis->swap_file;
1762         struct address_space *mapping = swap_file->f_mapping;
1763         struct inode *inode = mapping->host;
1764         int ret;
1765 
1766         if (S_ISBLK(inode->i_mode)) {
1767                 ret = add_swap_extent(sis, 0, sis->max, 0);
1768                 *span = sis->pages;
1769                 return ret;
1770         }
1771 
1772         if (mapping->a_ops->swap_activate) {
1773                 ret = mapping->a_ops->swap_activate(sis, swap_file, span);
1774                 if (!ret) {
1775                         sis->flags |= SWP_FILE;
1776                         ret = add_swap_extent(sis, 0, sis->max, 0);
1777                         *span = sis->pages;
1778                 }
1779                 return ret;
1780         }
1781 
1782         return generic_swapfile_activate(sis, swap_file, span);
1783 }
1784 
1785 static void _enable_swap_info(struct swap_info_struct *p, int prio,
1786                                 unsigned char *swap_map,
1787                                 struct swap_cluster_info *cluster_info)
1788 {
1789         if (prio >= 0)
1790                 p->prio = prio;
1791         else
1792                 p->prio = --least_priority;
1793         /*
1794          * the plist prio is negated because plist ordering is
1795          * low-to-high, while swap ordering is high-to-low
1796          */
1797         p->list.prio = -p->prio;
1798         p->avail_list.prio = -p->prio;
1799         p->swap_map = swap_map;
1800         p->cluster_info = cluster_info;
1801         p->flags |= SWP_WRITEOK;
1802         atomic_long_add(p->pages, &nr_swap_pages);
1803         total_swap_pages += p->pages;
1804 
1805         assert_spin_locked(&swap_lock);
1806         /*
1807          * both lists are plists, and thus priority ordered.
1808          * swap_active_head needs to be priority ordered for swapoff(),
1809          * which on removal of any swap_info_struct with an auto-assigned
1810          * (i.e. negative) priority increments the auto-assigned priority
1811          * of any lower-priority swap_info_structs.
1812          * swap_avail_head needs to be priority ordered for get_swap_page(),
1813          * which allocates swap pages from the highest available priority
1814          * swap_info_struct.
1815          */
1816         plist_add(&p->list, &swap_active_head);
1817         spin_lock(&swap_avail_lock);
1818         plist_add(&p->avail_list, &swap_avail_head);
1819         spin_unlock(&swap_avail_lock);
1820 }
1821 
1822 static void enable_swap_info(struct swap_info_struct *p, int prio,
1823                                 unsigned char *swap_map,
1824                                 struct swap_cluster_info *cluster_info,
1825                                 unsigned long *frontswap_map)
1826 {
1827         frontswap_init(p->type, frontswap_map);
1828         spin_lock(&swap_lock);
1829         spin_lock(&p->lock);
1830          _enable_swap_info(p, prio, swap_map, cluster_info);
1831         spin_unlock(&p->lock);
1832         spin_unlock(&swap_lock);
1833 }
1834 
1835 static void reinsert_swap_info(struct swap_info_struct *p)
1836 {
1837         spin_lock(&swap_lock);
1838         spin_lock(&p->lock);
1839         _enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
1840         spin_unlock(&p->lock);
1841         spin_unlock(&swap_lock);
1842 }
1843 
1844 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
1845 {
1846         struct swap_info_struct *p = NULL;
1847         unsigned char *swap_map;
1848         struct swap_cluster_info *cluster_info;
1849         unsigned long *frontswap_map;
1850         struct file *swap_file, *victim;
1851         struct address_space *mapping;
1852         struct inode *inode;
1853         struct filename *pathname;
1854         int err, found = 0;
1855         unsigned int old_block_size;
1856 
1857         if (!capable(CAP_SYS_ADMIN))
1858                 return -EPERM;
1859 
1860         BUG_ON(!current->mm);
1861 
1862         pathname = getname(specialfile);
1863         if (IS_ERR(pathname))
1864                 return PTR_ERR(pathname);
1865 
1866         victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
1867         err = PTR_ERR(victim);
1868         if (IS_ERR(victim))
1869                 goto out;
1870 
1871         mapping = victim->f_mapping;
1872         spin_lock(&swap_lock);
1873         plist_for_each_entry(p, &swap_active_head, list) {
1874                 if (p->flags & SWP_WRITEOK) {
1875                         if (p->swap_file->f_mapping == mapping) {
1876                                 found = 1;
1877                                 break;
1878                         }
1879                 }
1880         }
1881         if (!found) {
1882                 err = -EINVAL;
1883                 spin_unlock(&swap_lock);
1884                 goto out_dput;
1885         }
1886         if (!security_vm_enough_memory_mm(current->mm, p->pages))
1887                 vm_unacct_memory(p->pages);
1888         else {
1889                 err = -ENOMEM;
1890                 spin_unlock(&swap_lock);
1891                 goto out_dput;
1892         }
1893         spin_lock(&swap_avail_lock);
1894         plist_del(&p->avail_list, &swap_avail_head);
1895         spin_unlock(&swap_avail_lock);
1896         spin_lock(&p->lock);
1897         if (p->prio < 0) {
1898                 struct swap_info_struct *si = p;
1899 
1900                 plist_for_each_entry_continue(si, &swap_active_head, list) {
1901                         si->prio++;
1902                         si->list.prio--;
1903                         si->avail_list.prio--;
1904                 }
1905                 least_priority++;
1906         }
1907         plist_del(&p->list, &swap_active_head);
1908         atomic_long_sub(p->pages, &nr_swap_pages);
1909         total_swap_pages -= p->pages;
1910         p->flags &= ~SWP_WRITEOK;
1911         spin_unlock(&p->lock);
1912         spin_unlock(&swap_lock);
1913 
1914         set_current_oom_origin();
1915         err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
1916         clear_current_oom_origin();
1917 
1918         if (err) {
1919                 /* re-insert swap space back into swap_list */
1920                 reinsert_swap_info(p);
1921                 goto out_dput;
1922         }
1923 
1924         flush_work(&p->discard_work);
1925 
1926         destroy_swap_extents(p);
1927         if (p->flags & SWP_CONTINUED)
1928                 free_swap_count_continuations(p);
1929 
1930         mutex_lock(&swapon_mutex);
1931         spin_lock(&swap_lock);
1932         spin_lock(&p->lock);
1933         drain_mmlist();
1934 
1935         /* wait for anyone still in scan_swap_map */
1936         p->highest_bit = 0;             /* cuts scans short */
1937         while (p->flags >= SWP_SCANNING) {
1938                 spin_unlock(&p->lock);
1939                 spin_unlock(&swap_lock);
1940                 schedule_timeout_uninterruptible(1);
1941                 spin_lock(&swap_lock);
1942                 spin_lock(&p->lock);
1943         }
1944 
1945         swap_file = p->swap_file;
1946         old_block_size = p->old_block_size;
1947         p->swap_file = NULL;
1948         p->max = 0;
1949         swap_map = p->swap_map;
1950         p->swap_map = NULL;
1951         cluster_info = p->cluster_info;
1952         p->cluster_info = NULL;
1953         frontswap_map = frontswap_map_get(p);
1954         spin_unlock(&p->lock);
1955         spin_unlock(&swap_lock);
1956         frontswap_invalidate_area(p->type);
1957         frontswap_map_set(p, NULL);
1958         mutex_unlock(&swapon_mutex);
1959         free_percpu(p->percpu_cluster);
1960         p->percpu_cluster = NULL;
1961         vfree(swap_map);
1962         vfree(cluster_info);
1963         vfree(frontswap_map);
1964         /* Destroy swap account information */
1965         swap_cgroup_swapoff(p->type);
1966 
1967         inode = mapping->host;
1968         if (S_ISBLK(inode->i_mode)) {
1969                 struct block_device *bdev = I_BDEV(inode);
1970                 set_blocksize(bdev, old_block_size);
1971                 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
1972         } else {
1973                 mutex_lock(&inode->i_mutex);
1974                 inode->i_flags &= ~S_SWAPFILE;
1975                 mutex_unlock(&inode->i_mutex);
1976         }
1977         filp_close(swap_file, NULL);
1978 
1979         /*
1980          * Clear the SWP_USED flag after all resources are freed so that swapon
1981          * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
1982          * not hold p->lock after we cleared its SWP_WRITEOK.
1983          */
1984         spin_lock(&swap_lock);
1985         p->flags = 0;
1986         spin_unlock(&swap_lock);
1987 
1988         err = 0;
1989         atomic_inc(&proc_poll_event);
1990         wake_up_interruptible(&proc_poll_wait);
1991 
1992 out_dput:
1993         filp_close(victim, NULL);
1994 out:
1995         putname(pathname);
1996         return err;
1997 }
1998 
1999 #ifdef CONFIG_PROC_FS
2000 static unsigned swaps_poll(struct file *file, poll_table *wait)
2001 {
2002         struct seq_file *seq = file->private_data;
2003 
2004         poll_wait(file, &proc_poll_wait, wait);
2005 
2006         if (seq->poll_event != atomic_read(&proc_poll_event)) {
2007                 seq->poll_event = atomic_read(&proc_poll_event);
2008                 return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
2009         }
2010 
2011         return POLLIN | POLLRDNORM;
2012 }
2013 
2014 /* iterator */
2015 static void *swap_start(struct seq_file *swap, loff_t *pos)
2016 {
2017         struct swap_info_struct *si;
2018         int type;
2019         loff_t l = *pos;
2020 
2021         mutex_lock(&swapon_mutex);
2022 
2023         if (!l)
2024                 return SEQ_START_TOKEN;
2025 
2026         for (type = 0; type < nr_swapfiles; type++) {
2027                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
2028                 si = swap_info[type];
2029                 if (!(si->flags & SWP_USED) || !si->swap_map)
2030                         continue;
2031                 if (!--l)
2032                         return si;
2033         }
2034 
2035         return NULL;
2036 }
2037 
2038 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2039 {
2040         struct swap_info_struct *si = v;
2041         int type;
2042 
2043         if (v == SEQ_START_TOKEN)
2044                 type = 0;
2045         else
2046                 type = si->type + 1;
2047 
2048         for (; type < nr_swapfiles; type++) {
2049                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
2050                 si = swap_info[type];
2051                 if (!(si->flags & SWP_USED) || !si->swap_map)
2052                         continue;
2053                 ++*pos;
2054                 return si;
2055         }
2056 
2057         return NULL;
2058 }
2059 
2060 static void swap_stop(struct seq_file *swap, void *v)
2061 {
2062         mutex_unlock(&swapon_mutex);
2063 }
2064 
2065 static int swap_show(struct seq_file *swap, void *v)
2066 {
2067         struct swap_info_struct *si = v;
2068         struct file *file;
2069         int len;
2070 
2071         if (si == SEQ_START_TOKEN) {
2072                 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
2073                 return 0;
2074         }
2075 
2076         file = si->swap_file;
2077         len = seq_file_path(swap, file, " \t\n\\");
2078         seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2079                         len < 40 ? 40 - len : 1, " ",
2080                         S_ISBLK(file_inode(file)->i_mode) ?
2081                                 "partition" : "file\t",
2082                         si->pages << (PAGE_SHIFT - 10),
2083                         si->inuse_pages << (PAGE_SHIFT - 10),
2084                         si->prio);
2085         return 0;
2086 }
2087 
2088 static const struct seq_operations swaps_op = {
2089         .start =        swap_start,
2090         .next =         swap_next,
2091         .stop =         swap_stop,
2092         .show =         swap_show
2093 };
2094 
2095 static int swaps_open(struct inode *inode, struct file *file)
2096 {
2097         struct seq_file *seq;
2098         int ret;
2099 
2100         ret = seq_open(file, &swaps_op);
2101         if (ret)
2102                 return ret;
2103 
2104         seq = file->private_data;
2105         seq->poll_event = atomic_read(&proc_poll_event);
2106         return 0;
2107 }
2108 
2109 static const struct file_operations proc_swaps_operations = {
2110         .open           = swaps_open,
2111         .read           = seq_read,
2112         .llseek         = seq_lseek,
2113         .release        = seq_release,
2114         .poll           = swaps_poll,
2115 };
2116 
2117 static int __init procswaps_init(void)
2118 {
2119         proc_create("swaps", 0, NULL, &proc_swaps_operations);
2120         return 0;
2121 }
2122 __initcall(procswaps_init);
2123 #endif /* CONFIG_PROC_FS */
2124 
2125 #ifdef MAX_SWAPFILES_CHECK
2126 static int __init max_swapfiles_check(void)
2127 {
2128         MAX_SWAPFILES_CHECK();
2129         return 0;
2130 }
2131 late_initcall(max_swapfiles_check);
2132 #endif
2133 
2134 static struct swap_info_struct *alloc_swap_info(void)
2135 {
2136         struct swap_info_struct *p;
2137         unsigned int type;
2138 
2139         p = kzalloc(sizeof(*p), GFP_KERNEL);
2140         if (!p)
2141                 return ERR_PTR(-ENOMEM);
2142 
2143         spin_lock(&swap_lock);
2144         for (type = 0; type < nr_swapfiles; type++) {
2145                 if (!(swap_info[type]->flags & SWP_USED))
2146                         break;
2147         }
2148         if (type >= MAX_SWAPFILES) {
2149                 spin_unlock(&swap_lock);
2150                 kfree(p);
2151                 return ERR_PTR(-EPERM);
2152         }
2153         if (type >= nr_swapfiles) {
2154                 p->type = type;
2155                 swap_info[type] = p;
2156                 /*
2157                  * Write swap_info[type] before nr_swapfiles, in case a
2158                  * racing procfs swap_start() or swap_next() is reading them.
2159                  * (We never shrink nr_swapfiles, we never free this entry.)
2160                  */
2161                 smp_wmb();
2162                 nr_swapfiles++;
2163         } else {
2164                 kfree(p);
2165                 p = swap_info[type];
2166                 /*
2167                  * Do not memset this entry: a racing procfs swap_next()
2168                  * would be relying on p->type to remain valid.
2169                  */
2170         }
2171         INIT_LIST_HEAD(&p->first_swap_extent.list);
2172         plist_node_init(&p->list, 0);
2173         plist_node_init(&p->avail_list, 0);
2174         p->flags = SWP_USED;
2175         spin_unlock(&swap_lock);
2176         spin_lock_init(&p->lock);
2177 
2178         return p;
2179 }
2180 
2181 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2182 {
2183         int error;
2184 
2185         if (S_ISBLK(inode->i_mode)) {
2186                 p->bdev = bdgrab(I_BDEV(inode));
2187                 error = blkdev_get(p->bdev,
2188                                    FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2189                 if (error < 0) {
2190                         p->bdev = NULL;
2191                         return error;
2192                 }
2193                 p->old_block_size = block_size(p->bdev);
2194                 error = set_blocksize(p->bdev, PAGE_SIZE);
2195                 if (error < 0)
2196                         return error;
2197                 p->flags |= SWP_BLKDEV;
2198         } else if (S_ISREG(inode->i_mode)) {
2199                 p->bdev = inode->i_sb->s_bdev;
2200                 mutex_lock(&inode->i_mutex);
2201                 if (IS_SWAPFILE(inode))
2202                         return -EBUSY;
2203         } else
2204                 return -EINVAL;
2205 
2206         return 0;
2207 }
2208 
2209 static unsigned long read_swap_header(struct swap_info_struct *p,
2210                                         union swap_header *swap_header,
2211                                         struct inode *inode)
2212 {
2213         int i;
2214         unsigned long maxpages;
2215         unsigned long swapfilepages;
2216         unsigned long last_page;
2217 
2218         if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2219                 pr_err("Unable to find swap-space signature\n");
2220                 return 0;
2221         }
2222 
2223         /* swap partition endianess hack... */
2224         if (swab32(swap_header->info.version) == 1) {
2225                 swab32s(&swap_header->info.version);
2226                 swab32s(&swap_header->info.last_page);
2227                 swab32s(&swap_header->info.nr_badpages);
2228                 for (i = 0; i < swap_header->info.nr_badpages; i++)
2229                         swab32s(&swap_header->info.badpages[i]);
2230         }
2231         /* Check the swap header's sub-version */
2232         if (swap_header->info.version != 1) {
2233                 pr_warn("Unable to handle swap header version %d\n",
2234                         swap_header->info.version);
2235                 return 0;
2236         }
2237 
2238         p->lowest_bit  = 1;
2239         p->cluster_next = 1;
2240         p->cluster_nr = 0;
2241 
2242         /*
2243          * Find out how many pages are allowed for a single swap
2244          * device. There are two limiting factors: 1) the number
2245          * of bits for the swap offset in the swp_entry_t type, and
2246          * 2) the number of bits in the swap pte as defined by the
2247          * different architectures. In order to find the
2248          * largest possible bit mask, a swap entry with swap type 0
2249          * and swap offset ~0UL is created, encoded to a swap pte,
2250          * decoded to a swp_entry_t again, and finally the swap
2251          * offset is extracted. This will mask all the bits from
2252          * the initial ~0UL mask that can't be encoded in either
2253          * the swp_entry_t or the architecture definition of a
2254          * swap pte.
2255          */
2256         maxpages = swp_offset(pte_to_swp_entry(
2257                         swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2258         last_page = swap_header->info.last_page;
2259         if (last_page > maxpages) {
2260                 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2261                         maxpages << (PAGE_SHIFT - 10),
2262                         last_page << (PAGE_SHIFT - 10));
2263         }
2264         if (maxpages > last_page) {
2265                 maxpages = last_page + 1;
2266                 /* p->max is an unsigned int: don't overflow it */
2267                 if ((unsigned int)maxpages == 0)
2268                         maxpages = UINT_MAX;
2269         }
2270         p->highest_bit = maxpages - 1;
2271 
2272         if (!maxpages)
2273                 return 0;
2274         swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2275         if (swapfilepages && maxpages > swapfilepages) {
2276                 pr_warn("Swap area shorter than signature indicates\n");
2277                 return 0;
2278         }
2279         if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2280                 return 0;
2281         if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2282                 return 0;
2283 
2284         return maxpages;
2285 }
2286 
2287 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2288                                         union swap_header *swap_header,
2289                                         unsigned char *swap_map,
2290                                         struct swap_cluster_info *cluster_info,
2291                                         unsigned long maxpages,
2292                                         sector_t *span)
2293 {
2294         int i;
2295         unsigned int nr_good_pages;
2296         int nr_extents;
2297         unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2298         unsigned long idx = p->cluster_next / SWAPFILE_CLUSTER;
2299 
2300         nr_good_pages = maxpages - 1;   /* omit header page */
2301 
2302         cluster_set_null(&p->free_cluster_head);
2303         cluster_set_null(&p->free_cluster_tail);
2304         cluster_set_null(&p->discard_cluster_head);
2305         cluster_set_null(&p->discard_cluster_tail);
2306 
2307         for (i = 0; i < swap_header->info.nr_badpages; i++) {
2308                 unsigned int page_nr = swap_header->info.badpages[i];
2309                 if (page_nr == 0 || page_nr > swap_header->info.last_page)
2310                         return -EINVAL;
2311                 if (page_nr < maxpages) {
2312                         swap_map[page_nr] = SWAP_MAP_BAD;
2313                         nr_good_pages--;
2314                         /*
2315                          * Haven't marked the cluster free yet, no list
2316                          * operation involved
2317                          */
2318                         inc_cluster_info_page(p, cluster_info, page_nr);
2319                 }
2320         }
2321 
2322         /* Haven't marked the cluster free yet, no list operation involved */
2323         for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2324                 inc_cluster_info_page(p, cluster_info, i);
2325 
2326         if (nr_good_pages) {
2327                 swap_map[0] = SWAP_MAP_BAD;
2328                 /*
2329                  * Not mark the cluster free yet, no list
2330                  * operation involved
2331                  */
2332                 inc_cluster_info_page(p, cluster_info, 0);
2333                 p->max = maxpages;
2334                 p->pages = nr_good_pages;
2335                 nr_extents = setup_swap_extents(p, span);
2336                 if (nr_extents < 0)
2337                         return nr_extents;
2338                 nr_good_pages = p->pages;
2339         }
2340         if (!nr_good_pages) {
2341                 pr_warn("Empty swap-file\n");
2342                 return -EINVAL;
2343         }
2344 
2345         if (!cluster_info)
2346                 return nr_extents;
2347 
2348         for (i = 0; i < nr_clusters; i++) {
2349                 if (!cluster_count(&cluster_info[idx])) {
2350                         cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2351                         if (cluster_is_null(&p->free_cluster_head)) {
2352                                 cluster_set_next_flag(&p->free_cluster_head,
2353                                                                 idx, 0);
2354                                 cluster_set_next_flag(&p->free_cluster_tail,
2355                                                                 idx, 0);
2356                         } else {
2357                                 unsigned int tail;
2358 
2359                                 tail = cluster_next(&p->free_cluster_tail);
2360                                 cluster_set_next(&cluster_info[tail], idx);
2361                                 cluster_set_next_flag(&p->free_cluster_tail,
2362                                                                 idx, 0);
2363                         }
2364                 }
2365                 idx++;
2366                 if (idx == nr_clusters)
2367                         idx = 0;
2368         }
2369         return nr_extents;
2370 }
2371 
2372 /*
2373  * Helper to sys_swapon determining if a given swap
2374  * backing device queue supports DISCARD operations.
2375  */
2376 static bool swap_discardable(struct swap_info_struct *si)
2377 {
2378         struct request_queue *q = bdev_get_queue(si->bdev);
2379 
2380         if (!q || !blk_queue_discard(q))
2381                 return false;
2382 
2383         return true;
2384 }
2385 
2386 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2387 {
2388         struct swap_info_struct *p;
2389         struct filename *name;
2390         struct file *swap_file = NULL;
2391         struct address_space *mapping;
2392         int prio;
2393         int error;
2394         union swap_header *swap_header;
2395         int nr_extents;
2396         sector_t span;
2397         unsigned long maxpages;
2398         unsigned char *swap_map = NULL;
2399         struct swap_cluster_info *cluster_info = NULL;
2400         unsigned long *frontswap_map = NULL;
2401         struct page *page = NULL;
2402         struct inode *inode = NULL;
2403 
2404         if (swap_flags & ~SWAP_FLAGS_VALID)
2405                 return -EINVAL;
2406 
2407         if (!capable(CAP_SYS_ADMIN))
2408                 return -EPERM;
2409 
2410         p = alloc_swap_info();
2411         if (IS_ERR(p))
2412                 return PTR_ERR(p);
2413 
2414         INIT_WORK(&p->discard_work, swap_discard_work);
2415 
2416         name = getname(specialfile);
2417         if (IS_ERR(name)) {
2418                 error = PTR_ERR(name);
2419                 name = NULL;
2420                 goto bad_swap;
2421         }
2422         swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
2423         if (IS_ERR(swap_file)) {
2424                 error = PTR_ERR(swap_file);
2425                 swap_file = NULL;
2426                 goto bad_swap;
2427         }
2428 
2429         p->swap_file = swap_file;
2430         mapping = swap_file->f_mapping;
2431         inode = mapping->host;
2432 
2433         /* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */
2434         error = claim_swapfile(p, inode);
2435         if (unlikely(error))
2436                 goto bad_swap;
2437 
2438         /*
2439          * Read the swap header.
2440          */
2441         if (!mapping->a_ops->readpage) {
2442                 error = -EINVAL;
2443                 goto bad_swap;
2444         }
2445         page = read_mapping_page(mapping, 0, swap_file);
2446         if (IS_ERR(page)) {
2447                 error = PTR_ERR(page);
2448                 goto bad_swap;
2449         }
2450         swap_header = kmap(page);
2451 
2452         maxpages = read_swap_header(p, swap_header, inode);
2453         if (unlikely(!maxpages)) {
2454                 error = -EINVAL;
2455                 goto bad_swap;
2456         }
2457 
2458         /* OK, set up the swap map and apply the bad block list */
2459         swap_map = vzalloc(maxpages);
2460         if (!swap_map) {
2461                 error = -ENOMEM;
2462                 goto bad_swap;
2463         }
2464         if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
2465                 int cpu;
2466 
2467                 p->flags |= SWP_SOLIDSTATE;
2468                 /*
2469                  * select a random position to start with to help wear leveling
2470                  * SSD
2471                  */
2472                 p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
2473 
2474                 cluster_info = vzalloc(DIV_ROUND_UP(maxpages,
2475                         SWAPFILE_CLUSTER) * sizeof(*cluster_info));
2476                 if (!cluster_info) {
2477                         error = -ENOMEM;
2478                         goto bad_swap;
2479                 }
2480                 p->percpu_cluster = alloc_percpu(struct percpu_cluster);
2481                 if (!p->percpu_cluster) {
2482                         error = -ENOMEM;
2483                         goto bad_swap;
2484                 }
2485                 for_each_possible_cpu(cpu) {
2486                         struct percpu_cluster *cluster;
2487                         cluster = per_cpu_ptr(p->percpu_cluster, cpu);
2488                         cluster_set_null(&cluster->index);
2489                 }
2490         }
2491 
2492         error = swap_cgroup_swapon(p->type, maxpages);
2493         if (error)
2494                 goto bad_swap;
2495 
2496         nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
2497                 cluster_info, maxpages, &span);
2498         if (unlikely(nr_extents < 0)) {
2499                 error = nr_extents;
2500                 goto bad_swap;
2501         }
2502         /* frontswap enabled? set up bit-per-page map for frontswap */
2503         if (frontswap_enabled)
2504                 frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
2505 
2506         if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
2507                 /*
2508                  * When discard is enabled for swap with no particular
2509                  * policy flagged, we set all swap discard flags here in
2510                  * order to sustain backward compatibility with older
2511                  * swapon(8) releases.
2512                  */
2513                 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
2514                              SWP_PAGE_DISCARD);
2515 
2516                 /*
2517                  * By flagging sys_swapon, a sysadmin can tell us to
2518                  * either do single-time area discards only, or to just
2519                  * perform discards for released swap page-clusters.
2520                  * Now it's time to adjust the p->flags accordingly.
2521                  */
2522                 if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
2523                         p->flags &= ~SWP_PAGE_DISCARD;
2524                 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
2525                         p->flags &= ~SWP_AREA_DISCARD;
2526 
2527                 /* issue a swapon-time discard if it's still required */
2528                 if (p->flags & SWP_AREA_DISCARD) {
2529                         int err = discard_swap(p);
2530                         if (unlikely(err))
2531                                 pr_err("swapon: discard_swap(%p): %d\n",
2532                                         p, err);
2533                 }
2534         }
2535 
2536         mutex_lock(&swapon_mutex);
2537         prio = -1;
2538         if (swap_flags & SWAP_FLAG_PREFER)
2539                 prio =
2540                   (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2541         enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2542 
2543         pr_info("Adding %uk swap on %s.  "
2544                         "Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2545                 p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2546                 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2547                 (p->flags & SWP_SOLIDSTATE) ? "SS" : "",
2548                 (p->flags & SWP_DISCARDABLE) ? "D" : "",
2549                 (p->flags & SWP_AREA_DISCARD) ? "s" : "",
2550                 (p->flags & SWP_PAGE_DISCARD) ? "c" : "",
2551                 (frontswap_map) ? "FS" : "");
2552 
2553         mutex_unlock(&swapon_mutex);
2554         atomic_inc(&proc_poll_event);
2555         wake_up_interruptible(&proc_poll_wait);
2556 
2557         if (S_ISREG(inode->i_mode))
2558                 inode->i_flags |= S_SWAPFILE;
2559         error = 0;
2560         goto out;
2561 bad_swap:
2562         free_percpu(p->percpu_cluster);
2563         p->percpu_cluster = NULL;
2564         if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
2565                 set_blocksize(p->bdev, p->old_block_size);
2566                 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2567         }
2568         destroy_swap_extents(p);
2569         swap_cgroup_swapoff(p->type);
2570         spin_lock(&swap_lock);
2571         p->swap_file = NULL;
2572         p->flags = 0;
2573         spin_unlock(&swap_lock);
2574         vfree(swap_map);
2575         vfree(cluster_info);
2576         if (swap_file) {
2577                 if (inode && S_ISREG(inode->i_mode)) {
2578                         mutex_unlock(&inode->i_mutex);
2579                         inode = NULL;
2580                 }
2581                 filp_close(swap_file, NULL);
2582         }
2583 out:
2584         if (page && !IS_ERR(page)) {
2585                 kunmap(page);
2586                 page_cache_release(page);
2587         }
2588         if (name)
2589                 putname(name);
2590         if (inode && S_ISREG(inode->i_mode))
2591                 mutex_unlock(&inode->i_mutex);
2592         return error;
2593 }
2594 
2595 void si_swapinfo(struct sysinfo *val)
2596 {
2597         unsigned int type;
2598         unsigned long nr_to_be_unused = 0;
2599 
2600         spin_lock(&swap_lock);
2601         for (type = 0; type < nr_swapfiles; type++) {
2602                 struct swap_info_struct *si = swap_info[type];
2603 
2604                 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
2605                         nr_to_be_unused += si->inuse_pages;
2606         }
2607         val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
2608         val->totalswap = total_swap_pages + nr_to_be_unused;
2609         spin_unlock(&swap_lock);
2610 }
2611 
2612 /*
2613  * Verify that a swap entry is valid and increment its swap map count.
2614  *
2615  * Returns error code in following case.
2616  * - success -> 0
2617  * - swp_entry is invalid -> EINVAL
2618  * - swp_entry is migration entry -> EINVAL
2619  * - swap-cache reference is requested but there is already one. -> EEXIST
2620  * - swap-cache reference is requested but the entry is not used. -> ENOENT
2621  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
2622  */
2623 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
2624 {
2625         struct swap_info_struct *p;
2626         unsigned long offset, type;
2627         unsigned char count;
2628         unsigned char has_cache;
2629         int err = -EINVAL;
2630 
2631         if (non_swap_entry(entry))
2632                 goto out;
2633 
2634         type = swp_type(entry);
2635         if (type >= nr_swapfiles)
2636                 goto bad_file;
2637         p = swap_info[type];
2638         offset = swp_offset(entry);
2639 
2640         spin_lock(&p->lock);
2641         if (unlikely(offset >= p->max))
2642                 goto unlock_out;
2643 
2644         count = p->swap_map[offset];
2645 
2646         /*
2647          * swapin_readahead() doesn't check if a swap entry is valid, so the
2648          * swap entry could be SWAP_MAP_BAD. Check here with lock held.
2649          */
2650         if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
2651                 err = -ENOENT;
2652                 goto unlock_out;
2653         }
2654 
2655         has_cache = count & SWAP_HAS_CACHE;
2656         count &= ~SWAP_HAS_CACHE;
2657         err = 0;
2658 
2659         if (usage == SWAP_HAS_CACHE) {
2660 
2661                 /* set SWAP_HAS_CACHE if there is no cache and entry is used */
2662                 if (!has_cache && count)
2663                         has_cache = SWAP_HAS_CACHE;
2664                 else if (has_cache)             /* someone else added cache */
2665                         err = -EEXIST;
2666                 else                            /* no users remaining */
2667                         err = -ENOENT;
2668 
2669         } else if (count || has_cache) {
2670 
2671                 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
2672                         count += usage;
2673                 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
2674                         err = -EINVAL;
2675                 else if (swap_count_continued(p, offset, count))
2676                         count = COUNT_CONTINUED;
2677                 else
2678                         err = -ENOMEM;
2679         } else
2680                 err = -ENOENT;                  /* unused swap entry */
2681 
2682         p->swap_map[offset] = count | has_cache;
2683 
2684 unlock_out:
2685         spin_unlock(&p->lock);
2686 out:
2687         return err;
2688 
2689 bad_file:
2690         pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
2691         goto out;
2692 }
2693 
2694 /*
2695  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
2696  * (in which case its reference count is never incremented).
2697  */
2698 void swap_shmem_alloc(swp_entry_t entry)
2699 {
2700         __swap_duplicate(entry, SWAP_MAP_SHMEM);
2701 }
2702 
2703 /*
2704  * Increase reference count of swap entry by 1.
2705  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
2706  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
2707  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
2708  * might occur if a page table entry has got corrupted.
2709  */
2710 int swap_duplicate(swp_entry_t entry)
2711 {
2712         int err = 0;
2713 
2714         while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
2715                 err = add_swap_count_continuation(entry, GFP_ATOMIC);
2716         return err;
2717 }
2718 
2719 /*
2720  * @entry: swap entry for which we allocate swap cache.
2721  *
2722  * Called when allocating swap cache for existing swap entry,
2723  * This can return error codes. Returns 0 at success.
2724  * -EBUSY means there is a swap cache.
2725  * Note: return code is different from swap_duplicate().
2726  */
2727 int swapcache_prepare(swp_entry_t entry)
2728 {
2729         return __swap_duplicate(entry, SWAP_HAS_CACHE);
2730 }
2731 
2732 struct swap_info_struct *page_swap_info(struct page *page)
2733 {
2734         swp_entry_t swap = { .val = page_private(page) };
2735         BUG_ON(!PageSwapCache(page));
2736         return swap_info[swp_type(swap)];
2737 }
2738 
2739 /*
2740  * out-of-line __page_file_ methods to avoid include hell.
2741  */
2742 struct address_space *__page_file_mapping(struct page *page)
2743 {
2744         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2745         return page_swap_info(page)->swap_file->f_mapping;
2746 }
2747 EXPORT_SYMBOL_GPL(__page_file_mapping);
2748 
2749 pgoff_t __page_file_index(struct page *page)
2750 {
2751         swp_entry_t swap = { .val = page_private(page) };
2752         VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2753         return swp_offset(swap);
2754 }
2755 EXPORT_SYMBOL_GPL(__page_file_index);
2756 
2757 /*
2758  * add_swap_count_continuation - called when a swap count is duplicated
2759  * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
2760  * page of the original vmalloc'ed swap_map, to hold the continuation count
2761  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
2762  * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
2763  *
2764  * These continuation pages are seldom referenced: the common paths all work
2765  * on the original swap_map, only referring to a continuation page when the
2766  * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
2767  *
2768  * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
2769  * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
2770  * can be called after dropping locks.
2771  */
2772 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
2773 {
2774         struct swap_info_struct *si;
2775         struct page *head;
2776         struct page *page;
2777         struct page *list_page;
2778         pgoff_t offset;
2779         unsigned char count;
2780 
2781         /*
2782          * When debugging, it's easier to use __GFP_ZERO here; but it's better
2783          * for latency not to zero a page while GFP_ATOMIC and holding locks.
2784          */
2785         page = alloc_page(gfp_mask | __GFP_HIGHMEM);
2786 
2787         si = swap_info_get(entry);
2788         if (!si) {
2789                 /*
2790                  * An acceptable race has occurred since the failing
2791                  * __swap_duplicate(): the swap entry has been freed,
2792                  * perhaps even the whole swap_map cleared for swapoff.
2793                  */
2794                 goto outer;
2795         }
2796 
2797         offset = swp_offset(entry);
2798         count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
2799 
2800         if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
2801                 /*
2802                  * The higher the swap count, the more likely it is that tasks
2803                  * will race to add swap count continuation: we need to avoid
2804                  * over-provisioning.
2805                  */
2806                 goto out;
2807         }
2808 
2809         if (!page) {
2810                 spin_unlock(&si->lock);
2811                 return -ENOMEM;
2812         }
2813 
2814         /*
2815          * We are fortunate that although vmalloc_to_page uses pte_offset_map,
2816          * no architecture is using highmem pages for kernel page tables: so it
2817          * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
2818          */
2819         head = vmalloc_to_page(si->swap_map + offset);
2820         offset &= ~PAGE_MASK;
2821 
2822         /*
2823          * Page allocation does not initialize the page's lru field,
2824          * but it does always reset its private field.
2825          */
2826         if (!page_private(head)) {
2827                 BUG_ON(count & COUNT_CONTINUED);
2828                 INIT_LIST_HEAD(&head->lru);
2829                 set_page_private(head, SWP_CONTINUED);
2830                 si->flags |= SWP_CONTINUED;
2831         }
2832 
2833         list_for_each_entry(list_page, &head->lru, lru) {
2834                 unsigned char *map;
2835 
2836                 /*
2837                  * If the previous map said no continuation, but we've found
2838                  * a continuation page, free our allocation and use this one.
2839                  */
2840                 if (!(count & COUNT_CONTINUED))
2841                         goto out;
2842 
2843                 map = kmap_atomic(list_page) + offset;
2844                 count = *map;
2845                 kunmap_atomic(map);
2846 
2847                 /*
2848                  * If this continuation count now has some space in it,
2849                  * free our allocation and use this one.
2850                  */
2851                 if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
2852                         goto out;
2853         }
2854 
2855         list_add_tail(&page->lru, &head->lru);
2856         page = NULL;                    /* now it's attached, don't free it */
2857 out:
2858         spin_unlock(&si->lock);
2859 outer:
2860         if (page)
2861                 __free_page(page);
2862         return 0;
2863 }
2864 
2865 /*
2866  * swap_count_continued - when the original swap_map count is incremented
2867  * from SWAP_MAP_MAX, check if there is already a continuation page to carry
2868  * into, carry if so, or else fail until a new continuation page is allocated;
2869  * when the original swap_map count is decremented from 0 with continuation,
2870  * borrow from the continuation and report whether it still holds more.
2871  * Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
2872  */
2873 static bool swap_count_continued(struct swap_info_struct *si,
2874                                  pgoff_t offset, unsigned char count)
2875 {
2876         struct page *head;
2877         struct page *page;
2878         unsigned char *map;
2879 
2880         head = vmalloc_to_page(si->swap_map + offset);
2881         if (page_private(head) != SWP_CONTINUED) {
2882                 BUG_ON(count & COUNT_CONTINUED);
2883                 return false;           /* need to add count continuation */
2884         }
2885 
2886         offset &= ~PAGE_MASK;
2887         page = list_entry(head->lru.next, struct page, lru);
2888         map = kmap_atomic(page) + offset;
2889 
2890         if (count == SWAP_MAP_MAX)      /* initial increment from swap_map */
2891                 goto init_map;          /* jump over SWAP_CONT_MAX checks */
2892 
2893         if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
2894                 /*
2895                  * Think of how you add 1 to 999
2896                  */
2897                 while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
2898                         kunmap_atomic(map);
2899                         page = list_entry(page->lru.next, struct page, lru);
2900                         BUG_ON(page == head);
2901                         map = kmap_atomic(page) + offset;
2902                 }
2903                 if (*map == SWAP_CONT_MAX) {
2904                         kunmap_atomic(map);
2905                         page = list_entry(page->lru.next, struct page, lru);
2906                         if (page == head)
2907                                 return false;   /* add count continuation */
2908                         map = kmap_atomic(page) + offset;
2909 init_map:               *map = 0;               /* we didn't zero the page */
2910                 }
2911                 *map += 1;
2912                 kunmap_atomic(map);
2913                 page = list_entry(page->lru.prev, struct page, lru);
2914                 while (page != head) {
2915                         map = kmap_atomic(page) + offset;
2916                         *map = COUNT_CONTINUED;
2917                         kunmap_atomic(map);
2918                         page = list_entry(page->lru.prev, struct page, lru);
2919                 }
2920                 return true;                    /* incremented */
2921 
2922         } else {                                /* decrementing */
2923                 /*
2924                  * Think of how you subtract 1 from 1000
2925                  */
2926                 BUG_ON(count != COUNT_CONTINUED);
2927                 while (*map == COUNT_CONTINUED) {
2928                         kunmap_atomic(map);
2929                         page = list_entry(page->lru.next, struct page, lru);
2930                         BUG_ON(page == head);
2931                         map = kmap_atomic(page) + offset;
2932                 }
2933                 BUG_ON(*map == 0);
2934                 *map -= 1;
2935                 if (*map == 0)
2936                         count = 0;
2937                 kunmap_atomic(map);
2938                 page = list_entry(page->lru.prev, struct page, lru);
2939                 while (page != head) {
2940                         map = kmap_atomic(page) + offset;
2941                         *map = SWAP_CONT_MAX | count;
2942                         count = COUNT_CONTINUED;
2943                         kunmap_atomic(map);
2944                         page = list_entry(page->lru.prev, struct page, lru);
2945                 }
2946                 return count == COUNT_CONTINUED;
2947         }
2948 }
2949 
2950 /*
2951  * free_swap_count_continuations - swapoff free all the continuation pages
2952  * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
2953  */
2954 static void free_swap_count_continuations(struct swap_info_struct *si)
2955 {
2956         pgoff_t offset;
2957 
2958         for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
2959                 struct page *head;
2960                 head = vmalloc_to_page(si->swap_map + offset);
2961                 if (page_private(head)) {
2962                         struct list_head *this, *next;
2963                         list_for_each_safe(this, next, &head->lru) {
2964                                 struct page *page;
2965                                 page = list_entry(this, struct page, lru);
2966                                 list_del(this);
2967                                 __free_page(page);
2968                         }
2969                 }
2970         }
2971 }
2972 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp