~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/mm/swap_slots.c

Version: ~ [ linux-5.9-rc6 ] ~ [ linux-5.8.10 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.66 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.146 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.198 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.236 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.236 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Manage cache of swap slots to be used for and returned from
  3  * swap.
  4  *
  5  * Copyright(c) 2016 Intel Corporation.
  6  *
  7  * Author: Tim Chen <tim.c.chen@linux.intel.com>
  8  *
  9  * We allocate the swap slots from the global pool and put
 10  * it into local per cpu caches.  This has the advantage
 11  * of no needing to acquire the swap_info lock every time
 12  * we need a new slot.
 13  *
 14  * There is also opportunity to simply return the slot
 15  * to local caches without needing to acquire swap_info
 16  * lock.  We do not reuse the returned slots directly but
 17  * move them back to the global pool in a batch.  This
 18  * allows the slots to coaellesce and reduce fragmentation.
 19  *
 20  * The swap entry allocated is marked with SWAP_HAS_CACHE
 21  * flag in map_count that prevents it from being allocated
 22  * again from the global pool.
 23  *
 24  * The swap slots cache is protected by a mutex instead of
 25  * a spin lock as when we search for slots with scan_swap_map,
 26  * we can possibly sleep.
 27  */
 28 
 29 #include <linux/swap_slots.h>
 30 #include <linux/cpu.h>
 31 #include <linux/cpumask.h>
 32 #include <linux/vmalloc.h>
 33 #include <linux/mutex.h>
 34 #include <linux/mm.h>
 35 
 36 #ifdef CONFIG_SWAP
 37 
 38 static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
 39 static bool     swap_slot_cache_active;
 40 bool    swap_slot_cache_enabled;
 41 static bool     swap_slot_cache_initialized;
 42 DEFINE_MUTEX(swap_slots_cache_mutex);
 43 /* Serialize swap slots cache enable/disable operations */
 44 DEFINE_MUTEX(swap_slots_cache_enable_mutex);
 45 
 46 static void __drain_swap_slots_cache(unsigned int type);
 47 static void deactivate_swap_slots_cache(void);
 48 static void reactivate_swap_slots_cache(void);
 49 
 50 #define use_swap_slot_cache (swap_slot_cache_active && \
 51                 swap_slot_cache_enabled && swap_slot_cache_initialized)
 52 #define SLOTS_CACHE 0x1
 53 #define SLOTS_CACHE_RET 0x2
 54 
 55 static void deactivate_swap_slots_cache(void)
 56 {
 57         mutex_lock(&swap_slots_cache_mutex);
 58         swap_slot_cache_active = false;
 59         __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
 60         mutex_unlock(&swap_slots_cache_mutex);
 61 }
 62 
 63 static void reactivate_swap_slots_cache(void)
 64 {
 65         mutex_lock(&swap_slots_cache_mutex);
 66         swap_slot_cache_active = true;
 67         mutex_unlock(&swap_slots_cache_mutex);
 68 }
 69 
 70 /* Must not be called with cpu hot plug lock */
 71 void disable_swap_slots_cache_lock(void)
 72 {
 73         mutex_lock(&swap_slots_cache_enable_mutex);
 74         swap_slot_cache_enabled = false;
 75         if (swap_slot_cache_initialized) {
 76                 /* serialize with cpu hotplug operations */
 77                 get_online_cpus();
 78                 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
 79                 put_online_cpus();
 80         }
 81 }
 82 
 83 static void __reenable_swap_slots_cache(void)
 84 {
 85         swap_slot_cache_enabled = has_usable_swap();
 86 }
 87 
 88 void reenable_swap_slots_cache_unlock(void)
 89 {
 90         __reenable_swap_slots_cache();
 91         mutex_unlock(&swap_slots_cache_enable_mutex);
 92 }
 93 
 94 static bool check_cache_active(void)
 95 {
 96         long pages;
 97 
 98         if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
 99                 return false;
100 
101         pages = get_nr_swap_pages();
102         if (!swap_slot_cache_active) {
103                 if (pages > num_online_cpus() *
104                     THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
105                         reactivate_swap_slots_cache();
106                 goto out;
107         }
108 
109         /* if global pool of slot caches too low, deactivate cache */
110         if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
111                 deactivate_swap_slots_cache();
112 out:
113         return swap_slot_cache_active;
114 }
115 
116 static int alloc_swap_slot_cache(unsigned int cpu)
117 {
118         struct swap_slots_cache *cache;
119         swp_entry_t *slots, *slots_ret;
120 
121         /*
122          * Do allocation outside swap_slots_cache_mutex
123          * as kvzalloc could trigger reclaim and get_swap_page,
124          * which can lock swap_slots_cache_mutex.
125          */
126         slots = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,
127                          GFP_KERNEL);
128         if (!slots)
129                 return -ENOMEM;
130 
131         slots_ret = kvzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE,
132                              GFP_KERNEL);
133         if (!slots_ret) {
134                 kvfree(slots);
135                 return -ENOMEM;
136         }
137 
138         mutex_lock(&swap_slots_cache_mutex);
139         cache = &per_cpu(swp_slots, cpu);
140         if (cache->slots || cache->slots_ret)
141                 /* cache already allocated */
142                 goto out;
143         if (!cache->lock_initialized) {
144                 mutex_init(&cache->alloc_lock);
145                 spin_lock_init(&cache->free_lock);
146                 cache->lock_initialized = true;
147         }
148         cache->nr = 0;
149         cache->cur = 0;
150         cache->n_ret = 0;
151         cache->slots = slots;
152         slots = NULL;
153         cache->slots_ret = slots_ret;
154         slots_ret = NULL;
155 out:
156         mutex_unlock(&swap_slots_cache_mutex);
157         if (slots)
158                 kvfree(slots);
159         if (slots_ret)
160                 kvfree(slots_ret);
161         return 0;
162 }
163 
164 static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
165                                   bool free_slots)
166 {
167         struct swap_slots_cache *cache;
168         swp_entry_t *slots = NULL;
169 
170         cache = &per_cpu(swp_slots, cpu);
171         if ((type & SLOTS_CACHE) && cache->slots) {
172                 mutex_lock(&cache->alloc_lock);
173                 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
174                 cache->cur = 0;
175                 cache->nr = 0;
176                 if (free_slots && cache->slots) {
177                         kvfree(cache->slots);
178                         cache->slots = NULL;
179                 }
180                 mutex_unlock(&cache->alloc_lock);
181         }
182         if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
183                 spin_lock_irq(&cache->free_lock);
184                 swapcache_free_entries(cache->slots_ret, cache->n_ret);
185                 cache->n_ret = 0;
186                 if (free_slots && cache->slots_ret) {
187                         slots = cache->slots_ret;
188                         cache->slots_ret = NULL;
189                 }
190                 spin_unlock_irq(&cache->free_lock);
191                 if (slots)
192                         kvfree(slots);
193         }
194 }
195 
196 static void __drain_swap_slots_cache(unsigned int type)
197 {
198         unsigned int cpu;
199 
200         /*
201          * This function is called during
202          *      1) swapoff, when we have to make sure no
203          *         left over slots are in cache when we remove
204          *         a swap device;
205          *      2) disabling of swap slot cache, when we run low
206          *         on swap slots when allocating memory and need
207          *         to return swap slots to global pool.
208          *
209          * We cannot acquire cpu hot plug lock here as
210          * this function can be invoked in the cpu
211          * hot plug path:
212          * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
213          *   -> memory allocation -> direct reclaim -> get_swap_page
214          *   -> drain_swap_slots_cache
215          *
216          * Hence the loop over current online cpu below could miss cpu that
217          * is being brought online but not yet marked as online.
218          * That is okay as we do not schedule and run anything on a
219          * cpu before it has been marked online. Hence, we will not
220          * fill any swap slots in slots cache of such cpu.
221          * There are no slots on such cpu that need to be drained.
222          */
223         for_each_online_cpu(cpu)
224                 drain_slots_cache_cpu(cpu, type, false);
225 }
226 
227 static int free_slot_cache(unsigned int cpu)
228 {
229         mutex_lock(&swap_slots_cache_mutex);
230         drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
231         mutex_unlock(&swap_slots_cache_mutex);
232         return 0;
233 }
234 
235 int enable_swap_slots_cache(void)
236 {
237         int ret = 0;
238 
239         mutex_lock(&swap_slots_cache_enable_mutex);
240         if (swap_slot_cache_initialized) {
241                 __reenable_swap_slots_cache();
242                 goto out_unlock;
243         }
244 
245         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
246                                 alloc_swap_slot_cache, free_slot_cache);
247         if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
248                                "without swap slots cache.\n", __func__))
249                 goto out_unlock;
250 
251         swap_slot_cache_initialized = true;
252         __reenable_swap_slots_cache();
253 out_unlock:
254         mutex_unlock(&swap_slots_cache_enable_mutex);
255         return 0;
256 }
257 
258 /* called with swap slot cache's alloc lock held */
259 static int refill_swap_slots_cache(struct swap_slots_cache *cache)
260 {
261         if (!use_swap_slot_cache || cache->nr)
262                 return 0;
263 
264         cache->cur = 0;
265         if (swap_slot_cache_active)
266                 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE, cache->slots);
267 
268         return cache->nr;
269 }
270 
271 int free_swap_slot(swp_entry_t entry)
272 {
273         struct swap_slots_cache *cache;
274 
275         cache = &get_cpu_var(swp_slots);
276         if (use_swap_slot_cache && cache->slots_ret) {
277                 spin_lock_irq(&cache->free_lock);
278                 /* Swap slots cache may be deactivated before acquiring lock */
279                 if (!use_swap_slot_cache) {
280                         spin_unlock_irq(&cache->free_lock);
281                         goto direct_free;
282                 }
283                 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
284                         /*
285                          * Return slots to global pool.
286                          * The current swap_map value is SWAP_HAS_CACHE.
287                          * Set it to 0 to indicate it is available for
288                          * allocation in global pool
289                          */
290                         swapcache_free_entries(cache->slots_ret, cache->n_ret);
291                         cache->n_ret = 0;
292                 }
293                 cache->slots_ret[cache->n_ret++] = entry;
294                 spin_unlock_irq(&cache->free_lock);
295         } else {
296 direct_free:
297                 swapcache_free_entries(&entry, 1);
298         }
299         put_cpu_var(swp_slots);
300 
301         return 0;
302 }
303 
304 swp_entry_t get_swap_page(void)
305 {
306         swp_entry_t entry, *pentry;
307         struct swap_slots_cache *cache;
308 
309         /*
310          * Preemption is allowed here, because we may sleep
311          * in refill_swap_slots_cache().  But it is safe, because
312          * accesses to the per-CPU data structure are protected by the
313          * mutex cache->alloc_lock.
314          *
315          * The alloc path here does not touch cache->slots_ret
316          * so cache->free_lock is not taken.
317          */
318         cache = raw_cpu_ptr(&swp_slots);
319 
320         entry.val = 0;
321         if (check_cache_active()) {
322                 mutex_lock(&cache->alloc_lock);
323                 if (cache->slots) {
324 repeat:
325                         if (cache->nr) {
326                                 pentry = &cache->slots[cache->cur++];
327                                 entry = *pentry;
328                                 pentry->val = 0;
329                                 cache->nr--;
330                         } else {
331                                 if (refill_swap_slots_cache(cache))
332                                         goto repeat;
333                         }
334                 }
335                 mutex_unlock(&cache->alloc_lock);
336                 if (entry.val)
337                         return entry;
338         }
339 
340         get_swap_pages(1, &entry);
341 
342         return entry;
343 }
344 
345 #endif /* CONFIG_SWAP */
346 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp