~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/sched/mm.h

Version: ~ [ linux-5.1-rc1 ] ~ [ linux-5.0.2 ] ~ [ linux-4.20.16 ] ~ [ linux-4.19.29 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.106 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.163 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.176 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.136 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.63 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 #ifndef _LINUX_SCHED_MM_H
  3 #define _LINUX_SCHED_MM_H
  4 
  5 #include <linux/kernel.h>
  6 #include <linux/atomic.h>
  7 #include <linux/sched.h>
  8 #include <linux/mm_types.h>
  9 #include <linux/gfp.h>
 10 #include <linux/sync_core.h>
 11 
 12 /*
 13  * Routines for handling mm_structs
 14  */
 15 extern struct mm_struct *mm_alloc(void);
 16 
 17 /**
 18  * mmgrab() - Pin a &struct mm_struct.
 19  * @mm: The &struct mm_struct to pin.
 20  *
 21  * Make sure that @mm will not get freed even after the owning task
 22  * exits. This doesn't guarantee that the associated address space
 23  * will still exist later on and mmget_not_zero() has to be used before
 24  * accessing it.
 25  *
 26  * This is a preferred way to to pin @mm for a longer/unbounded amount
 27  * of time.
 28  *
 29  * Use mmdrop() to release the reference acquired by mmgrab().
 30  *
 31  * See also <Documentation/vm/active_mm.rst> for an in-depth explanation
 32  * of &mm_struct.mm_count vs &mm_struct.mm_users.
 33  */
 34 static inline void mmgrab(struct mm_struct *mm)
 35 {
 36         atomic_inc(&mm->mm_count);
 37 }
 38 
 39 extern void __mmdrop(struct mm_struct *mm);
 40 
 41 static inline void mmdrop(struct mm_struct *mm)
 42 {
 43         /*
 44          * The implicit full barrier implied by atomic_dec_and_test() is
 45          * required by the membarrier system call before returning to
 46          * user-space, after storing to rq->curr.
 47          */
 48         if (unlikely(atomic_dec_and_test(&mm->mm_count)))
 49                 __mmdrop(mm);
 50 }
 51 
 52 /**
 53  * mmget() - Pin the address space associated with a &struct mm_struct.
 54  * @mm: The address space to pin.
 55  *
 56  * Make sure that the address space of the given &struct mm_struct doesn't
 57  * go away. This does not protect against parts of the address space being
 58  * modified or freed, however.
 59  *
 60  * Never use this function to pin this address space for an
 61  * unbounded/indefinite amount of time.
 62  *
 63  * Use mmput() to release the reference acquired by mmget().
 64  *
 65  * See also <Documentation/vm/active_mm.rst> for an in-depth explanation
 66  * of &mm_struct.mm_count vs &mm_struct.mm_users.
 67  */
 68 static inline void mmget(struct mm_struct *mm)
 69 {
 70         atomic_inc(&mm->mm_users);
 71 }
 72 
 73 static inline bool mmget_not_zero(struct mm_struct *mm)
 74 {
 75         return atomic_inc_not_zero(&mm->mm_users);
 76 }
 77 
 78 /* mmput gets rid of the mappings and all user-space */
 79 extern void mmput(struct mm_struct *);
 80 #ifdef CONFIG_MMU
 81 /* same as above but performs the slow path from the async context. Can
 82  * be called from the atomic context as well
 83  */
 84 void mmput_async(struct mm_struct *);
 85 #endif
 86 
 87 /* Grab a reference to a task's mm, if it is not already going away */
 88 extern struct mm_struct *get_task_mm(struct task_struct *task);
 89 /*
 90  * Grab a reference to a task's mm, if it is not already going away
 91  * and ptrace_may_access with the mode parameter passed to it
 92  * succeeds.
 93  */
 94 extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode);
 95 /* Remove the current tasks stale references to the old mm_struct */
 96 extern void mm_release(struct task_struct *, struct mm_struct *);
 97 
 98 #ifdef CONFIG_MEMCG
 99 extern void mm_update_next_owner(struct mm_struct *mm);
100 #else
101 static inline void mm_update_next_owner(struct mm_struct *mm)
102 {
103 }
104 #endif /* CONFIG_MEMCG */
105 
106 #ifdef CONFIG_MMU
107 extern void arch_pick_mmap_layout(struct mm_struct *mm,
108                                   struct rlimit *rlim_stack);
109 extern unsigned long
110 arch_get_unmapped_area(struct file *, unsigned long, unsigned long,
111                        unsigned long, unsigned long);
112 extern unsigned long
113 arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
114                           unsigned long len, unsigned long pgoff,
115                           unsigned long flags);
116 #else
117 static inline void arch_pick_mmap_layout(struct mm_struct *mm,
118                                          struct rlimit *rlim_stack) {}
119 #endif
120 
121 static inline bool in_vfork(struct task_struct *tsk)
122 {
123         bool ret;
124 
125         /*
126          * need RCU to access ->real_parent if CLONE_VM was used along with
127          * CLONE_PARENT.
128          *
129          * We check real_parent->mm == tsk->mm because CLONE_VFORK does not
130          * imply CLONE_VM
131          *
132          * CLONE_VFORK can be used with CLONE_PARENT/CLONE_THREAD and thus
133          * ->real_parent is not necessarily the task doing vfork(), so in
134          * theory we can't rely on task_lock() if we want to dereference it.
135          *
136          * And in this case we can't trust the real_parent->mm == tsk->mm
137          * check, it can be false negative. But we do not care, if init or
138          * another oom-unkillable task does this it should blame itself.
139          */
140         rcu_read_lock();
141         ret = tsk->vfork_done && tsk->real_parent->mm == tsk->mm;
142         rcu_read_unlock();
143 
144         return ret;
145 }
146 
147 /*
148  * Applies per-task gfp context to the given allocation flags.
149  * PF_MEMALLOC_NOIO implies GFP_NOIO
150  * PF_MEMALLOC_NOFS implies GFP_NOFS
151  * PF_MEMALLOC_NOCMA implies no allocation from CMA region.
152  */
153 static inline gfp_t current_gfp_context(gfp_t flags)
154 {
155         if (unlikely(current->flags &
156                      (PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) {
157                 /*
158                  * NOIO implies both NOIO and NOFS and it is a weaker context
159                  * so always make sure it makes precedence
160                  */
161                 if (current->flags & PF_MEMALLOC_NOIO)
162                         flags &= ~(__GFP_IO | __GFP_FS);
163                 else if (current->flags & PF_MEMALLOC_NOFS)
164                         flags &= ~__GFP_FS;
165 #ifdef CONFIG_CMA
166                 if (current->flags & PF_MEMALLOC_NOCMA)
167                         flags &= ~__GFP_MOVABLE;
168 #endif
169         }
170         return flags;
171 }
172 
173 #ifdef CONFIG_LOCKDEP
174 extern void __fs_reclaim_acquire(void);
175 extern void __fs_reclaim_release(void);
176 extern void fs_reclaim_acquire(gfp_t gfp_mask);
177 extern void fs_reclaim_release(gfp_t gfp_mask);
178 #else
179 static inline void __fs_reclaim_acquire(void) { }
180 static inline void __fs_reclaim_release(void) { }
181 static inline void fs_reclaim_acquire(gfp_t gfp_mask) { }
182 static inline void fs_reclaim_release(gfp_t gfp_mask) { }
183 #endif
184 
185 /**
186  * memalloc_noio_save - Marks implicit GFP_NOIO allocation scope.
187  *
188  * This functions marks the beginning of the GFP_NOIO allocation scope.
189  * All further allocations will implicitly drop __GFP_IO flag and so
190  * they are safe for the IO critical section from the allocation recursion
191  * point of view. Use memalloc_noio_restore to end the scope with flags
192  * returned by this function.
193  *
194  * This function is safe to be used from any context.
195  */
196 static inline unsigned int memalloc_noio_save(void)
197 {
198         unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
199         current->flags |= PF_MEMALLOC_NOIO;
200         return flags;
201 }
202 
203 /**
204  * memalloc_noio_restore - Ends the implicit GFP_NOIO scope.
205  * @flags: Flags to restore.
206  *
207  * Ends the implicit GFP_NOIO scope started by memalloc_noio_save function.
208  * Always make sure that that the given flags is the return value from the
209  * pairing memalloc_noio_save call.
210  */
211 static inline void memalloc_noio_restore(unsigned int flags)
212 {
213         current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
214 }
215 
216 /**
217  * memalloc_nofs_save - Marks implicit GFP_NOFS allocation scope.
218  *
219  * This functions marks the beginning of the GFP_NOFS allocation scope.
220  * All further allocations will implicitly drop __GFP_FS flag and so
221  * they are safe for the FS critical section from the allocation recursion
222  * point of view. Use memalloc_nofs_restore to end the scope with flags
223  * returned by this function.
224  *
225  * This function is safe to be used from any context.
226  */
227 static inline unsigned int memalloc_nofs_save(void)
228 {
229         unsigned int flags = current->flags & PF_MEMALLOC_NOFS;
230         current->flags |= PF_MEMALLOC_NOFS;
231         return flags;
232 }
233 
234 /**
235  * memalloc_nofs_restore - Ends the implicit GFP_NOFS scope.
236  * @flags: Flags to restore.
237  *
238  * Ends the implicit GFP_NOFS scope started by memalloc_nofs_save function.
239  * Always make sure that that the given flags is the return value from the
240  * pairing memalloc_nofs_save call.
241  */
242 static inline void memalloc_nofs_restore(unsigned int flags)
243 {
244         current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
245 }
246 
247 static inline unsigned int memalloc_noreclaim_save(void)
248 {
249         unsigned int flags = current->flags & PF_MEMALLOC;
250         current->flags |= PF_MEMALLOC;
251         return flags;
252 }
253 
254 static inline void memalloc_noreclaim_restore(unsigned int flags)
255 {
256         current->flags = (current->flags & ~PF_MEMALLOC) | flags;
257 }
258 
259 #ifdef CONFIG_CMA
260 static inline unsigned int memalloc_nocma_save(void)
261 {
262         unsigned int flags = current->flags & PF_MEMALLOC_NOCMA;
263 
264         current->flags |= PF_MEMALLOC_NOCMA;
265         return flags;
266 }
267 
268 static inline void memalloc_nocma_restore(unsigned int flags)
269 {
270         current->flags = (current->flags & ~PF_MEMALLOC_NOCMA) | flags;
271 }
272 #else
273 static inline unsigned int memalloc_nocma_save(void)
274 {
275         return 0;
276 }
277 
278 static inline void memalloc_nocma_restore(unsigned int flags)
279 {
280 }
281 #endif
282 
283 #ifdef CONFIG_MEMCG
284 /**
285  * memalloc_use_memcg - Starts the remote memcg charging scope.
286  * @memcg: memcg to charge.
287  *
288  * This function marks the beginning of the remote memcg charging scope. All the
289  * __GFP_ACCOUNT allocations till the end of the scope will be charged to the
290  * given memcg.
291  *
292  * NOTE: This function is not nesting safe.
293  */
294 static inline void memalloc_use_memcg(struct mem_cgroup *memcg)
295 {
296         WARN_ON_ONCE(current->active_memcg);
297         current->active_memcg = memcg;
298 }
299 
300 /**
301  * memalloc_unuse_memcg - Ends the remote memcg charging scope.
302  *
303  * This function marks the end of the remote memcg charging scope started by
304  * memalloc_use_memcg().
305  */
306 static inline void memalloc_unuse_memcg(void)
307 {
308         current->active_memcg = NULL;
309 }
310 #else
311 static inline void memalloc_use_memcg(struct mem_cgroup *memcg)
312 {
313 }
314 
315 static inline void memalloc_unuse_memcg(void)
316 {
317 }
318 #endif
319 
320 #ifdef CONFIG_MEMBARRIER
321 enum {
322         MEMBARRIER_STATE_PRIVATE_EXPEDITED_READY                = (1U << 0),
323         MEMBARRIER_STATE_PRIVATE_EXPEDITED                      = (1U << 1),
324         MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY                 = (1U << 2),
325         MEMBARRIER_STATE_GLOBAL_EXPEDITED                       = (1U << 3),
326         MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE_READY      = (1U << 4),
327         MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE            = (1U << 5),
328 };
329 
330 enum {
331         MEMBARRIER_FLAG_SYNC_CORE       = (1U << 0),
332 };
333 
334 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
335 #include <asm/membarrier.h>
336 #endif
337 
338 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
339 {
340         if (likely(!(atomic_read(&mm->membarrier_state) &
341                      MEMBARRIER_STATE_PRIVATE_EXPEDITED_SYNC_CORE)))
342                 return;
343         sync_core_before_usermode();
344 }
345 
346 static inline void membarrier_execve(struct task_struct *t)
347 {
348         atomic_set(&t->mm->membarrier_state, 0);
349 }
350 #else
351 #ifdef CONFIG_ARCH_HAS_MEMBARRIER_CALLBACKS
352 static inline void membarrier_arch_switch_mm(struct mm_struct *prev,
353                                              struct mm_struct *next,
354                                              struct task_struct *tsk)
355 {
356 }
357 #endif
358 static inline void membarrier_execve(struct task_struct *t)
359 {
360 }
361 static inline void membarrier_mm_sync_core_before_usermode(struct mm_struct *mm)
362 {
363 }
364 #endif
365 
366 #endif /* _LINUX_SCHED_MM_H */
367 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp