~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/jump_label.h

Version: ~ [ linux-5.2-rc6 ] ~ [ linux-5.1.15 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.56 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.130 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.183 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.183 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.69 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #ifndef _LINUX_JUMP_LABEL_H
  2 #define _LINUX_JUMP_LABEL_H
  3 
  4 /*
  5  * Jump label support
  6  *
  7  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
  8  * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
  9  *
 10  * Jump labels provide an interface to generate dynamic branches using
 11  * self-modifying code. Assuming toolchain and architecture support the result
 12  * of a "if (static_key_false(&key))" statement is a unconditional branch (which
 13  * defaults to false - and the true block is placed out of line).
 14  *
 15  * However at runtime we can change the branch target using
 16  * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
 17  * object and for as long as there are references all branches referring to
 18  * that particular key will point to the (out of line) true block.
 19  *
 20  * Since this relies on modifying code the static_key_slow_{inc,dec}() functions
 21  * must be considered absolute slow paths (machine wide synchronization etc.).
 22  * OTOH, since the affected branches are unconditional their runtime overhead
 23  * will be absolutely minimal, esp. in the default (off) case where the total
 24  * effect is a single NOP of appropriate size. The on case will patch in a jump
 25  * to the out-of-line block.
 26  *
 27  * When the control is directly exposed to userspace it is prudent to delay the
 28  * decrement to avoid high frequency code modifications which can (and do)
 29  * cause significant performance degradation. Struct static_key_deferred and
 30  * static_key_slow_dec_deferred() provide for this.
 31  *
 32  * Lacking toolchain and or architecture support, it falls back to a simple
 33  * conditional branch.
 34  *
 35  * struct static_key my_key = STATIC_KEY_INIT_TRUE;
 36  *
 37  *   if (static_key_true(&my_key)) {
 38  *   }
 39  *
 40  * will result in the true case being in-line and starts the key with a single
 41  * reference. Mixing static_key_true() and static_key_false() on the same key is not
 42  * allowed.
 43  *
 44  * Not initializing the key (static data is initialized to 0s anyway) is the
 45  * same as using STATIC_KEY_INIT_FALSE.
 46  *
 47 */
 48 
 49 #include <linux/types.h>
 50 #include <linux/compiler.h>
 51 #include <linux/workqueue.h>
 52 
 53 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
 54 
 55 struct static_key {
 56         atomic_t enabled;
 57 /* Set lsb bit to 1 if branch is default true, 0 ot */
 58         struct jump_entry *entries;
 59 #ifdef CONFIG_MODULES
 60         struct static_key_mod *next;
 61 #endif
 62 };
 63 
 64 struct static_key_deferred {
 65         struct static_key key;
 66         unsigned long timeout;
 67         struct delayed_work work;
 68 };
 69 
 70 # include <asm/jump_label.h>
 71 # define HAVE_JUMP_LABEL
 72 #endif  /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
 73 
 74 enum jump_label_type {
 75         JUMP_LABEL_DISABLE = 0,
 76         JUMP_LABEL_ENABLE,
 77 };
 78 
 79 struct module;
 80 
 81 #ifdef HAVE_JUMP_LABEL
 82 
 83 #define JUMP_LABEL_TRUE_BRANCH 1UL
 84 
 85 static
 86 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 87 {
 88         return (struct jump_entry *)((unsigned long)key->entries
 89                                                 & ~JUMP_LABEL_TRUE_BRANCH);
 90 }
 91 
 92 static inline bool jump_label_get_branch_default(struct static_key *key)
 93 {
 94         if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
 95                 return true;
 96         return false;
 97 }
 98 
 99 static __always_inline bool static_key_false(struct static_key *key)
100 {
101         return arch_static_branch(key);
102 }
103 
104 static __always_inline bool static_key_true(struct static_key *key)
105 {
106         return !static_key_false(key);
107 }
108 
109 extern struct jump_entry __start___jump_table[];
110 extern struct jump_entry __stop___jump_table[];
111 
112 extern void jump_label_init(void);
113 extern void jump_label_lock(void);
114 extern void jump_label_unlock(void);
115 extern void arch_jump_label_transform(struct jump_entry *entry,
116                                       enum jump_label_type type);
117 extern void arch_jump_label_transform_static(struct jump_entry *entry,
118                                              enum jump_label_type type);
119 extern int jump_label_text_reserved(void *start, void *end);
120 extern void static_key_slow_inc(struct static_key *key);
121 extern void static_key_slow_dec(struct static_key *key);
122 extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
123 extern void jump_label_apply_nops(struct module *mod);
124 extern void
125 jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
126 
127 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
128         { .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
129 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
130         { .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
131 
132 #else  /* !HAVE_JUMP_LABEL */
133 
134 #include <linux/atomic.h>
135 
136 struct static_key {
137         atomic_t enabled;
138 };
139 
140 static __always_inline void jump_label_init(void)
141 {
142 }
143 
144 struct static_key_deferred {
145         struct static_key  key;
146 };
147 
148 static __always_inline bool static_key_false(struct static_key *key)
149 {
150         if (unlikely(atomic_read(&key->enabled)) > 0)
151                 return true;
152         return false;
153 }
154 
155 static __always_inline bool static_key_true(struct static_key *key)
156 {
157         if (likely(atomic_read(&key->enabled)) > 0)
158                 return true;
159         return false;
160 }
161 
162 static inline void static_key_slow_inc(struct static_key *key)
163 {
164         atomic_inc(&key->enabled);
165 }
166 
167 static inline void static_key_slow_dec(struct static_key *key)
168 {
169         atomic_dec(&key->enabled);
170 }
171 
172 static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
173 {
174         static_key_slow_dec(&key->key);
175 }
176 
177 static inline int jump_label_text_reserved(void *start, void *end)
178 {
179         return 0;
180 }
181 
182 static inline void jump_label_lock(void) {}
183 static inline void jump_label_unlock(void) {}
184 
185 static inline int jump_label_apply_nops(struct module *mod)
186 {
187         return 0;
188 }
189 
190 static inline void
191 jump_label_rate_limit(struct static_key_deferred *key,
192                 unsigned long rl)
193 {
194 }
195 
196 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
197                 { .enabled = ATOMIC_INIT(1) })
198 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
199                 { .enabled = ATOMIC_INIT(0) })
200 
201 #endif  /* HAVE_JUMP_LABEL */
202 
203 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
204 #define jump_label_enabled static_key_enabled
205 
206 static inline bool static_key_enabled(struct static_key *key)
207 {
208         return (atomic_read(&key->enabled) > 0);
209 }
210 
211 #endif  /* _LINUX_JUMP_LABEL_H */
212 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp