~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/include/linux/jump_label.h

Version: ~ [ linux-5.9 ] ~ [ linux-5.8.14 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.70 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.150 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.200 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.238 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.238 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #ifndef _LINUX_JUMP_LABEL_H
  2 #define _LINUX_JUMP_LABEL_H
  3 
  4 /*
  5  * Jump label support
  6  *
  7  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
  8  * Copyright (C) 2011-2012 Peter Zijlstra <pzijlstr@redhat.com>
  9  *
 10  * Jump labels provide an interface to generate dynamic branches using
 11  * self-modifying code. Assuming toolchain and architecture support, the result
 12  * of a "if (static_key_false(&key))" statement is an unconditional branch (which
 13  * defaults to false - and the true block is placed out of line).
 14  *
 15  * However at runtime we can change the branch target using
 16  * static_key_slow_{inc,dec}(). These function as a 'reference' count on the key
 17  * object, and for as long as there are references all branches referring to
 18  * that particular key will point to the (out of line) true block.
 19  *
 20  * Since this relies on modifying code, the static_key_slow_{inc,dec}() functions
 21  * must be considered absolute slow paths (machine wide synchronization etc.).
 22  * OTOH, since the affected branches are unconditional, their runtime overhead
 23  * will be absolutely minimal, esp. in the default (off) case where the total
 24  * effect is a single NOP of appropriate size. The on case will patch in a jump
 25  * to the out-of-line block.
 26  *
 27  * When the control is directly exposed to userspace, it is prudent to delay the
 28  * decrement to avoid high frequency code modifications which can (and do)
 29  * cause significant performance degradation. Struct static_key_deferred and
 30  * static_key_slow_dec_deferred() provide for this.
 31  *
 32  * Lacking toolchain and or architecture support, jump labels fall back to a simple
 33  * conditional branch.
 34  *
 35  * struct static_key my_key = STATIC_KEY_INIT_TRUE;
 36  *
 37  *   if (static_key_true(&my_key)) {
 38  *   }
 39  *
 40  * will result in the true case being in-line and starts the key with a single
 41  * reference. Mixing static_key_true() and static_key_false() on the same key is not
 42  * allowed.
 43  *
 44  * Not initializing the key (static data is initialized to 0s anyway) is the
 45  * same as using STATIC_KEY_INIT_FALSE.
 46  */
 47 
 48 #include <linux/types.h>
 49 #include <linux/compiler.h>
 50 #include <linux/bug.h>
 51 
 52 extern bool static_key_initialized;
 53 
 54 #define STATIC_KEY_CHECK_USE() WARN(!static_key_initialized,                  \
 55                                     "%s used before call to jump_label_init", \
 56                                     __func__)
 57 
 58 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
 59 
 60 struct static_key {
 61         atomic_t enabled;
 62 /* Set lsb bit to 1 if branch is default true, 0 ot */
 63         struct jump_entry *entries;
 64 #ifdef CONFIG_MODULES
 65         struct static_key_mod *next;
 66 #endif
 67 };
 68 
 69 # include <asm/jump_label.h>
 70 # define HAVE_JUMP_LABEL
 71 #else
 72 struct static_key {
 73         atomic_t enabled;
 74 };
 75 #endif  /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */
 76 
 77 enum jump_label_type {
 78         JUMP_LABEL_DISABLE = 0,
 79         JUMP_LABEL_ENABLE,
 80 };
 81 
 82 struct module;
 83 
 84 #include <linux/atomic.h>
 85 
 86 static inline int static_key_count(struct static_key *key)
 87 {
 88         return atomic_read(&key->enabled);
 89 }
 90 
 91 #ifdef HAVE_JUMP_LABEL
 92 
 93 #define JUMP_LABEL_TYPE_FALSE_BRANCH    0UL
 94 #define JUMP_LABEL_TYPE_TRUE_BRANCH     1UL
 95 #define JUMP_LABEL_TYPE_MASK            1UL
 96 
 97 static
 98 inline struct jump_entry *jump_label_get_entries(struct static_key *key)
 99 {
100         return (struct jump_entry *)((unsigned long)key->entries
101                                                 & ~JUMP_LABEL_TYPE_MASK);
102 }
103 
104 static inline bool jump_label_get_branch_default(struct static_key *key)
105 {
106         if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
107             JUMP_LABEL_TYPE_TRUE_BRANCH)
108                 return true;
109         return false;
110 }
111 
112 static __always_inline bool static_key_false(struct static_key *key)
113 {
114         return arch_static_branch(key);
115 }
116 
117 static __always_inline bool static_key_true(struct static_key *key)
118 {
119         return !static_key_false(key);
120 }
121 
122 extern struct jump_entry __start___jump_table[];
123 extern struct jump_entry __stop___jump_table[];
124 
125 extern void jump_label_init(void);
126 extern void jump_label_lock(void);
127 extern void jump_label_unlock(void);
128 extern void arch_jump_label_transform(struct jump_entry *entry,
129                                       enum jump_label_type type);
130 extern void arch_jump_label_transform_static(struct jump_entry *entry,
131                                              enum jump_label_type type);
132 extern int jump_label_text_reserved(void *start, void *end);
133 extern void static_key_slow_inc(struct static_key *key);
134 extern void static_key_slow_dec(struct static_key *key);
135 extern void jump_label_apply_nops(struct module *mod);
136 
137 #define STATIC_KEY_INIT_TRUE ((struct static_key)               \
138         { .enabled = ATOMIC_INIT(1),                            \
139           .entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
140 #define STATIC_KEY_INIT_FALSE ((struct static_key)              \
141         { .enabled = ATOMIC_INIT(0),                            \
142           .entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
143 
144 #else  /* !HAVE_JUMP_LABEL */
145 
146 static __always_inline void jump_label_init(void)
147 {
148         static_key_initialized = true;
149 }
150 
151 static __always_inline bool static_key_false(struct static_key *key)
152 {
153         if (unlikely(static_key_count(key) > 0))
154                 return true;
155         return false;
156 }
157 
158 static __always_inline bool static_key_true(struct static_key *key)
159 {
160         if (likely(static_key_count(key) > 0))
161                 return true;
162         return false;
163 }
164 
165 static inline void static_key_slow_inc(struct static_key *key)
166 {
167         STATIC_KEY_CHECK_USE();
168         atomic_inc(&key->enabled);
169 }
170 
171 static inline void static_key_slow_dec(struct static_key *key)
172 {
173         STATIC_KEY_CHECK_USE();
174         atomic_dec(&key->enabled);
175 }
176 
177 static inline int jump_label_text_reserved(void *start, void *end)
178 {
179         return 0;
180 }
181 
182 static inline void jump_label_lock(void) {}
183 static inline void jump_label_unlock(void) {}
184 
185 static inline int jump_label_apply_nops(struct module *mod)
186 {
187         return 0;
188 }
189 
190 #define STATIC_KEY_INIT_TRUE ((struct static_key) \
191                 { .enabled = ATOMIC_INIT(1) })
192 #define STATIC_KEY_INIT_FALSE ((struct static_key) \
193                 { .enabled = ATOMIC_INIT(0) })
194 
195 #endif  /* HAVE_JUMP_LABEL */
196 
197 #define STATIC_KEY_INIT STATIC_KEY_INIT_FALSE
198 #define jump_label_enabled static_key_enabled
199 
200 static inline bool static_key_enabled(struct static_key *key)
201 {
202         return static_key_count(key) > 0;
203 }
204 
205 #endif  /* _LINUX_JUMP_LABEL_H */
206 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp