~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/include/asm/alternative.h

Version: ~ [ linux-5.11 ] ~ [ linux-5.10.17 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.99 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.176 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.221 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.257 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.257 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /* SPDX-License-Identifier: GPL-2.0 */
  2 #ifndef _ASM_X86_ALTERNATIVE_H
  3 #define _ASM_X86_ALTERNATIVE_H
  4 
  5 #ifndef __ASSEMBLY__
  6 
  7 #include <linux/types.h>
  8 #include <linux/stddef.h>
  9 #include <linux/stringify.h>
 10 #include <asm/asm.h>
 11 
 12 /*
 13  * Alternative inline assembly for SMP.
 14  *
 15  * The LOCK_PREFIX macro defined here replaces the LOCK and
 16  * LOCK_PREFIX macros used everywhere in the source tree.
 17  *
 18  * SMP alternatives use the same data structures as the other
 19  * alternatives and the X86_FEATURE_UP flag to indicate the case of a
 20  * UP system running a SMP kernel.  The existing apply_alternatives()
 21  * works fine for patching a SMP kernel for UP.
 22  *
 23  * The SMP alternative tables can be kept after boot and contain both
 24  * UP and SMP versions of the instructions to allow switching back to
 25  * SMP at runtime, when hotplugging in a new CPU, which is especially
 26  * useful in virtualized environments.
 27  *
 28  * The very common lock prefix is handled as special case in a
 29  * separate table which is a pure address list without replacement ptr
 30  * and size information.  That keeps the table sizes small.
 31  */
 32 
 33 #ifdef CONFIG_SMP
 34 #define LOCK_PREFIX_HERE \
 35                 ".pushsection .smp_locks,\"a\"\n"       \
 36                 ".balign 4\n"                           \
 37                 ".long 671f - .\n" /* offset */         \
 38                 ".popsection\n"                         \
 39                 "671:"
 40 
 41 #define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
 42 
 43 #else /* ! CONFIG_SMP */
 44 #define LOCK_PREFIX_HERE ""
 45 #define LOCK_PREFIX ""
 46 #endif
 47 
 48 /*
 49  * objtool annotation to ignore the alternatives and only consider the original
 50  * instruction(s).
 51  */
 52 #define ANNOTATE_IGNORE_ALTERNATIVE                             \
 53         "999:\n\t"                                              \
 54         ".pushsection .discard.ignore_alts\n\t"                 \
 55         ".long 999b - .\n\t"                                    \
 56         ".popsection\n\t"
 57 
 58 struct alt_instr {
 59         s32 instr_offset;       /* original instruction */
 60         s32 repl_offset;        /* offset to replacement instruction */
 61         u16 cpuid;              /* cpuid bit set for replacement */
 62         u8  instrlen;           /* length of original instruction */
 63         u8  replacementlen;     /* length of new instruction */
 64         u8  padlen;             /* length of build-time padding */
 65 } __packed;
 66 
 67 /*
 68  * Debug flag that can be tested to see whether alternative
 69  * instructions were patched in already:
 70  */
 71 extern int alternatives_patched;
 72 
 73 extern void alternative_instructions(void);
 74 extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
 75 
 76 struct module;
 77 
 78 #ifdef CONFIG_SMP
 79 extern void alternatives_smp_module_add(struct module *mod, char *name,
 80                                         void *locks, void *locks_end,
 81                                         void *text, void *text_end);
 82 extern void alternatives_smp_module_del(struct module *mod);
 83 extern void alternatives_enable_smp(void);
 84 extern int alternatives_text_reserved(void *start, void *end);
 85 extern bool skip_smp_alternatives;
 86 #else
 87 static inline void alternatives_smp_module_add(struct module *mod, char *name,
 88                                                void *locks, void *locks_end,
 89                                                void *text, void *text_end) {}
 90 static inline void alternatives_smp_module_del(struct module *mod) {}
 91 static inline void alternatives_enable_smp(void) {}
 92 static inline int alternatives_text_reserved(void *start, void *end)
 93 {
 94         return 0;
 95 }
 96 #endif  /* CONFIG_SMP */
 97 
 98 #define b_replacement(num)      "664"#num
 99 #define e_replacement(num)      "665"#num
100 
101 #define alt_end_marker          "663"
102 #define alt_slen                "662b-661b"
103 #define alt_pad_len             alt_end_marker"b-662b"
104 #define alt_total_slen          alt_end_marker"b-661b"
105 #define alt_rlen(num)           e_replacement(num)"f-"b_replacement(num)"f"
106 
107 #define OLDINSTR(oldinstr, num)                                         \
108         "# ALT: oldnstr\n"                                              \
109         "661:\n\t" oldinstr "\n662:\n"                                  \
110         "# ALT: padding\n"                                              \
111         ".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * "          \
112                 "((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"           \
113         alt_end_marker ":\n"
114 
115 /*
116  * gas compatible max based on the idea from:
117  * http://graphics.stanford.edu/~seander/bithacks.html#IntegerMinOrMax
118  *
119  * The additional "-" is needed because gas uses a "true" value of -1.
120  */
121 #define alt_max_short(a, b)     "((" a ") ^ (((" a ") ^ (" b ")) & -(-((" a ") < (" b ")))))"
122 
123 /*
124  * Pad the second replacement alternative with additional NOPs if it is
125  * additionally longer than the first replacement alternative.
126  */
127 #define OLDINSTR_2(oldinstr, num1, num2) \
128         "# ALT: oldinstr2\n"                                                                    \
129         "661:\n\t" oldinstr "\n662:\n"                                                          \
130         "# ALT: padding2\n"                                                                     \
131         ".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * "  \
132                 "(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n"  \
133         alt_end_marker ":\n"
134 
135 #define OLDINSTR_3(oldinsn, n1, n2, n3)                                                         \
136         "# ALT: oldinstr3\n"                                                                    \
137         "661:\n\t" oldinsn "\n662:\n"                                                           \
138         "# ALT: padding3\n"                                                                     \
139         ".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))      \
140                 " - (" alt_slen ")) > 0) * "                                                    \
141                 "(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3))      \
142                 " - (" alt_slen ")), 0x90\n"                                                    \
143         alt_end_marker ":\n"
144 
145 #define ALTINSTR_ENTRY(feature, num)                                          \
146         " .long 661b - .\n"                             /* label           */ \
147         " .long " b_replacement(num)"f - .\n"           /* new instruction */ \
148         " .word " __stringify(feature) "\n"             /* feature bit     */ \
149         " .byte " alt_total_slen "\n"                   /* source len      */ \
150         " .byte " alt_rlen(num) "\n"                    /* replacement len */ \
151         " .byte " alt_pad_len "\n"                      /* pad len */
152 
153 #define ALTINSTR_REPLACEMENT(newinstr, feature, num)    /* replacement */       \
154         "# ALT: replacement " #num "\n"                                         \
155         b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
156 
157 /* alternative assembly primitive: */
158 #define ALTERNATIVE(oldinstr, newinstr, feature)                        \
159         OLDINSTR(oldinstr, 1)                                           \
160         ".pushsection .altinstructions,\"a\"\n"                         \
161         ALTINSTR_ENTRY(feature, 1)                                      \
162         ".popsection\n"                                                 \
163         ".pushsection .altinstr_replacement, \"ax\"\n"                  \
164         ALTINSTR_REPLACEMENT(newinstr, feature, 1)                      \
165         ".popsection\n"
166 
167 #define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
168         OLDINSTR_2(oldinstr, 1, 2)                                      \
169         ".pushsection .altinstructions,\"a\"\n"                         \
170         ALTINSTR_ENTRY(feature1, 1)                                     \
171         ALTINSTR_ENTRY(feature2, 2)                                     \
172         ".popsection\n"                                                 \
173         ".pushsection .altinstr_replacement, \"ax\"\n"                  \
174         ALTINSTR_REPLACEMENT(newinstr1, feature1, 1)                    \
175         ALTINSTR_REPLACEMENT(newinstr2, feature2, 2)                    \
176         ".popsection\n"
177 
178 #define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \
179         OLDINSTR_3(oldinsn, 1, 2, 3)                                            \
180         ".pushsection .altinstructions,\"a\"\n"                                 \
181         ALTINSTR_ENTRY(feat1, 1)                                                \
182         ALTINSTR_ENTRY(feat2, 2)                                                \
183         ALTINSTR_ENTRY(feat3, 3)                                                \
184         ".popsection\n"                                                         \
185         ".pushsection .altinstr_replacement, \"ax\"\n"                          \
186         ALTINSTR_REPLACEMENT(newinsn1, feat1, 1)                                \
187         ALTINSTR_REPLACEMENT(newinsn2, feat2, 2)                                \
188         ALTINSTR_REPLACEMENT(newinsn3, feat3, 3)                                \
189         ".popsection\n"
190 
191 /*
192  * Alternative instructions for different CPU types or capabilities.
193  *
194  * This allows to use optimized instructions even on generic binary
195  * kernels.
196  *
197  * length of oldinstr must be longer or equal the length of newinstr
198  * It can be padded with nops as needed.
199  *
200  * For non barrier like inlines please define new variants
201  * without volatile and memory clobber.
202  */
203 #define alternative(oldinstr, newinstr, feature)                        \
204         asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
205 
206 #define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
207         asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
208 
209 /*
210  * Alternative inline assembly with input.
211  *
212  * Peculiarities:
213  * No memory clobber here.
214  * Argument numbers start with 1.
215  * Best is to use constraints that are fixed size (like (%1) ... "r")
216  * If you use variable sized constraints like "m" or "g" in the
217  * replacement make sure to pad to the worst case length.
218  * Leaving an unused argument 0 to keep API compatibility.
219  */
220 #define alternative_input(oldinstr, newinstr, feature, input...)        \
221         asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
222                 : : "i" (0), ## input)
223 
224 /*
225  * This is similar to alternative_input. But it has two features and
226  * respective instructions.
227  *
228  * If CPU has feature2, newinstr2 is used.
229  * Otherwise, if CPU has feature1, newinstr1 is used.
230  * Otherwise, oldinstr is used.
231  */
232 #define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2,        \
233                            feature2, input...)                               \
234         asm volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1,            \
235                 newinstr2, feature2)                                         \
236                 : : "i" (0), ## input)
237 
238 /* Like alternative_input, but with a single output argument */
239 #define alternative_io(oldinstr, newinstr, feature, output, input...)   \
240         asm volatile (ALTERNATIVE(oldinstr, newinstr, feature)          \
241                 : output : "i" (0), ## input)
242 
243 /* Like alternative_io, but for replacing a direct call with another one. */
244 #define alternative_call(oldfunc, newfunc, feature, output, input...)   \
245         asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
246                 : output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
247 
248 /*
249  * Like alternative_call, but there are two features and respective functions.
250  * If CPU has feature2, function2 is used.
251  * Otherwise, if CPU has feature1, function1 is used.
252  * Otherwise, old function is used.
253  */
254 #define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2,   \
255                            output, input...)                                  \
256         asm volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
257                 "call %P[new2]", feature2)                                    \
258                 : output, ASM_CALL_CONSTRAINT                                 \
259                 : [old] "i" (oldfunc), [new1] "i" (newfunc1),                 \
260                   [new2] "i" (newfunc2), ## input)
261 
262 /*
263  * use this macro(s) if you need more than one output parameter
264  * in alternative_io
265  */
266 #define ASM_OUTPUT2(a...) a
267 
268 /*
269  * use this macro if you need clobbers but no inputs in
270  * alternative_{input,io,call}()
271  */
272 #define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
273 
274 #endif /* __ASSEMBLY__ */
275 
276 #endif /* _ASM_X86_ALTERNATIVE_H */
277 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp