~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/alternative.c

Version: ~ [ linux-5.1-rc5 ] ~ [ linux-5.0.7 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.34 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.111 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.168 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.178 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.138 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.65 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #define pr_fmt(fmt) "SMP alternatives: " fmt
  2 
  3 #include <linux/module.h>
  4 #include <linux/sched.h>
  5 #include <linux/mutex.h>
  6 #include <linux/list.h>
  7 #include <linux/stringify.h>
  8 #include <linux/mm.h>
  9 #include <linux/vmalloc.h>
 10 #include <linux/memory.h>
 11 #include <linux/stop_machine.h>
 12 #include <linux/slab.h>
 13 #include <linux/kdebug.h>
 14 #include <linux/kprobes.h>
 15 #include <asm/text-patching.h>
 16 #include <asm/alternative.h>
 17 #include <asm/sections.h>
 18 #include <asm/pgtable.h>
 19 #include <asm/mce.h>
 20 #include <asm/nmi.h>
 21 #include <asm/cacheflush.h>
 22 #include <asm/tlbflush.h>
 23 #include <asm/io.h>
 24 #include <asm/fixmap.h>
 25 
 26 int __read_mostly alternatives_patched;
 27 
 28 EXPORT_SYMBOL_GPL(alternatives_patched);
 29 
 30 #define MAX_PATCH_LEN (255-1)
 31 
 32 static int __initdata_or_module debug_alternative;
 33 
 34 static int __init debug_alt(char *str)
 35 {
 36         debug_alternative = 1;
 37         return 1;
 38 }
 39 __setup("debug-alternative", debug_alt);
 40 
 41 static int noreplace_smp;
 42 
 43 static int __init setup_noreplace_smp(char *str)
 44 {
 45         noreplace_smp = 1;
 46         return 1;
 47 }
 48 __setup("noreplace-smp", setup_noreplace_smp);
 49 
 50 #define DPRINTK(fmt, args...)                                           \
 51 do {                                                                    \
 52         if (debug_alternative)                                          \
 53                 printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args);   \
 54 } while (0)
 55 
 56 #define DUMP_BYTES(buf, len, fmt, args...)                              \
 57 do {                                                                    \
 58         if (unlikely(debug_alternative)) {                              \
 59                 int j;                                                  \
 60                                                                         \
 61                 if (!(len))                                             \
 62                         break;                                          \
 63                                                                         \
 64                 printk(KERN_DEBUG fmt, ##args);                         \
 65                 for (j = 0; j < (len) - 1; j++)                         \
 66                         printk(KERN_CONT "%02hhx ", buf[j]);            \
 67                 printk(KERN_CONT "%02hhx\n", buf[j]);                   \
 68         }                                                               \
 69 } while (0)
 70 
 71 /*
 72  * Each GENERIC_NOPX is of X bytes, and defined as an array of bytes
 73  * that correspond to that nop. Getting from one nop to the next, we
 74  * add to the array the offset that is equal to the sum of all sizes of
 75  * nops preceding the one we are after.
 76  *
 77  * Note: The GENERIC_NOP5_ATOMIC is at the end, as it breaks the
 78  * nice symmetry of sizes of the previous nops.
 79  */
 80 #if defined(GENERIC_NOP1) && !defined(CONFIG_X86_64)
 81 static const unsigned char intelnops[] =
 82 {
 83         GENERIC_NOP1,
 84         GENERIC_NOP2,
 85         GENERIC_NOP3,
 86         GENERIC_NOP4,
 87         GENERIC_NOP5,
 88         GENERIC_NOP6,
 89         GENERIC_NOP7,
 90         GENERIC_NOP8,
 91         GENERIC_NOP5_ATOMIC
 92 };
 93 static const unsigned char * const intel_nops[ASM_NOP_MAX+2] =
 94 {
 95         NULL,
 96         intelnops,
 97         intelnops + 1,
 98         intelnops + 1 + 2,
 99         intelnops + 1 + 2 + 3,
100         intelnops + 1 + 2 + 3 + 4,
101         intelnops + 1 + 2 + 3 + 4 + 5,
102         intelnops + 1 + 2 + 3 + 4 + 5 + 6,
103         intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
104         intelnops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
105 };
106 #endif
107 
108 #ifdef K8_NOP1
109 static const unsigned char k8nops[] =
110 {
111         K8_NOP1,
112         K8_NOP2,
113         K8_NOP3,
114         K8_NOP4,
115         K8_NOP5,
116         K8_NOP6,
117         K8_NOP7,
118         K8_NOP8,
119         K8_NOP5_ATOMIC
120 };
121 static const unsigned char * const k8_nops[ASM_NOP_MAX+2] =
122 {
123         NULL,
124         k8nops,
125         k8nops + 1,
126         k8nops + 1 + 2,
127         k8nops + 1 + 2 + 3,
128         k8nops + 1 + 2 + 3 + 4,
129         k8nops + 1 + 2 + 3 + 4 + 5,
130         k8nops + 1 + 2 + 3 + 4 + 5 + 6,
131         k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
132         k8nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
133 };
134 #endif
135 
136 #if defined(K7_NOP1) && !defined(CONFIG_X86_64)
137 static const unsigned char k7nops[] =
138 {
139         K7_NOP1,
140         K7_NOP2,
141         K7_NOP3,
142         K7_NOP4,
143         K7_NOP5,
144         K7_NOP6,
145         K7_NOP7,
146         K7_NOP8,
147         K7_NOP5_ATOMIC
148 };
149 static const unsigned char * const k7_nops[ASM_NOP_MAX+2] =
150 {
151         NULL,
152         k7nops,
153         k7nops + 1,
154         k7nops + 1 + 2,
155         k7nops + 1 + 2 + 3,
156         k7nops + 1 + 2 + 3 + 4,
157         k7nops + 1 + 2 + 3 + 4 + 5,
158         k7nops + 1 + 2 + 3 + 4 + 5 + 6,
159         k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
160         k7nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
161 };
162 #endif
163 
164 #ifdef P6_NOP1
165 static const unsigned char p6nops[] =
166 {
167         P6_NOP1,
168         P6_NOP2,
169         P6_NOP3,
170         P6_NOP4,
171         P6_NOP5,
172         P6_NOP6,
173         P6_NOP7,
174         P6_NOP8,
175         P6_NOP5_ATOMIC
176 };
177 static const unsigned char * const p6_nops[ASM_NOP_MAX+2] =
178 {
179         NULL,
180         p6nops,
181         p6nops + 1,
182         p6nops + 1 + 2,
183         p6nops + 1 + 2 + 3,
184         p6nops + 1 + 2 + 3 + 4,
185         p6nops + 1 + 2 + 3 + 4 + 5,
186         p6nops + 1 + 2 + 3 + 4 + 5 + 6,
187         p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7,
188         p6nops + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8,
189 };
190 #endif
191 
192 /* Initialize these to a safe default */
193 #ifdef CONFIG_X86_64
194 const unsigned char * const *ideal_nops = p6_nops;
195 #else
196 const unsigned char * const *ideal_nops = intel_nops;
197 #endif
198 
199 void __init arch_init_ideal_nops(void)
200 {
201         switch (boot_cpu_data.x86_vendor) {
202         case X86_VENDOR_INTEL:
203                 /*
204                  * Due to a decoder implementation quirk, some
205                  * specific Intel CPUs actually perform better with
206                  * the "k8_nops" than with the SDM-recommended NOPs.
207                  */
208                 if (boot_cpu_data.x86 == 6 &&
209                     boot_cpu_data.x86_model >= 0x0f &&
210                     boot_cpu_data.x86_model != 0x1c &&
211                     boot_cpu_data.x86_model != 0x26 &&
212                     boot_cpu_data.x86_model != 0x27 &&
213                     boot_cpu_data.x86_model < 0x30) {
214                         ideal_nops = k8_nops;
215                 } else if (boot_cpu_has(X86_FEATURE_NOPL)) {
216                            ideal_nops = p6_nops;
217                 } else {
218 #ifdef CONFIG_X86_64
219                         ideal_nops = k8_nops;
220 #else
221                         ideal_nops = intel_nops;
222 #endif
223                 }
224                 break;
225 
226         case X86_VENDOR_HYGON:
227                 ideal_nops = p6_nops;
228                 return;
229 
230         case X86_VENDOR_AMD:
231                 if (boot_cpu_data.x86 > 0xf) {
232                         ideal_nops = p6_nops;
233                         return;
234                 }
235 
236                 /* fall through */
237 
238         default:
239 #ifdef CONFIG_X86_64
240                 ideal_nops = k8_nops;
241 #else
242                 if (boot_cpu_has(X86_FEATURE_K8))
243                         ideal_nops = k8_nops;
244                 else if (boot_cpu_has(X86_FEATURE_K7))
245                         ideal_nops = k7_nops;
246                 else
247                         ideal_nops = intel_nops;
248 #endif
249         }
250 }
251 
252 /* Use this to add nops to a buffer, then text_poke the whole buffer. */
253 static void __init_or_module add_nops(void *insns, unsigned int len)
254 {
255         while (len > 0) {
256                 unsigned int noplen = len;
257                 if (noplen > ASM_NOP_MAX)
258                         noplen = ASM_NOP_MAX;
259                 memcpy(insns, ideal_nops[noplen], noplen);
260                 insns += noplen;
261                 len -= noplen;
262         }
263 }
264 
265 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
266 extern s32 __smp_locks[], __smp_locks_end[];
267 void *text_poke_early(void *addr, const void *opcode, size_t len);
268 
269 /*
270  * Are we looking at a near JMP with a 1 or 4-byte displacement.
271  */
272 static inline bool is_jmp(const u8 opcode)
273 {
274         return opcode == 0xeb || opcode == 0xe9;
275 }
276 
277 static void __init_or_module
278 recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
279 {
280         u8 *next_rip, *tgt_rip;
281         s32 n_dspl, o_dspl;
282         int repl_len;
283 
284         if (a->replacementlen != 5)
285                 return;
286 
287         o_dspl = *(s32 *)(insnbuf + 1);
288 
289         /* next_rip of the replacement JMP */
290         next_rip = repl_insn + a->replacementlen;
291         /* target rip of the replacement JMP */
292         tgt_rip  = next_rip + o_dspl;
293         n_dspl = tgt_rip - orig_insn;
294 
295         DPRINTK("target RIP: %px, new_displ: 0x%x", tgt_rip, n_dspl);
296 
297         if (tgt_rip - orig_insn >= 0) {
298                 if (n_dspl - 2 <= 127)
299                         goto two_byte_jmp;
300                 else
301                         goto five_byte_jmp;
302         /* negative offset */
303         } else {
304                 if (((n_dspl - 2) & 0xff) == (n_dspl - 2))
305                         goto two_byte_jmp;
306                 else
307                         goto five_byte_jmp;
308         }
309 
310 two_byte_jmp:
311         n_dspl -= 2;
312 
313         insnbuf[0] = 0xeb;
314         insnbuf[1] = (s8)n_dspl;
315         add_nops(insnbuf + 2, 3);
316 
317         repl_len = 2;
318         goto done;
319 
320 five_byte_jmp:
321         n_dspl -= 5;
322 
323         insnbuf[0] = 0xe9;
324         *(s32 *)&insnbuf[1] = n_dspl;
325 
326         repl_len = 5;
327 
328 done:
329 
330         DPRINTK("final displ: 0x%08x, JMP 0x%lx",
331                 n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
332 }
333 
334 /*
335  * "noinline" to cause control flow change and thus invalidate I$ and
336  * cause refetch after modification.
337  */
338 static void __init_or_module noinline optimize_nops(struct alt_instr *a, u8 *instr)
339 {
340         unsigned long flags;
341         int i;
342 
343         for (i = 0; i < a->padlen; i++) {
344                 if (instr[i] != 0x90)
345                         return;
346         }
347 
348         local_irq_save(flags);
349         add_nops(instr + (a->instrlen - a->padlen), a->padlen);
350         local_irq_restore(flags);
351 
352         DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",
353                    instr, a->instrlen - a->padlen, a->padlen);
354 }
355 
356 /*
357  * Replace instructions with better alternatives for this CPU type. This runs
358  * before SMP is initialized to avoid SMP problems with self modifying code.
359  * This implies that asymmetric systems where APs have less capabilities than
360  * the boot processor are not handled. Tough. Make sure you disable such
361  * features by hand.
362  *
363  * Marked "noinline" to cause control flow change and thus insn cache
364  * to refetch changed I$ lines.
365  */
366 void __init_or_module noinline apply_alternatives(struct alt_instr *start,
367                                                   struct alt_instr *end)
368 {
369         struct alt_instr *a;
370         u8 *instr, *replacement;
371         u8 insnbuf[MAX_PATCH_LEN];
372 
373         DPRINTK("alt table %px, -> %px", start, end);
374         /*
375          * The scan order should be from start to end. A later scanned
376          * alternative code can overwrite previously scanned alternative code.
377          * Some kernel functions (e.g. memcpy, memset, etc) use this order to
378          * patch code.
379          *
380          * So be careful if you want to change the scan order to any other
381          * order.
382          */
383         for (a = start; a < end; a++) {
384                 int insnbuf_sz = 0;
385 
386                 instr = (u8 *)&a->instr_offset + a->instr_offset;
387                 replacement = (u8 *)&a->repl_offset + a->repl_offset;
388                 BUG_ON(a->instrlen > sizeof(insnbuf));
389                 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
390                 if (!boot_cpu_has(a->cpuid)) {
391                         if (a->padlen > 1)
392                                 optimize_nops(a, instr);
393 
394                         continue;
395                 }
396 
397                 DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
398                         a->cpuid >> 5,
399                         a->cpuid & 0x1f,
400                         instr, instr, a->instrlen,
401                         replacement, a->replacementlen, a->padlen);
402 
403                 DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);
404                 DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);
405 
406                 memcpy(insnbuf, replacement, a->replacementlen);
407                 insnbuf_sz = a->replacementlen;
408 
409                 /*
410                  * 0xe8 is a relative jump; fix the offset.
411                  *
412                  * Instruction length is checked before the opcode to avoid
413                  * accessing uninitialized bytes for zero-length replacements.
414                  */
415                 if (a->replacementlen == 5 && *insnbuf == 0xe8) {
416                         *(s32 *)(insnbuf + 1) += replacement - instr;
417                         DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
418                                 *(s32 *)(insnbuf + 1),
419                                 (unsigned long)instr + *(s32 *)(insnbuf + 1) + 5);
420                 }
421 
422                 if (a->replacementlen && is_jmp(replacement[0]))
423                         recompute_jump(a, instr, replacement, insnbuf);
424 
425                 if (a->instrlen > a->replacementlen) {
426                         add_nops(insnbuf + a->replacementlen,
427                                  a->instrlen - a->replacementlen);
428                         insnbuf_sz += a->instrlen - a->replacementlen;
429                 }
430                 DUMP_BYTES(insnbuf, insnbuf_sz, "%px: final_insn: ", instr);
431 
432                 text_poke_early(instr, insnbuf, insnbuf_sz);
433         }
434 }
435 
436 #ifdef CONFIG_SMP
437 static void alternatives_smp_lock(const s32 *start, const s32 *end,
438                                   u8 *text, u8 *text_end)
439 {
440         const s32 *poff;
441 
442         for (poff = start; poff < end; poff++) {
443                 u8 *ptr = (u8 *)poff + *poff;
444 
445                 if (!*poff || ptr < text || ptr >= text_end)
446                         continue;
447                 /* turn DS segment override prefix into lock prefix */
448                 if (*ptr == 0x3e)
449                         text_poke(ptr, ((unsigned char []){0xf0}), 1);
450         }
451 }
452 
453 static void alternatives_smp_unlock(const s32 *start, const s32 *end,
454                                     u8 *text, u8 *text_end)
455 {
456         const s32 *poff;
457 
458         for (poff = start; poff < end; poff++) {
459                 u8 *ptr = (u8 *)poff + *poff;
460 
461                 if (!*poff || ptr < text || ptr >= text_end)
462                         continue;
463                 /* turn lock prefix into DS segment override prefix */
464                 if (*ptr == 0xf0)
465                         text_poke(ptr, ((unsigned char []){0x3E}), 1);
466         }
467 }
468 
469 struct smp_alt_module {
470         /* what is this ??? */
471         struct module   *mod;
472         char            *name;
473 
474         /* ptrs to lock prefixes */
475         const s32       *locks;
476         const s32       *locks_end;
477 
478         /* .text segment, needed to avoid patching init code ;) */
479         u8              *text;
480         u8              *text_end;
481 
482         struct list_head next;
483 };
484 static LIST_HEAD(smp_alt_modules);
485 static bool uniproc_patched = false;    /* protected by text_mutex */
486 
487 void __init_or_module alternatives_smp_module_add(struct module *mod,
488                                                   char *name,
489                                                   void *locks, void *locks_end,
490                                                   void *text,  void *text_end)
491 {
492         struct smp_alt_module *smp;
493 
494         mutex_lock(&text_mutex);
495         if (!uniproc_patched)
496                 goto unlock;
497 
498         if (num_possible_cpus() == 1)
499                 /* Don't bother remembering, we'll never have to undo it. */
500                 goto smp_unlock;
501 
502         smp = kzalloc(sizeof(*smp), GFP_KERNEL);
503         if (NULL == smp)
504                 /* we'll run the (safe but slow) SMP code then ... */
505                 goto unlock;
506 
507         smp->mod        = mod;
508         smp->name       = name;
509         smp->locks      = locks;
510         smp->locks_end  = locks_end;
511         smp->text       = text;
512         smp->text_end   = text_end;
513         DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",
514                 smp->locks, smp->locks_end,
515                 smp->text, smp->text_end, smp->name);
516 
517         list_add_tail(&smp->next, &smp_alt_modules);
518 smp_unlock:
519         alternatives_smp_unlock(locks, locks_end, text, text_end);
520 unlock:
521         mutex_unlock(&text_mutex);
522 }
523 
524 void __init_or_module alternatives_smp_module_del(struct module *mod)
525 {
526         struct smp_alt_module *item;
527 
528         mutex_lock(&text_mutex);
529         list_for_each_entry(item, &smp_alt_modules, next) {
530                 if (mod != item->mod)
531                         continue;
532                 list_del(&item->next);
533                 kfree(item);
534                 break;
535         }
536         mutex_unlock(&text_mutex);
537 }
538 
539 void alternatives_enable_smp(void)
540 {
541         struct smp_alt_module *mod;
542 
543         /* Why bother if there are no other CPUs? */
544         BUG_ON(num_possible_cpus() == 1);
545 
546         mutex_lock(&text_mutex);
547 
548         if (uniproc_patched) {
549                 pr_info("switching to SMP code\n");
550                 BUG_ON(num_online_cpus() != 1);
551                 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
552                 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
553                 list_for_each_entry(mod, &smp_alt_modules, next)
554                         alternatives_smp_lock(mod->locks, mod->locks_end,
555                                               mod->text, mod->text_end);
556                 uniproc_patched = false;
557         }
558         mutex_unlock(&text_mutex);
559 }
560 
561 /*
562  * Return 1 if the address range is reserved for SMP-alternatives.
563  * Must hold text_mutex.
564  */
565 int alternatives_text_reserved(void *start, void *end)
566 {
567         struct smp_alt_module *mod;
568         const s32 *poff;
569         u8 *text_start = start;
570         u8 *text_end = end;
571 
572         lockdep_assert_held(&text_mutex);
573 
574         list_for_each_entry(mod, &smp_alt_modules, next) {
575                 if (mod->text > text_end || mod->text_end < text_start)
576                         continue;
577                 for (poff = mod->locks; poff < mod->locks_end; poff++) {
578                         const u8 *ptr = (const u8 *)poff + *poff;
579 
580                         if (text_start <= ptr && text_end > ptr)
581                                 return 1;
582                 }
583         }
584 
585         return 0;
586 }
587 #endif /* CONFIG_SMP */
588 
589 #ifdef CONFIG_PARAVIRT
590 void __init_or_module apply_paravirt(struct paravirt_patch_site *start,
591                                      struct paravirt_patch_site *end)
592 {
593         struct paravirt_patch_site *p;
594         char insnbuf[MAX_PATCH_LEN];
595 
596         for (p = start; p < end; p++) {
597                 unsigned int used;
598 
599                 BUG_ON(p->len > MAX_PATCH_LEN);
600                 /* prep the buffer with the original instructions */
601                 memcpy(insnbuf, p->instr, p->len);
602                 used = pv_ops.init.patch(p->instrtype, insnbuf,
603                                          (unsigned long)p->instr, p->len);
604 
605                 BUG_ON(used > p->len);
606 
607                 /* Pad the rest with nops */
608                 add_nops(insnbuf + used, p->len - used);
609                 text_poke_early(p->instr, insnbuf, p->len);
610         }
611 }
612 extern struct paravirt_patch_site __start_parainstructions[],
613         __stop_parainstructions[];
614 #endif  /* CONFIG_PARAVIRT */
615 
616 void __init alternative_instructions(void)
617 {
618         /* The patching is not fully atomic, so try to avoid local interruptions
619            that might execute the to be patched code.
620            Other CPUs are not running. */
621         stop_nmi();
622 
623         /*
624          * Don't stop machine check exceptions while patching.
625          * MCEs only happen when something got corrupted and in this
626          * case we must do something about the corruption.
627          * Ignoring it is worse than a unlikely patching race.
628          * Also machine checks tend to be broadcast and if one CPU
629          * goes into machine check the others follow quickly, so we don't
630          * expect a machine check to cause undue problems during to code
631          * patching.
632          */
633 
634         apply_alternatives(__alt_instructions, __alt_instructions_end);
635 
636 #ifdef CONFIG_SMP
637         /* Patch to UP if other cpus not imminent. */
638         if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
639                 uniproc_patched = true;
640                 alternatives_smp_module_add(NULL, "core kernel",
641                                             __smp_locks, __smp_locks_end,
642                                             _text, _etext);
643         }
644 
645         if (!uniproc_patched || num_possible_cpus() == 1)
646                 free_init_pages("SMP alternatives",
647                                 (unsigned long)__smp_locks,
648                                 (unsigned long)__smp_locks_end);
649 #endif
650 
651         apply_paravirt(__parainstructions, __parainstructions_end);
652 
653         restart_nmi();
654         alternatives_patched = 1;
655 }
656 
657 /**
658  * text_poke_early - Update instructions on a live kernel at boot time
659  * @addr: address to modify
660  * @opcode: source of the copy
661  * @len: length to copy
662  *
663  * When you use this code to patch more than one byte of an instruction
664  * you need to make sure that other CPUs cannot execute this code in parallel.
665  * Also no thread must be currently preempted in the middle of these
666  * instructions. And on the local CPU you need to be protected again NMI or MCE
667  * handlers seeing an inconsistent instruction while you patch.
668  */
669 void *__init_or_module text_poke_early(void *addr, const void *opcode,
670                                               size_t len)
671 {
672         unsigned long flags;
673         local_irq_save(flags);
674         memcpy(addr, opcode, len);
675         local_irq_restore(flags);
676         sync_core();
677         /* Could also do a CLFLUSH here to speed up CPU recovery; but
678            that causes hangs on some VIA CPUs. */
679         return addr;
680 }
681 
682 /**
683  * text_poke - Update instructions on a live kernel
684  * @addr: address to modify
685  * @opcode: source of the copy
686  * @len: length to copy
687  *
688  * Only atomic text poke/set should be allowed when not doing early patching.
689  * It means the size must be writable atomically and the address must be aligned
690  * in a way that permits an atomic write. It also makes sure we fit on a single
691  * page.
692  */
693 void *text_poke(void *addr, const void *opcode, size_t len)
694 {
695         unsigned long flags;
696         char *vaddr;
697         struct page *pages[2];
698         int i;
699 
700         /*
701          * While boot memory allocator is runnig we cannot use struct
702          * pages as they are not yet initialized.
703          */
704         BUG_ON(!after_bootmem);
705 
706         lockdep_assert_held(&text_mutex);
707 
708         if (!core_kernel_text((unsigned long)addr)) {
709                 pages[0] = vmalloc_to_page(addr);
710                 pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
711         } else {
712                 pages[0] = virt_to_page(addr);
713                 WARN_ON(!PageReserved(pages[0]));
714                 pages[1] = virt_to_page(addr + PAGE_SIZE);
715         }
716         BUG_ON(!pages[0]);
717         local_irq_save(flags);
718         set_fixmap(FIX_TEXT_POKE0, page_to_phys(pages[0]));
719         if (pages[1])
720                 set_fixmap(FIX_TEXT_POKE1, page_to_phys(pages[1]));
721         vaddr = (char *)fix_to_virt(FIX_TEXT_POKE0);
722         memcpy(&vaddr[(unsigned long)addr & ~PAGE_MASK], opcode, len);
723         clear_fixmap(FIX_TEXT_POKE0);
724         if (pages[1])
725                 clear_fixmap(FIX_TEXT_POKE1);
726         local_flush_tlb();
727         sync_core();
728         /* Could also do a CLFLUSH here to speed up CPU recovery; but
729            that causes hangs on some VIA CPUs. */
730         for (i = 0; i < len; i++)
731                 BUG_ON(((char *)addr)[i] != ((char *)opcode)[i]);
732         local_irq_restore(flags);
733         return addr;
734 }
735 
736 static void do_sync_core(void *info)
737 {
738         sync_core();
739 }
740 
741 static bool bp_patching_in_progress;
742 static void *bp_int3_handler, *bp_int3_addr;
743 
744 int poke_int3_handler(struct pt_regs *regs)
745 {
746         /*
747          * Having observed our INT3 instruction, we now must observe
748          * bp_patching_in_progress.
749          *
750          *      in_progress = TRUE              INT3
751          *      WMB                             RMB
752          *      write INT3                      if (in_progress)
753          *
754          * Idem for bp_int3_handler.
755          */
756         smp_rmb();
757 
758         if (likely(!bp_patching_in_progress))
759                 return 0;
760 
761         if (user_mode(regs) || regs->ip != (unsigned long)bp_int3_addr)
762                 return 0;
763 
764         /* set up the specified breakpoint handler */
765         regs->ip = (unsigned long) bp_int3_handler;
766 
767         return 1;
768 }
769 NOKPROBE_SYMBOL(poke_int3_handler);
770 
771 /**
772  * text_poke_bp() -- update instructions on live kernel on SMP
773  * @addr:       address to patch
774  * @opcode:     opcode of new instruction
775  * @len:        length to copy
776  * @handler:    address to jump to when the temporary breakpoint is hit
777  *
778  * Modify multi-byte instruction by using int3 breakpoint on SMP.
779  * We completely avoid stop_machine() here, and achieve the
780  * synchronization using int3 breakpoint.
781  *
782  * The way it is done:
783  *      - add a int3 trap to the address that will be patched
784  *      - sync cores
785  *      - update all but the first byte of the patched range
786  *      - sync cores
787  *      - replace the first byte (int3) by the first byte of
788  *        replacing opcode
789  *      - sync cores
790  */
791 void *text_poke_bp(void *addr, const void *opcode, size_t len, void *handler)
792 {
793         unsigned char int3 = 0xcc;
794 
795         bp_int3_handler = handler;
796         bp_int3_addr = (u8 *)addr + sizeof(int3);
797         bp_patching_in_progress = true;
798 
799         lockdep_assert_held(&text_mutex);
800 
801         /*
802          * Corresponding read barrier in int3 notifier for making sure the
803          * in_progress and handler are correctly ordered wrt. patching.
804          */
805         smp_wmb();
806 
807         text_poke(addr, &int3, sizeof(int3));
808 
809         on_each_cpu(do_sync_core, NULL, 1);
810 
811         if (len - sizeof(int3) > 0) {
812                 /* patch all but the first byte */
813                 text_poke((char *)addr + sizeof(int3),
814                           (const char *) opcode + sizeof(int3),
815                           len - sizeof(int3));
816                 /*
817                  * According to Intel, this core syncing is very likely
818                  * not necessary and we'd be safe even without it. But
819                  * better safe than sorry (plus there's not only Intel).
820                  */
821                 on_each_cpu(do_sync_core, NULL, 1);
822         }
823 
824         /* patch the first byte */
825         text_poke(addr, opcode, sizeof(int3));
826 
827         on_each_cpu(do_sync_core, NULL, 1);
828         /*
829          * sync_core() implies an smp_mb() and orders this store against
830          * the writing of the new instruction.
831          */
832         bp_patching_in_progress = false;
833 
834         return addr;
835 }
836 
837 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp