~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/kernel/paravirt.c

Version: ~ [ linux-5.16 ] ~ [ linux-5.15.13 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.90 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.170 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.224 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.261 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.296 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.298 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-or-later
  2 /*  Paravirtualization interfaces
  3     Copyright (C) 2006 Rusty Russell IBM Corporation
  4 
  5 
  6     2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc
  7 */
  8 
  9 #include <linux/errno.h>
 10 #include <linux/init.h>
 11 #include <linux/export.h>
 12 #include <linux/efi.h>
 13 #include <linux/bcd.h>
 14 #include <linux/highmem.h>
 15 #include <linux/kprobes.h>
 16 #include <linux/pgtable.h>
 17 #include <linux/static_call.h>
 18 
 19 #include <asm/bug.h>
 20 #include <asm/paravirt.h>
 21 #include <asm/debugreg.h>
 22 #include <asm/desc.h>
 23 #include <asm/setup.h>
 24 #include <asm/time.h>
 25 #include <asm/pgalloc.h>
 26 #include <asm/irq.h>
 27 #include <asm/delay.h>
 28 #include <asm/fixmap.h>
 29 #include <asm/apic.h>
 30 #include <asm/tlbflush.h>
 31 #include <asm/timer.h>
 32 #include <asm/special_insns.h>
 33 #include <asm/tlb.h>
 34 #include <asm/io_bitmap.h>
 35 
 36 /*
 37  * nop stub, which must not clobber anything *including the stack* to
 38  * avoid confusing the entry prologues.
 39  */
 40 extern void _paravirt_nop(void);
 41 asm (".pushsection .entry.text, \"ax\"\n"
 42      ".global _paravirt_nop\n"
 43      "_paravirt_nop:\n\t"
 44      "ret\n\t"
 45      ".size _paravirt_nop, . - _paravirt_nop\n\t"
 46      ".type _paravirt_nop, @function\n\t"
 47      ".popsection");
 48 
 49 /* stub always returning 0. */
 50 asm (".pushsection .entry.text, \"ax\"\n"
 51      ".global paravirt_ret0\n"
 52      "paravirt_ret0:\n\t"
 53      "xor %" _ASM_AX ", %" _ASM_AX ";\n\t"
 54      "ret\n\t"
 55      ".size paravirt_ret0, . - paravirt_ret0\n\t"
 56      ".type paravirt_ret0, @function\n\t"
 57      ".popsection");
 58 
 59 
 60 void __init default_banner(void)
 61 {
 62         printk(KERN_INFO "Booting paravirtualized kernel on %s\n",
 63                pv_info.name);
 64 }
 65 
 66 /* Undefined instruction for dealing with missing ops pointers. */
 67 noinstr void paravirt_BUG(void)
 68 {
 69         BUG();
 70 }
 71 
 72 struct branch {
 73         unsigned char opcode;
 74         u32 delta;
 75 } __attribute__((packed));
 76 
 77 static unsigned paravirt_patch_call(void *insn_buff, const void *target,
 78                                     unsigned long addr, unsigned len)
 79 {
 80         const int call_len = 5;
 81         struct branch *b = insn_buff;
 82         unsigned long delta = (unsigned long)target - (addr+call_len);
 83 
 84         if (len < call_len) {
 85                 pr_warn("paravirt: Failed to patch indirect CALL at %ps\n", (void *)addr);
 86                 /* Kernel might not be viable if patching fails, bail out: */
 87                 BUG_ON(1);
 88         }
 89 
 90         b->opcode = 0xe8; /* call */
 91         b->delta = delta;
 92         BUILD_BUG_ON(sizeof(*b) != call_len);
 93 
 94         return call_len;
 95 }
 96 
 97 #ifdef CONFIG_PARAVIRT_XXL
 98 /* identity function, which can be inlined */
 99 u64 notrace _paravirt_ident_64(u64 x)
100 {
101         return x;
102 }
103 #endif
104 
105 DEFINE_STATIC_KEY_TRUE(virt_spin_lock_key);
106 
107 void __init native_pv_lock_init(void)
108 {
109         if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
110                 static_branch_disable(&virt_spin_lock_key);
111 }
112 
113 unsigned int paravirt_patch(u8 type, void *insn_buff, unsigned long addr,
114                             unsigned int len)
115 {
116         /*
117          * Neat trick to map patch type back to the call within the
118          * corresponding structure.
119          */
120         void *opfunc = *((void **)&pv_ops + type);
121         unsigned ret;
122 
123         if (opfunc == NULL)
124                 /* If there's no function, patch it with paravirt_BUG() */
125                 ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len);
126         else if (opfunc == _paravirt_nop)
127                 ret = 0;
128         else
129                 /* Otherwise call the function. */
130                 ret = paravirt_patch_call(insn_buff, opfunc, addr, len);
131 
132         return ret;
133 }
134 
135 struct static_key paravirt_steal_enabled;
136 struct static_key paravirt_steal_rq_enabled;
137 
138 static u64 native_steal_clock(int cpu)
139 {
140         return 0;
141 }
142 
143 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
144 DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock);
145 
146 void paravirt_set_sched_clock(u64 (*func)(void))
147 {
148         static_call_update(pv_sched_clock, func);
149 }
150 
151 /* These are in entry.S */
152 extern void native_iret(void);
153 
154 static struct resource reserve_ioports = {
155         .start = 0,
156         .end = IO_SPACE_LIMIT,
157         .name = "paravirt-ioport",
158         .flags = IORESOURCE_IO | IORESOURCE_BUSY,
159 };
160 
161 /*
162  * Reserve the whole legacy IO space to prevent any legacy drivers
163  * from wasting time probing for their hardware.  This is a fairly
164  * brute-force approach to disabling all non-virtual drivers.
165  *
166  * Note that this must be called very early to have any effect.
167  */
168 int paravirt_disable_iospace(void)
169 {
170         return request_resource(&ioport_resource, &reserve_ioports);
171 }
172 
173 static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LAZY_NONE;
174 
175 static inline void enter_lazy(enum paravirt_lazy_mode mode)
176 {
177         BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
178 
179         this_cpu_write(paravirt_lazy_mode, mode);
180 }
181 
182 static void leave_lazy(enum paravirt_lazy_mode mode)
183 {
184         BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
185 
186         this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
187 }
188 
189 void paravirt_enter_lazy_mmu(void)
190 {
191         enter_lazy(PARAVIRT_LAZY_MMU);
192 }
193 
194 void paravirt_leave_lazy_mmu(void)
195 {
196         leave_lazy(PARAVIRT_LAZY_MMU);
197 }
198 
199 void paravirt_flush_lazy_mmu(void)
200 {
201         preempt_disable();
202 
203         if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_MMU) {
204                 arch_leave_lazy_mmu_mode();
205                 arch_enter_lazy_mmu_mode();
206         }
207 
208         preempt_enable();
209 }
210 
211 #ifdef CONFIG_PARAVIRT_XXL
212 void paravirt_start_context_switch(struct task_struct *prev)
213 {
214         BUG_ON(preemptible());
215 
216         if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
217                 arch_leave_lazy_mmu_mode();
218                 set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
219         }
220         enter_lazy(PARAVIRT_LAZY_CPU);
221 }
222 
223 void paravirt_end_context_switch(struct task_struct *next)
224 {
225         BUG_ON(preemptible());
226 
227         leave_lazy(PARAVIRT_LAZY_CPU);
228 
229         if (test_and_clear_ti_thread_flag(task_thread_info(next), TIF_LAZY_MMU_UPDATES))
230                 arch_enter_lazy_mmu_mode();
231 }
232 
233 static noinstr unsigned long pv_native_read_cr2(void)
234 {
235         return native_read_cr2();
236 }
237 
238 static noinstr void pv_native_write_cr2(unsigned long val)
239 {
240         native_write_cr2(val);
241 }
242 
243 static noinstr unsigned long pv_native_get_debugreg(int regno)
244 {
245         return native_get_debugreg(regno);
246 }
247 
248 static noinstr void pv_native_set_debugreg(int regno, unsigned long val)
249 {
250         native_set_debugreg(regno, val);
251 }
252 
253 static noinstr void pv_native_irq_enable(void)
254 {
255         native_irq_enable();
256 }
257 
258 static noinstr void pv_native_irq_disable(void)
259 {
260         native_irq_disable();
261 }
262 #endif
263 
264 enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
265 {
266         if (in_interrupt())
267                 return PARAVIRT_LAZY_NONE;
268 
269         return this_cpu_read(paravirt_lazy_mode);
270 }
271 
272 struct pv_info pv_info = {
273         .name = "bare hardware",
274 #ifdef CONFIG_PARAVIRT_XXL
275         .extra_user_64bit_cs = __USER_CS,
276 #endif
277 };
278 
279 /* 64-bit pagetable entries */
280 #define PTE_IDENT       __PV_IS_CALLEE_SAVE(_paravirt_ident_64)
281 
282 struct paravirt_patch_template pv_ops = {
283         /* Cpu ops. */
284         .cpu.io_delay           = native_io_delay,
285 
286 #ifdef CONFIG_PARAVIRT_XXL
287         .cpu.cpuid              = native_cpuid,
288         .cpu.get_debugreg       = pv_native_get_debugreg,
289         .cpu.set_debugreg       = pv_native_set_debugreg,
290         .cpu.read_cr0           = native_read_cr0,
291         .cpu.write_cr0          = native_write_cr0,
292         .cpu.write_cr4          = native_write_cr4,
293         .cpu.wbinvd             = native_wbinvd,
294         .cpu.read_msr           = native_read_msr,
295         .cpu.write_msr          = native_write_msr,
296         .cpu.read_msr_safe      = native_read_msr_safe,
297         .cpu.write_msr_safe     = native_write_msr_safe,
298         .cpu.read_pmc           = native_read_pmc,
299         .cpu.load_tr_desc       = native_load_tr_desc,
300         .cpu.set_ldt            = native_set_ldt,
301         .cpu.load_gdt           = native_load_gdt,
302         .cpu.load_idt           = native_load_idt,
303         .cpu.store_tr           = native_store_tr,
304         .cpu.load_tls           = native_load_tls,
305         .cpu.load_gs_index      = native_load_gs_index,
306         .cpu.write_ldt_entry    = native_write_ldt_entry,
307         .cpu.write_gdt_entry    = native_write_gdt_entry,
308         .cpu.write_idt_entry    = native_write_idt_entry,
309 
310         .cpu.alloc_ldt          = paravirt_nop,
311         .cpu.free_ldt           = paravirt_nop,
312 
313         .cpu.load_sp0           = native_load_sp0,
314 
315 #ifdef CONFIG_X86_IOPL_IOPERM
316         .cpu.invalidate_io_bitmap       = native_tss_invalidate_io_bitmap,
317         .cpu.update_io_bitmap           = native_tss_update_io_bitmap,
318 #endif
319 
320         .cpu.start_context_switch       = paravirt_nop,
321         .cpu.end_context_switch         = paravirt_nop,
322 
323         /* Irq ops. */
324         .irq.save_fl            = __PV_IS_CALLEE_SAVE(native_save_fl),
325         .irq.irq_disable        = __PV_IS_CALLEE_SAVE(pv_native_irq_disable),
326         .irq.irq_enable         = __PV_IS_CALLEE_SAVE(pv_native_irq_enable),
327         .irq.safe_halt          = native_safe_halt,
328         .irq.halt               = native_halt,
329 #endif /* CONFIG_PARAVIRT_XXL */
330 
331         /* Mmu ops. */
332         .mmu.flush_tlb_user     = native_flush_tlb_local,
333         .mmu.flush_tlb_kernel   = native_flush_tlb_global,
334         .mmu.flush_tlb_one_user = native_flush_tlb_one_user,
335         .mmu.flush_tlb_multi    = native_flush_tlb_multi,
336         .mmu.tlb_remove_table   =
337                         (void (*)(struct mmu_gather *, void *))tlb_remove_page,
338 
339         .mmu.exit_mmap          = paravirt_nop,
340         .mmu.notify_page_enc_status_changed     = paravirt_nop,
341 
342 #ifdef CONFIG_PARAVIRT_XXL
343         .mmu.read_cr2           = __PV_IS_CALLEE_SAVE(pv_native_read_cr2),
344         .mmu.write_cr2          = pv_native_write_cr2,
345         .mmu.read_cr3           = __native_read_cr3,
346         .mmu.write_cr3          = native_write_cr3,
347 
348         .mmu.pgd_alloc          = __paravirt_pgd_alloc,
349         .mmu.pgd_free           = paravirt_nop,
350 
351         .mmu.alloc_pte          = paravirt_nop,
352         .mmu.alloc_pmd          = paravirt_nop,
353         .mmu.alloc_pud          = paravirt_nop,
354         .mmu.alloc_p4d          = paravirt_nop,
355         .mmu.release_pte        = paravirt_nop,
356         .mmu.release_pmd        = paravirt_nop,
357         .mmu.release_pud        = paravirt_nop,
358         .mmu.release_p4d        = paravirt_nop,
359 
360         .mmu.set_pte            = native_set_pte,
361         .mmu.set_pmd            = native_set_pmd,
362 
363         .mmu.ptep_modify_prot_start     = __ptep_modify_prot_start,
364         .mmu.ptep_modify_prot_commit    = __ptep_modify_prot_commit,
365 
366         .mmu.set_pud            = native_set_pud,
367 
368         .mmu.pmd_val            = PTE_IDENT,
369         .mmu.make_pmd           = PTE_IDENT,
370 
371         .mmu.pud_val            = PTE_IDENT,
372         .mmu.make_pud           = PTE_IDENT,
373 
374         .mmu.set_p4d            = native_set_p4d,
375 
376 #if CONFIG_PGTABLE_LEVELS >= 5
377         .mmu.p4d_val            = PTE_IDENT,
378         .mmu.make_p4d           = PTE_IDENT,
379 
380         .mmu.set_pgd            = native_set_pgd,
381 #endif /* CONFIG_PGTABLE_LEVELS >= 5 */
382 
383         .mmu.pte_val            = PTE_IDENT,
384         .mmu.pgd_val            = PTE_IDENT,
385 
386         .mmu.make_pte           = PTE_IDENT,
387         .mmu.make_pgd           = PTE_IDENT,
388 
389         .mmu.dup_mmap           = paravirt_nop,
390         .mmu.activate_mm        = paravirt_nop,
391 
392         .mmu.lazy_mode = {
393                 .enter          = paravirt_nop,
394                 .leave          = paravirt_nop,
395                 .flush          = paravirt_nop,
396         },
397 
398         .mmu.set_fixmap         = native_set_fixmap,
399 #endif /* CONFIG_PARAVIRT_XXL */
400 
401 #if defined(CONFIG_PARAVIRT_SPINLOCKS)
402         /* Lock ops. */
403 #ifdef CONFIG_SMP
404         .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath,
405         .lock.queued_spin_unlock        =
406                                 PV_CALLEE_SAVE(__native_queued_spin_unlock),
407         .lock.wait                      = paravirt_nop,
408         .lock.kick                      = paravirt_nop,
409         .lock.vcpu_is_preempted         =
410                                 PV_CALLEE_SAVE(__native_vcpu_is_preempted),
411 #endif /* SMP */
412 #endif
413 };
414 
415 #ifdef CONFIG_PARAVIRT_XXL
416 NOKPROBE_SYMBOL(native_load_idt);
417 
418 void (*paravirt_iret)(void) = native_iret;
419 #endif
420 
421 EXPORT_SYMBOL(pv_ops);
422 EXPORT_SYMBOL_GPL(pv_info);
423 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp