~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/include/asm/paravirt.h

Version: ~ [ linux-5.4 ] ~ [ linux-5.3.13 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.86 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.156 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.203 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.202 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 #ifndef _ASM_X86_PARAVIRT_H
  2 #define _ASM_X86_PARAVIRT_H
  3 /* Various instructions on x86 need to be replaced for
  4  * para-virtualization: those hooks are defined here. */
  5 
  6 #ifdef CONFIG_PARAVIRT
  7 #include <asm/pgtable_types.h>
  8 #include <asm/asm.h>
  9 
 10 #include <asm/paravirt_types.h>
 11 
 12 #ifndef __ASSEMBLY__
 13 #include <linux/bug.h>
 14 #include <linux/types.h>
 15 #include <linux/cpumask.h>
 16 #include <asm/frame.h>
 17 
 18 static inline void load_sp0(struct tss_struct *tss,
 19                              struct thread_struct *thread)
 20 {
 21         PVOP_VCALL2(pv_cpu_ops.load_sp0, tss, thread);
 22 }
 23 
 24 /* The paravirtualized CPUID instruction. */
 25 static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
 26                            unsigned int *ecx, unsigned int *edx)
 27 {
 28         PVOP_VCALL4(pv_cpu_ops.cpuid, eax, ebx, ecx, edx);
 29 }
 30 
 31 /*
 32  * These special macros can be used to get or set a debugging register
 33  */
 34 static inline unsigned long paravirt_get_debugreg(int reg)
 35 {
 36         return PVOP_CALL1(unsigned long, pv_cpu_ops.get_debugreg, reg);
 37 }
 38 #define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
 39 static inline void set_debugreg(unsigned long val, int reg)
 40 {
 41         PVOP_VCALL2(pv_cpu_ops.set_debugreg, reg, val);
 42 }
 43 
 44 static inline void clts(void)
 45 {
 46         PVOP_VCALL0(pv_cpu_ops.clts);
 47 }
 48 
 49 static inline unsigned long read_cr0(void)
 50 {
 51         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr0);
 52 }
 53 
 54 static inline void write_cr0(unsigned long x)
 55 {
 56         PVOP_VCALL1(pv_cpu_ops.write_cr0, x);
 57 }
 58 
 59 static inline unsigned long read_cr2(void)
 60 {
 61         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr2);
 62 }
 63 
 64 static inline void write_cr2(unsigned long x)
 65 {
 66         PVOP_VCALL1(pv_mmu_ops.write_cr2, x);
 67 }
 68 
 69 static inline unsigned long read_cr3(void)
 70 {
 71         return PVOP_CALL0(unsigned long, pv_mmu_ops.read_cr3);
 72 }
 73 
 74 static inline void write_cr3(unsigned long x)
 75 {
 76         PVOP_VCALL1(pv_mmu_ops.write_cr3, x);
 77 }
 78 
 79 static inline unsigned long __read_cr4(void)
 80 {
 81         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4);
 82 }
 83 static inline unsigned long __read_cr4_safe(void)
 84 {
 85         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr4_safe);
 86 }
 87 
 88 static inline void __write_cr4(unsigned long x)
 89 {
 90         PVOP_VCALL1(pv_cpu_ops.write_cr4, x);
 91 }
 92 
 93 #ifdef CONFIG_X86_64
 94 static inline unsigned long read_cr8(void)
 95 {
 96         return PVOP_CALL0(unsigned long, pv_cpu_ops.read_cr8);
 97 }
 98 
 99 static inline void write_cr8(unsigned long x)
100 {
101         PVOP_VCALL1(pv_cpu_ops.write_cr8, x);
102 }
103 #endif
104 
105 static inline void arch_safe_halt(void)
106 {
107         PVOP_VCALL0(pv_irq_ops.safe_halt);
108 }
109 
110 static inline void halt(void)
111 {
112         PVOP_VCALL0(pv_irq_ops.halt);
113 }
114 
115 static inline void wbinvd(void)
116 {
117         PVOP_VCALL0(pv_cpu_ops.wbinvd);
118 }
119 
120 #define get_kernel_rpl()  (pv_info.kernel_rpl)
121 
122 static inline u64 paravirt_read_msr(unsigned msr)
123 {
124         return PVOP_CALL1(u64, pv_cpu_ops.read_msr, msr);
125 }
126 
127 static inline void paravirt_write_msr(unsigned msr,
128                                       unsigned low, unsigned high)
129 {
130         return PVOP_VCALL3(pv_cpu_ops.write_msr, msr, low, high);
131 }
132 
133 static inline u64 paravirt_read_msr_safe(unsigned msr, int *err)
134 {
135         return PVOP_CALL2(u64, pv_cpu_ops.read_msr_safe, msr, err);
136 }
137 
138 static inline int paravirt_write_msr_safe(unsigned msr,
139                                           unsigned low, unsigned high)
140 {
141         return PVOP_CALL3(int, pv_cpu_ops.write_msr_safe, msr, low, high);
142 }
143 
144 #define rdmsr(msr, val1, val2)                  \
145 do {                                            \
146         u64 _l = paravirt_read_msr(msr);        \
147         val1 = (u32)_l;                         \
148         val2 = _l >> 32;                        \
149 } while (0)
150 
151 #define wrmsr(msr, val1, val2)                  \
152 do {                                            \
153         paravirt_write_msr(msr, val1, val2);    \
154 } while (0)
155 
156 #define rdmsrl(msr, val)                        \
157 do {                                            \
158         val = paravirt_read_msr(msr);           \
159 } while (0)
160 
161 static inline void wrmsrl(unsigned msr, u64 val)
162 {
163         wrmsr(msr, (u32)val, (u32)(val>>32));
164 }
165 
166 #define wrmsr_safe(msr, a, b)   paravirt_write_msr_safe(msr, a, b)
167 
168 /* rdmsr with exception handling */
169 #define rdmsr_safe(msr, a, b)                           \
170 ({                                                      \
171         int _err;                                       \
172         u64 _l = paravirt_read_msr_safe(msr, &_err);    \
173         (*a) = (u32)_l;                                 \
174         (*b) = _l >> 32;                                \
175         _err;                                           \
176 })
177 
178 static inline int rdmsrl_safe(unsigned msr, unsigned long long *p)
179 {
180         int err;
181 
182         *p = paravirt_read_msr_safe(msr, &err);
183         return err;
184 }
185 
186 static inline unsigned long long paravirt_sched_clock(void)
187 {
188         return PVOP_CALL0(unsigned long long, pv_time_ops.sched_clock);
189 }
190 
191 struct static_key;
192 extern struct static_key paravirt_steal_enabled;
193 extern struct static_key paravirt_steal_rq_enabled;
194 
195 static inline u64 paravirt_steal_clock(int cpu)
196 {
197         return PVOP_CALL1(u64, pv_time_ops.steal_clock, cpu);
198 }
199 
200 static inline unsigned long long paravirt_read_pmc(int counter)
201 {
202         return PVOP_CALL1(u64, pv_cpu_ops.read_pmc, counter);
203 }
204 
205 #define rdpmc(counter, low, high)               \
206 do {                                            \
207         u64 _l = paravirt_read_pmc(counter);    \
208         low = (u32)_l;                          \
209         high = _l >> 32;                        \
210 } while (0)
211 
212 #define rdpmcl(counter, val) ((val) = paravirt_read_pmc(counter))
213 
214 static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
215 {
216         PVOP_VCALL2(pv_cpu_ops.alloc_ldt, ldt, entries);
217 }
218 
219 static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
220 {
221         PVOP_VCALL2(pv_cpu_ops.free_ldt, ldt, entries);
222 }
223 
224 static inline void load_TR_desc(void)
225 {
226         PVOP_VCALL0(pv_cpu_ops.load_tr_desc);
227 }
228 static inline void load_gdt(const struct desc_ptr *dtr)
229 {
230         PVOP_VCALL1(pv_cpu_ops.load_gdt, dtr);
231 }
232 static inline void load_idt(const struct desc_ptr *dtr)
233 {
234         PVOP_VCALL1(pv_cpu_ops.load_idt, dtr);
235 }
236 static inline void set_ldt(const void *addr, unsigned entries)
237 {
238         PVOP_VCALL2(pv_cpu_ops.set_ldt, addr, entries);
239 }
240 static inline void store_idt(struct desc_ptr *dtr)
241 {
242         PVOP_VCALL1(pv_cpu_ops.store_idt, dtr);
243 }
244 static inline unsigned long paravirt_store_tr(void)
245 {
246         return PVOP_CALL0(unsigned long, pv_cpu_ops.store_tr);
247 }
248 #define store_tr(tr)    ((tr) = paravirt_store_tr())
249 static inline void load_TLS(struct thread_struct *t, unsigned cpu)
250 {
251         PVOP_VCALL2(pv_cpu_ops.load_tls, t, cpu);
252 }
253 
254 #ifdef CONFIG_X86_64
255 static inline void load_gs_index(unsigned int gs)
256 {
257         PVOP_VCALL1(pv_cpu_ops.load_gs_index, gs);
258 }
259 #endif
260 
261 static inline void write_ldt_entry(struct desc_struct *dt, int entry,
262                                    const void *desc)
263 {
264         PVOP_VCALL3(pv_cpu_ops.write_ldt_entry, dt, entry, desc);
265 }
266 
267 static inline void write_gdt_entry(struct desc_struct *dt, int entry,
268                                    void *desc, int type)
269 {
270         PVOP_VCALL4(pv_cpu_ops.write_gdt_entry, dt, entry, desc, type);
271 }
272 
273 static inline void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g)
274 {
275         PVOP_VCALL3(pv_cpu_ops.write_idt_entry, dt, entry, g);
276 }
277 static inline void set_iopl_mask(unsigned mask)
278 {
279         PVOP_VCALL1(pv_cpu_ops.set_iopl_mask, mask);
280 }
281 
282 /* The paravirtualized I/O functions */
283 static inline void slow_down_io(void)
284 {
285         pv_cpu_ops.io_delay();
286 #ifdef REALLY_SLOW_IO
287         pv_cpu_ops.io_delay();
288         pv_cpu_ops.io_delay();
289         pv_cpu_ops.io_delay();
290 #endif
291 }
292 
293 static inline void paravirt_activate_mm(struct mm_struct *prev,
294                                         struct mm_struct *next)
295 {
296         PVOP_VCALL2(pv_mmu_ops.activate_mm, prev, next);
297 }
298 
299 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
300                                           struct mm_struct *mm)
301 {
302         PVOP_VCALL2(pv_mmu_ops.dup_mmap, oldmm, mm);
303 }
304 
305 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
306 {
307         PVOP_VCALL1(pv_mmu_ops.exit_mmap, mm);
308 }
309 
310 static inline void __flush_tlb(void)
311 {
312         PVOP_VCALL0(pv_mmu_ops.flush_tlb_user);
313 }
314 static inline void __flush_tlb_global(void)
315 {
316         PVOP_VCALL0(pv_mmu_ops.flush_tlb_kernel);
317 }
318 static inline void __flush_tlb_single(unsigned long addr)
319 {
320         PVOP_VCALL1(pv_mmu_ops.flush_tlb_single, addr);
321 }
322 
323 static inline void flush_tlb_others(const struct cpumask *cpumask,
324                                     struct mm_struct *mm,
325                                     unsigned long start,
326                                     unsigned long end)
327 {
328         PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
329 }
330 
331 static inline int paravirt_pgd_alloc(struct mm_struct *mm)
332 {
333         return PVOP_CALL1(int, pv_mmu_ops.pgd_alloc, mm);
334 }
335 
336 static inline void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd)
337 {
338         PVOP_VCALL2(pv_mmu_ops.pgd_free, mm, pgd);
339 }
340 
341 static inline void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn)
342 {
343         PVOP_VCALL2(pv_mmu_ops.alloc_pte, mm, pfn);
344 }
345 static inline void paravirt_release_pte(unsigned long pfn)
346 {
347         PVOP_VCALL1(pv_mmu_ops.release_pte, pfn);
348 }
349 
350 static inline void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn)
351 {
352         PVOP_VCALL2(pv_mmu_ops.alloc_pmd, mm, pfn);
353 }
354 
355 static inline void paravirt_release_pmd(unsigned long pfn)
356 {
357         PVOP_VCALL1(pv_mmu_ops.release_pmd, pfn);
358 }
359 
360 static inline void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn)
361 {
362         PVOP_VCALL2(pv_mmu_ops.alloc_pud, mm, pfn);
363 }
364 static inline void paravirt_release_pud(unsigned long pfn)
365 {
366         PVOP_VCALL1(pv_mmu_ops.release_pud, pfn);
367 }
368 
369 static inline void pte_update(struct mm_struct *mm, unsigned long addr,
370                               pte_t *ptep)
371 {
372         PVOP_VCALL3(pv_mmu_ops.pte_update, mm, addr, ptep);
373 }
374 
375 static inline pte_t __pte(pteval_t val)
376 {
377         pteval_t ret;
378 
379         if (sizeof(pteval_t) > sizeof(long))
380                 ret = PVOP_CALLEE2(pteval_t,
381                                    pv_mmu_ops.make_pte,
382                                    val, (u64)val >> 32);
383         else
384                 ret = PVOP_CALLEE1(pteval_t,
385                                    pv_mmu_ops.make_pte,
386                                    val);
387 
388         return (pte_t) { .pte = ret };
389 }
390 
391 static inline pteval_t pte_val(pte_t pte)
392 {
393         pteval_t ret;
394 
395         if (sizeof(pteval_t) > sizeof(long))
396                 ret = PVOP_CALLEE2(pteval_t, pv_mmu_ops.pte_val,
397                                    pte.pte, (u64)pte.pte >> 32);
398         else
399                 ret = PVOP_CALLEE1(pteval_t, pv_mmu_ops.pte_val,
400                                    pte.pte);
401 
402         return ret;
403 }
404 
405 static inline pgd_t __pgd(pgdval_t val)
406 {
407         pgdval_t ret;
408 
409         if (sizeof(pgdval_t) > sizeof(long))
410                 ret = PVOP_CALLEE2(pgdval_t, pv_mmu_ops.make_pgd,
411                                    val, (u64)val >> 32);
412         else
413                 ret = PVOP_CALLEE1(pgdval_t, pv_mmu_ops.make_pgd,
414                                    val);
415 
416         return (pgd_t) { ret };
417 }
418 
419 static inline pgdval_t pgd_val(pgd_t pgd)
420 {
421         pgdval_t ret;
422 
423         if (sizeof(pgdval_t) > sizeof(long))
424                 ret =  PVOP_CALLEE2(pgdval_t, pv_mmu_ops.pgd_val,
425                                     pgd.pgd, (u64)pgd.pgd >> 32);
426         else
427                 ret =  PVOP_CALLEE1(pgdval_t, pv_mmu_ops.pgd_val,
428                                     pgd.pgd);
429 
430         return ret;
431 }
432 
433 #define  __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
434 static inline pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr,
435                                            pte_t *ptep)
436 {
437         pteval_t ret;
438 
439         ret = PVOP_CALL3(pteval_t, pv_mmu_ops.ptep_modify_prot_start,
440                          mm, addr, ptep);
441 
442         return (pte_t) { .pte = ret };
443 }
444 
445 static inline void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
446                                            pte_t *ptep, pte_t pte)
447 {
448         if (sizeof(pteval_t) > sizeof(long))
449                 /* 5 arg words */
450                 pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte);
451         else
452                 PVOP_VCALL4(pv_mmu_ops.ptep_modify_prot_commit,
453                             mm, addr, ptep, pte.pte);
454 }
455 
456 static inline void set_pte(pte_t *ptep, pte_t pte)
457 {
458         if (sizeof(pteval_t) > sizeof(long))
459                 PVOP_VCALL3(pv_mmu_ops.set_pte, ptep,
460                             pte.pte, (u64)pte.pte >> 32);
461         else
462                 PVOP_VCALL2(pv_mmu_ops.set_pte, ptep,
463                             pte.pte);
464 }
465 
466 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
467                               pte_t *ptep, pte_t pte)
468 {
469         if (sizeof(pteval_t) > sizeof(long))
470                 /* 5 arg words */
471                 pv_mmu_ops.set_pte_at(mm, addr, ptep, pte);
472         else
473                 PVOP_VCALL4(pv_mmu_ops.set_pte_at, mm, addr, ptep, pte.pte);
474 }
475 
476 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
477                               pmd_t *pmdp, pmd_t pmd)
478 {
479         if (sizeof(pmdval_t) > sizeof(long))
480                 /* 5 arg words */
481                 pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd);
482         else
483                 PVOP_VCALL4(pv_mmu_ops.set_pmd_at, mm, addr, pmdp,
484                             native_pmd_val(pmd));
485 }
486 
487 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
488 {
489         pmdval_t val = native_pmd_val(pmd);
490 
491         if (sizeof(pmdval_t) > sizeof(long))
492                 PVOP_VCALL3(pv_mmu_ops.set_pmd, pmdp, val, (u64)val >> 32);
493         else
494                 PVOP_VCALL2(pv_mmu_ops.set_pmd, pmdp, val);
495 }
496 
497 #if CONFIG_PGTABLE_LEVELS >= 3
498 static inline pmd_t __pmd(pmdval_t val)
499 {
500         pmdval_t ret;
501 
502         if (sizeof(pmdval_t) > sizeof(long))
503                 ret = PVOP_CALLEE2(pmdval_t, pv_mmu_ops.make_pmd,
504                                    val, (u64)val >> 32);
505         else
506                 ret = PVOP_CALLEE1(pmdval_t, pv_mmu_ops.make_pmd,
507                                    val);
508 
509         return (pmd_t) { ret };
510 }
511 
512 static inline pmdval_t pmd_val(pmd_t pmd)
513 {
514         pmdval_t ret;
515 
516         if (sizeof(pmdval_t) > sizeof(long))
517                 ret =  PVOP_CALLEE2(pmdval_t, pv_mmu_ops.pmd_val,
518                                     pmd.pmd, (u64)pmd.pmd >> 32);
519         else
520                 ret =  PVOP_CALLEE1(pmdval_t, pv_mmu_ops.pmd_val,
521                                     pmd.pmd);
522 
523         return ret;
524 }
525 
526 static inline void set_pud(pud_t *pudp, pud_t pud)
527 {
528         pudval_t val = native_pud_val(pud);
529 
530         if (sizeof(pudval_t) > sizeof(long))
531                 PVOP_VCALL3(pv_mmu_ops.set_pud, pudp,
532                             val, (u64)val >> 32);
533         else
534                 PVOP_VCALL2(pv_mmu_ops.set_pud, pudp,
535                             val);
536 }
537 #if CONFIG_PGTABLE_LEVELS == 4
538 static inline pud_t __pud(pudval_t val)
539 {
540         pudval_t ret;
541 
542         if (sizeof(pudval_t) > sizeof(long))
543                 ret = PVOP_CALLEE2(pudval_t, pv_mmu_ops.make_pud,
544                                    val, (u64)val >> 32);
545         else
546                 ret = PVOP_CALLEE1(pudval_t, pv_mmu_ops.make_pud,
547                                    val);
548 
549         return (pud_t) { ret };
550 }
551 
552 static inline pudval_t pud_val(pud_t pud)
553 {
554         pudval_t ret;
555 
556         if (sizeof(pudval_t) > sizeof(long))
557                 ret =  PVOP_CALLEE2(pudval_t, pv_mmu_ops.pud_val,
558                                     pud.pud, (u64)pud.pud >> 32);
559         else
560                 ret =  PVOP_CALLEE1(pudval_t, pv_mmu_ops.pud_val,
561                                     pud.pud);
562 
563         return ret;
564 }
565 
566 static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
567 {
568         pgdval_t val = native_pgd_val(pgd);
569 
570         if (sizeof(pgdval_t) > sizeof(long))
571                 PVOP_VCALL3(pv_mmu_ops.set_pgd, pgdp,
572                             val, (u64)val >> 32);
573         else
574                 PVOP_VCALL2(pv_mmu_ops.set_pgd, pgdp,
575                             val);
576 }
577 
578 static inline void pgd_clear(pgd_t *pgdp)
579 {
580         set_pgd(pgdp, __pgd(0));
581 }
582 
583 static inline void pud_clear(pud_t *pudp)
584 {
585         set_pud(pudp, __pud(0));
586 }
587 
588 #endif  /* CONFIG_PGTABLE_LEVELS == 4 */
589 
590 #endif  /* CONFIG_PGTABLE_LEVELS >= 3 */
591 
592 #ifdef CONFIG_X86_PAE
593 /* Special-case pte-setting operations for PAE, which can't update a
594    64-bit pte atomically */
595 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
596 {
597         PVOP_VCALL3(pv_mmu_ops.set_pte_atomic, ptep,
598                     pte.pte, pte.pte >> 32);
599 }
600 
601 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
602                              pte_t *ptep)
603 {
604         PVOP_VCALL3(pv_mmu_ops.pte_clear, mm, addr, ptep);
605 }
606 
607 static inline void pmd_clear(pmd_t *pmdp)
608 {
609         PVOP_VCALL1(pv_mmu_ops.pmd_clear, pmdp);
610 }
611 #else  /* !CONFIG_X86_PAE */
612 static inline void set_pte_atomic(pte_t *ptep, pte_t pte)
613 {
614         set_pte(ptep, pte);
615 }
616 
617 static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
618                              pte_t *ptep)
619 {
620         set_pte_at(mm, addr, ptep, __pte(0));
621 }
622 
623 static inline void pmd_clear(pmd_t *pmdp)
624 {
625         set_pmd(pmdp, __pmd(0));
626 }
627 #endif  /* CONFIG_X86_PAE */
628 
629 #define  __HAVE_ARCH_START_CONTEXT_SWITCH
630 static inline void arch_start_context_switch(struct task_struct *prev)
631 {
632         PVOP_VCALL1(pv_cpu_ops.start_context_switch, prev);
633 }
634 
635 static inline void arch_end_context_switch(struct task_struct *next)
636 {
637         PVOP_VCALL1(pv_cpu_ops.end_context_switch, next);
638 }
639 
640 #define  __HAVE_ARCH_ENTER_LAZY_MMU_MODE
641 static inline void arch_enter_lazy_mmu_mode(void)
642 {
643         PVOP_VCALL0(pv_mmu_ops.lazy_mode.enter);
644 }
645 
646 static inline void arch_leave_lazy_mmu_mode(void)
647 {
648         PVOP_VCALL0(pv_mmu_ops.lazy_mode.leave);
649 }
650 
651 static inline void arch_flush_lazy_mmu_mode(void)
652 {
653         PVOP_VCALL0(pv_mmu_ops.lazy_mode.flush);
654 }
655 
656 static inline void __set_fixmap(unsigned /* enum fixed_addresses */ idx,
657                                 phys_addr_t phys, pgprot_t flags)
658 {
659         pv_mmu_ops.set_fixmap(idx, phys, flags);
660 }
661 
662 #if defined(CONFIG_SMP) && defined(CONFIG_PARAVIRT_SPINLOCKS)
663 
664 #ifdef CONFIG_QUEUED_SPINLOCKS
665 
666 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
667                                                         u32 val)
668 {
669         PVOP_VCALL2(pv_lock_ops.queued_spin_lock_slowpath, lock, val);
670 }
671 
672 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
673 {
674         PVOP_VCALLEE1(pv_lock_ops.queued_spin_unlock, lock);
675 }
676 
677 static __always_inline void pv_wait(u8 *ptr, u8 val)
678 {
679         PVOP_VCALL2(pv_lock_ops.wait, ptr, val);
680 }
681 
682 static __always_inline void pv_kick(int cpu)
683 {
684         PVOP_VCALL1(pv_lock_ops.kick, cpu);
685 }
686 
687 #else /* !CONFIG_QUEUED_SPINLOCKS */
688 
689 static __always_inline void __ticket_lock_spinning(struct arch_spinlock *lock,
690                                                         __ticket_t ticket)
691 {
692         PVOP_VCALLEE2(pv_lock_ops.lock_spinning, lock, ticket);
693 }
694 
695 static __always_inline void __ticket_unlock_kick(struct arch_spinlock *lock,
696                                                         __ticket_t ticket)
697 {
698         PVOP_VCALL2(pv_lock_ops.unlock_kick, lock, ticket);
699 }
700 
701 #endif /* CONFIG_QUEUED_SPINLOCKS */
702 
703 #endif /* SMP && PARAVIRT_SPINLOCKS */
704 
705 #ifdef CONFIG_X86_32
706 #define PV_SAVE_REGS "pushl %ecx; pushl %edx;"
707 #define PV_RESTORE_REGS "popl %edx; popl %ecx;"
708 
709 /* save and restore all caller-save registers, except return value */
710 #define PV_SAVE_ALL_CALLER_REGS         "pushl %ecx;"
711 #define PV_RESTORE_ALL_CALLER_REGS      "popl  %ecx;"
712 
713 #define PV_FLAGS_ARG ""
714 #define PV_EXTRA_CLOBBERS
715 #define PV_VEXTRA_CLOBBERS
716 #else
717 /* save and restore all caller-save registers, except return value */
718 #define PV_SAVE_ALL_CALLER_REGS                                         \
719         "push %rcx;"                                                    \
720         "push %rdx;"                                                    \
721         "push %rsi;"                                                    \
722         "push %rdi;"                                                    \
723         "push %r8;"                                                     \
724         "push %r9;"                                                     \
725         "push %r10;"                                                    \
726         "push %r11;"
727 #define PV_RESTORE_ALL_CALLER_REGS                                      \
728         "pop %r11;"                                                     \
729         "pop %r10;"                                                     \
730         "pop %r9;"                                                      \
731         "pop %r8;"                                                      \
732         "pop %rdi;"                                                     \
733         "pop %rsi;"                                                     \
734         "pop %rdx;"                                                     \
735         "pop %rcx;"
736 
737 /* We save some registers, but all of them, that's too much. We clobber all
738  * caller saved registers but the argument parameter */
739 #define PV_SAVE_REGS "pushq %%rdi;"
740 #define PV_RESTORE_REGS "popq %%rdi;"
741 #define PV_EXTRA_CLOBBERS EXTRA_CLOBBERS, "rcx" , "rdx", "rsi"
742 #define PV_VEXTRA_CLOBBERS EXTRA_CLOBBERS, "rdi", "rcx" , "rdx", "rsi"
743 #define PV_FLAGS_ARG "D"
744 #endif
745 
746 /*
747  * Generate a thunk around a function which saves all caller-save
748  * registers except for the return value.  This allows C functions to
749  * be called from assembler code where fewer than normal registers are
750  * available.  It may also help code generation around calls from C
751  * code if the common case doesn't use many registers.
752  *
753  * When a callee is wrapped in a thunk, the caller can assume that all
754  * arg regs and all scratch registers are preserved across the
755  * call. The return value in rax/eax will not be saved, even for void
756  * functions.
757  */
758 #define PV_THUNK_NAME(func) "__raw_callee_save_" #func
759 #define PV_CALLEE_SAVE_REGS_THUNK(func)                                 \
760         extern typeof(func) __raw_callee_save_##func;                   \
761                                                                         \
762         asm(".pushsection .text;"                                       \
763             ".globl " PV_THUNK_NAME(func) ";"                           \
764             ".type " PV_THUNK_NAME(func) ", @function;"                 \
765             PV_THUNK_NAME(func) ":"                                     \
766             FRAME_BEGIN                                                 \
767             PV_SAVE_ALL_CALLER_REGS                                     \
768             "call " #func ";"                                           \
769             PV_RESTORE_ALL_CALLER_REGS                                  \
770             FRAME_END                                                   \
771             "ret;"                                                      \
772             ".popsection")
773 
774 /* Get a reference to a callee-save function */
775 #define PV_CALLEE_SAVE(func)                                            \
776         ((struct paravirt_callee_save) { __raw_callee_save_##func })
777 
778 /* Promise that "func" already uses the right calling convention */
779 #define __PV_IS_CALLEE_SAVE(func)                       \
780         ((struct paravirt_callee_save) { func })
781 
782 static inline notrace unsigned long arch_local_save_flags(void)
783 {
784         return PVOP_CALLEE0(unsigned long, pv_irq_ops.save_fl);
785 }
786 
787 static inline notrace void arch_local_irq_restore(unsigned long f)
788 {
789         PVOP_VCALLEE1(pv_irq_ops.restore_fl, f);
790 }
791 
792 static inline notrace void arch_local_irq_disable(void)
793 {
794         PVOP_VCALLEE0(pv_irq_ops.irq_disable);
795 }
796 
797 static inline notrace void arch_local_irq_enable(void)
798 {
799         PVOP_VCALLEE0(pv_irq_ops.irq_enable);
800 }
801 
802 static inline notrace unsigned long arch_local_irq_save(void)
803 {
804         unsigned long f;
805 
806         f = arch_local_save_flags();
807         arch_local_irq_disable();
808         return f;
809 }
810 
811 
812 /* Make sure as little as possible of this mess escapes. */
813 #undef PARAVIRT_CALL
814 #undef __PVOP_CALL
815 #undef __PVOP_VCALL
816 #undef PVOP_VCALL0
817 #undef PVOP_CALL0
818 #undef PVOP_VCALL1
819 #undef PVOP_CALL1
820 #undef PVOP_VCALL2
821 #undef PVOP_CALL2
822 #undef PVOP_VCALL3
823 #undef PVOP_CALL3
824 #undef PVOP_VCALL4
825 #undef PVOP_CALL4
826 
827 extern void default_banner(void);
828 
829 #else  /* __ASSEMBLY__ */
830 
831 #define _PVSITE(ptype, clobbers, ops, word, algn)       \
832 771:;                                           \
833         ops;                                    \
834 772:;                                           \
835         .pushsection .parainstructions,"a";     \
836          .align algn;                           \
837          word 771b;                             \
838          .byte ptype;                           \
839          .byte 772b-771b;                       \
840          .short clobbers;                       \
841         .popsection
842 
843 
844 #define COND_PUSH(set, mask, reg)                       \
845         .if ((~(set)) & mask); push %reg; .endif
846 #define COND_POP(set, mask, reg)                        \
847         .if ((~(set)) & mask); pop %reg; .endif
848 
849 #ifdef CONFIG_X86_64
850 
851 #define PV_SAVE_REGS(set)                       \
852         COND_PUSH(set, CLBR_RAX, rax);          \
853         COND_PUSH(set, CLBR_RCX, rcx);          \
854         COND_PUSH(set, CLBR_RDX, rdx);          \
855         COND_PUSH(set, CLBR_RSI, rsi);          \
856         COND_PUSH(set, CLBR_RDI, rdi);          \
857         COND_PUSH(set, CLBR_R8, r8);            \
858         COND_PUSH(set, CLBR_R9, r9);            \
859         COND_PUSH(set, CLBR_R10, r10);          \
860         COND_PUSH(set, CLBR_R11, r11)
861 #define PV_RESTORE_REGS(set)                    \
862         COND_POP(set, CLBR_R11, r11);           \
863         COND_POP(set, CLBR_R10, r10);           \
864         COND_POP(set, CLBR_R9, r9);             \
865         COND_POP(set, CLBR_R8, r8);             \
866         COND_POP(set, CLBR_RDI, rdi);           \
867         COND_POP(set, CLBR_RSI, rsi);           \
868         COND_POP(set, CLBR_RDX, rdx);           \
869         COND_POP(set, CLBR_RCX, rcx);           \
870         COND_POP(set, CLBR_RAX, rax)
871 
872 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 8)
873 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .quad, 8)
874 #define PARA_INDIRECT(addr)     *addr(%rip)
875 #else
876 #define PV_SAVE_REGS(set)                       \
877         COND_PUSH(set, CLBR_EAX, eax);          \
878         COND_PUSH(set, CLBR_EDI, edi);          \
879         COND_PUSH(set, CLBR_ECX, ecx);          \
880         COND_PUSH(set, CLBR_EDX, edx)
881 #define PV_RESTORE_REGS(set)                    \
882         COND_POP(set, CLBR_EDX, edx);           \
883         COND_POP(set, CLBR_ECX, ecx);           \
884         COND_POP(set, CLBR_EDI, edi);           \
885         COND_POP(set, CLBR_EAX, eax)
886 
887 #define PARA_PATCH(struct, off)        ((PARAVIRT_PATCH_##struct + (off)) / 4)
888 #define PARA_SITE(ptype, clobbers, ops) _PVSITE(ptype, clobbers, ops, .long, 4)
889 #define PARA_INDIRECT(addr)     *%cs:addr
890 #endif
891 
892 #define INTERRUPT_RETURN                                                \
893         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE,       \
894                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_iret))
895 
896 #define DISABLE_INTERRUPTS(clobbers)                                    \
897         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \
898                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
899                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_disable);    \
900                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
901 
902 #define ENABLE_INTERRUPTS(clobbers)                                     \
903         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_enable), clobbers,  \
904                   PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE);            \
905                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_irq_enable);     \
906                   PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
907 
908 #ifdef CONFIG_X86_32
909 #define GET_CR0_INTO_EAX                                \
910         push %ecx; push %edx;                           \
911         call PARA_INDIRECT(pv_cpu_ops+PV_CPU_read_cr0); \
912         pop %edx; pop %ecx
913 #else   /* !CONFIG_X86_32 */
914 
915 /*
916  * If swapgs is used while the userspace stack is still current,
917  * there's no way to call a pvop.  The PV replacement *must* be
918  * inlined, or the swapgs instruction must be trapped and emulated.
919  */
920 #define SWAPGS_UNSAFE_STACK                                             \
921         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
922                   swapgs)
923 
924 /*
925  * Note: swapgs is very special, and in practise is either going to be
926  * implemented with a single "swapgs" instruction or something very
927  * special.  Either way, we don't need to save any registers for
928  * it.
929  */
930 #define SWAPGS                                                          \
931         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_swapgs), CLBR_NONE,     \
932                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
933                  )
934 
935 #define GET_CR2_INTO_RAX                                \
936         call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
937 
938 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
939         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
940                   CLBR_NONE,                                            \
941                   call PARA_INDIRECT(pv_irq_ops+PV_IRQ_adjust_exception_frame))
942 
943 #define USERGS_SYSRET64                                                 \
944         PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_usergs_sysret64),       \
945                   CLBR_NONE,                                            \
946                   jmp PARA_INDIRECT(pv_cpu_ops+PV_CPU_usergs_sysret64))
947 #endif  /* CONFIG_X86_32 */
948 
949 #endif /* __ASSEMBLY__ */
950 #else  /* CONFIG_PARAVIRT */
951 # define default_banner x86_init_noop
952 #ifndef __ASSEMBLY__
953 static inline void paravirt_arch_dup_mmap(struct mm_struct *oldmm,
954                                           struct mm_struct *mm)
955 {
956 }
957 
958 static inline void paravirt_arch_exit_mmap(struct mm_struct *mm)
959 {
960 }
961 #endif /* __ASSEMBLY__ */
962 #endif /* !CONFIG_PARAVIRT */
963 #endif /* _ASM_X86_PARAVIRT_H */
964 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp