~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/mm/slb.c

Version: ~ [ linux-5.12-rc1 ] ~ [ linux-5.11.2 ] ~ [ linux-5.10.19 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.101 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.177 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.222 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.258 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.258 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * PowerPC64 SLB support.
  3  *
  4  * Copyright (C) 2004 David Gibson <dwg@au.ibm.com>, IBM
  5  * Based on earlier code written by:
  6  * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
  7  *    Copyright (c) 2001 Dave Engebretsen
  8  * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
  9  *
 10  *
 11  *      This program is free software; you can redistribute it and/or
 12  *      modify it under the terms of the GNU General Public License
 13  *      as published by the Free Software Foundation; either version
 14  *      2 of the License, or (at your option) any later version.
 15  */
 16 
 17 #include <asm/pgtable.h>
 18 #include <asm/mmu.h>
 19 #include <asm/mmu_context.h>
 20 #include <asm/paca.h>
 21 #include <asm/cputable.h>
 22 #include <asm/cacheflush.h>
 23 #include <asm/smp.h>
 24 #include <linux/compiler.h>
 25 #include <asm/udbg.h>
 26 #include <asm/code-patching.h>
 27 
 28 
 29 extern void slb_allocate_realmode(unsigned long ea);
 30 extern void slb_allocate_user(unsigned long ea);
 31 
 32 static void slb_allocate(unsigned long ea)
 33 {
 34         /* Currently, we do real mode for all SLBs including user, but
 35          * that will change if we bring back dynamic VSIDs
 36          */
 37         slb_allocate_realmode(ea);
 38 }
 39 
 40 #define slb_esid_mask(ssize)    \
 41         (((ssize) == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T)
 42 
 43 static inline unsigned long mk_esid_data(unsigned long ea, int ssize,
 44                                          unsigned long slot)
 45 {
 46         return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | slot;
 47 }
 48 
 49 static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
 50                                          unsigned long flags)
 51 {
 52         return (get_kernel_vsid(ea, ssize) << slb_vsid_shift(ssize)) | flags |
 53                 ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT);
 54 }
 55 
 56 static inline void slb_shadow_update(unsigned long ea, int ssize,
 57                                      unsigned long flags,
 58                                      unsigned long entry)
 59 {
 60         /*
 61          * Clear the ESID first so the entry is not valid while we are
 62          * updating it.  No write barriers are needed here, provided
 63          * we only update the current CPU's SLB shadow buffer.
 64          */
 65         get_slb_shadow()->save_area[entry].esid = 0;
 66         get_slb_shadow()->save_area[entry].vsid =
 67                                 cpu_to_be64(mk_vsid_data(ea, ssize, flags));
 68         get_slb_shadow()->save_area[entry].esid =
 69                                 cpu_to_be64(mk_esid_data(ea, ssize, entry));
 70 }
 71 
 72 static inline void slb_shadow_clear(unsigned long entry)
 73 {
 74         get_slb_shadow()->save_area[entry].esid = 0;
 75 }
 76 
 77 static inline void create_shadowed_slbe(unsigned long ea, int ssize,
 78                                         unsigned long flags,
 79                                         unsigned long entry)
 80 {
 81         /*
 82          * Updating the shadow buffer before writing the SLB ensures
 83          * we don't get a stale entry here if we get preempted by PHYP
 84          * between these two statements.
 85          */
 86         slb_shadow_update(ea, ssize, flags, entry);
 87 
 88         asm volatile("slbmte  %0,%1" :
 89                      : "r" (mk_vsid_data(ea, ssize, flags)),
 90                        "r" (mk_esid_data(ea, ssize, entry))
 91                      : "memory" );
 92 }
 93 
 94 static void __slb_flush_and_rebolt(void)
 95 {
 96         /* If you change this make sure you change SLB_NUM_BOLTED
 97          * and PR KVM appropriately too. */
 98         unsigned long linear_llp, vmalloc_llp, lflags, vflags;
 99         unsigned long ksp_esid_data, ksp_vsid_data;
100 
101         linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
102         vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp;
103         lflags = SLB_VSID_KERNEL | linear_llp;
104         vflags = SLB_VSID_KERNEL | vmalloc_llp;
105 
106         ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, 2);
107         if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) {
108                 ksp_esid_data &= ~SLB_ESID_V;
109                 ksp_vsid_data = 0;
110                 slb_shadow_clear(2);
111         } else {
112                 /* Update stack entry; others don't change */
113                 slb_shadow_update(get_paca()->kstack, mmu_kernel_ssize, lflags, 2);
114                 ksp_vsid_data =
115                         be64_to_cpu(get_slb_shadow()->save_area[2].vsid);
116         }
117 
118         /* We need to do this all in asm, so we're sure we don't touch
119          * the stack between the slbia and rebolting it. */
120         asm volatile("isync\n"
121                      "slbia\n"
122                      /* Slot 1 - first VMALLOC segment */
123                      "slbmte    %0,%1\n"
124                      /* Slot 2 - kernel stack */
125                      "slbmte    %2,%3\n"
126                      "isync"
127                      :: "r"(mk_vsid_data(VMALLOC_START, mmu_kernel_ssize, vflags)),
128                         "r"(mk_esid_data(VMALLOC_START, mmu_kernel_ssize, 1)),
129                         "r"(ksp_vsid_data),
130                         "r"(ksp_esid_data)
131                      : "memory");
132 }
133 
134 void slb_flush_and_rebolt(void)
135 {
136 
137         WARN_ON(!irqs_disabled());
138 
139         /*
140          * We can't take a PMU exception in the following code, so hard
141          * disable interrupts.
142          */
143         hard_irq_disable();
144 
145         __slb_flush_and_rebolt();
146         get_paca()->slb_cache_ptr = 0;
147 }
148 
149 void slb_vmalloc_update(void)
150 {
151         unsigned long vflags;
152 
153         vflags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmalloc_psize].sllp;
154         slb_shadow_update(VMALLOC_START, mmu_kernel_ssize, vflags, 1);
155         slb_flush_and_rebolt();
156 }
157 
158 /* Helper function to compare esids.  There are four cases to handle.
159  * 1. The system is not 1T segment size capable.  Use the GET_ESID compare.
160  * 2. The system is 1T capable, both addresses are < 1T, use the GET_ESID compare.
161  * 3. The system is 1T capable, only one of the two addresses is > 1T.  This is not a match.
162  * 4. The system is 1T capable, both addresses are > 1T, use the GET_ESID_1T macro to compare.
163  */
164 static inline int esids_match(unsigned long addr1, unsigned long addr2)
165 {
166         int esid_1t_count;
167 
168         /* System is not 1T segment size capable. */
169         if (!mmu_has_feature(MMU_FTR_1T_SEGMENT))
170                 return (GET_ESID(addr1) == GET_ESID(addr2));
171 
172         esid_1t_count = (((addr1 >> SID_SHIFT_1T) != 0) +
173                                 ((addr2 >> SID_SHIFT_1T) != 0));
174 
175         /* both addresses are < 1T */
176         if (esid_1t_count == 0)
177                 return (GET_ESID(addr1) == GET_ESID(addr2));
178 
179         /* One address < 1T, the other > 1T.  Not a match */
180         if (esid_1t_count == 1)
181                 return 0;
182 
183         /* Both addresses are > 1T. */
184         return (GET_ESID_1T(addr1) == GET_ESID_1T(addr2));
185 }
186 
187 /* Flush all user entries from the segment table of the current processor. */
188 void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
189 {
190         unsigned long offset;
191         unsigned long slbie_data = 0;
192         unsigned long pc = KSTK_EIP(tsk);
193         unsigned long stack = KSTK_ESP(tsk);
194         unsigned long exec_base;
195 
196         /*
197          * We need interrupts hard-disabled here, not just soft-disabled,
198          * so that a PMU interrupt can't occur, which might try to access
199          * user memory (to get a stack trace) and possible cause an SLB miss
200          * which would update the slb_cache/slb_cache_ptr fields in the PACA.
201          */
202         hard_irq_disable();
203         offset = get_paca()->slb_cache_ptr;
204         if (!mmu_has_feature(MMU_FTR_NO_SLBIE_B) &&
205             offset <= SLB_CACHE_ENTRIES) {
206                 int i;
207                 asm volatile("isync" : : : "memory");
208                 for (i = 0; i < offset; i++) {
209                         slbie_data = (unsigned long)get_paca()->slb_cache[i]
210                                 << SID_SHIFT; /* EA */
211                         slbie_data |= user_segment_size(slbie_data)
212                                 << SLBIE_SSIZE_SHIFT;
213                         slbie_data |= SLBIE_C; /* C set for user addresses */
214                         asm volatile("slbie %0" : : "r" (slbie_data));
215                 }
216                 asm volatile("isync" : : : "memory");
217         } else {
218                 __slb_flush_and_rebolt();
219         }
220 
221         /* Workaround POWER5 < DD2.1 issue */
222         if (offset == 1 || offset > SLB_CACHE_ENTRIES)
223                 asm volatile("slbie %0" : : "r" (slbie_data));
224 
225         get_paca()->slb_cache_ptr = 0;
226         get_paca()->context = mm->context;
227 
228         /*
229          * preload some userspace segments into the SLB.
230          * Almost all 32 and 64bit PowerPC executables are linked at
231          * 0x10000000 so it makes sense to preload this segment.
232          */
233         exec_base = 0x10000000;
234 
235         if (is_kernel_addr(pc) || is_kernel_addr(stack) ||
236             is_kernel_addr(exec_base))
237                 return;
238 
239         slb_allocate(pc);
240 
241         if (!esids_match(pc, stack))
242                 slb_allocate(stack);
243 
244         if (!esids_match(pc, exec_base) &&
245             !esids_match(stack, exec_base))
246                 slb_allocate(exec_base);
247 }
248 
249 static inline void patch_slb_encoding(unsigned int *insn_addr,
250                                       unsigned int immed)
251 {
252         int insn = (*insn_addr & 0xffff0000) | immed;
253         patch_instruction(insn_addr, insn);
254 }
255 
256 extern u32 slb_compare_rr_to_size[];
257 extern u32 slb_miss_kernel_load_linear[];
258 extern u32 slb_miss_kernel_load_io[];
259 extern u32 slb_compare_rr_to_size[];
260 extern u32 slb_miss_kernel_load_vmemmap[];
261 
262 void slb_set_size(u16 size)
263 {
264         if (mmu_slb_size == size)
265                 return;
266 
267         mmu_slb_size = size;
268         patch_slb_encoding(slb_compare_rr_to_size, mmu_slb_size);
269 }
270 
271 void slb_initialize(void)
272 {
273         unsigned long linear_llp, vmalloc_llp, io_llp;
274         unsigned long lflags, vflags;
275         static int slb_encoding_inited;
276 #ifdef CONFIG_SPARSEMEM_VMEMMAP
277         unsigned long vmemmap_llp;
278 #endif
279 
280         /* Prepare our SLB miss handler based on our page size */
281         linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
282         io_llp = mmu_psize_defs[mmu_io_psize].sllp;
283         vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp;
284         get_paca()->vmalloc_sllp = SLB_VSID_KERNEL | vmalloc_llp;
285 #ifdef CONFIG_SPARSEMEM_VMEMMAP
286         vmemmap_llp = mmu_psize_defs[mmu_vmemmap_psize].sllp;
287 #endif
288         if (!slb_encoding_inited) {
289                 slb_encoding_inited = 1;
290                 patch_slb_encoding(slb_miss_kernel_load_linear,
291                                    SLB_VSID_KERNEL | linear_llp);
292                 patch_slb_encoding(slb_miss_kernel_load_io,
293                                    SLB_VSID_KERNEL | io_llp);
294                 patch_slb_encoding(slb_compare_rr_to_size,
295                                    mmu_slb_size);
296 
297                 pr_devel("SLB: linear  LLP = %04lx\n", linear_llp);
298                 pr_devel("SLB: io      LLP = %04lx\n", io_llp);
299 
300 #ifdef CONFIG_SPARSEMEM_VMEMMAP
301                 patch_slb_encoding(slb_miss_kernel_load_vmemmap,
302                                    SLB_VSID_KERNEL | vmemmap_llp);
303                 pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp);
304 #endif
305         }
306 
307         get_paca()->stab_rr = SLB_NUM_BOLTED;
308 
309         lflags = SLB_VSID_KERNEL | linear_llp;
310         vflags = SLB_VSID_KERNEL | vmalloc_llp;
311 
312         /* Invalidate the entire SLB (even slot 0) & all the ERATS */
313         asm volatile("isync":::"memory");
314         asm volatile("slbmte  %0,%0"::"r" (0) : "memory");
315         asm volatile("isync; slbia; isync":::"memory");
316         create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, 0);
317 
318         create_shadowed_slbe(VMALLOC_START, mmu_kernel_ssize, vflags, 1);
319 
320         /* For the boot cpu, we're running on the stack in init_thread_union,
321          * which is in the first segment of the linear mapping, and also
322          * get_paca()->kstack hasn't been initialized yet.
323          * For secondary cpus, we need to bolt the kernel stack entry now.
324          */
325         slb_shadow_clear(2);
326         if (raw_smp_processor_id() != boot_cpuid &&
327             (get_paca()->kstack & slb_esid_mask(mmu_kernel_ssize)) > PAGE_OFFSET)
328                 create_shadowed_slbe(get_paca()->kstack,
329                                      mmu_kernel_ssize, lflags, 2);
330 
331         asm volatile("isync":::"memory");
332 }
333 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp