~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/powerpc/platforms/powernv/subcore.c

Version: ~ [ linux-5.3 ] ~ [ linux-5.2.14 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.72 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.143 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.192 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.192 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.73 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Copyright 2013, Michael (Ellerman|Neuling), IBM Corporation.
  3  *
  4  * This program is free software; you can redistribute it and/or
  5  * modify it under the terms of the GNU General Public License
  6  * as published by the Free Software Foundation; either version
  7  * 2 of the License, or (at your option) any later version.
  8  */
  9 
 10 #define pr_fmt(fmt)     "powernv: " fmt
 11 
 12 #include <linux/kernel.h>
 13 #include <linux/cpu.h>
 14 #include <linux/cpumask.h>
 15 #include <linux/device.h>
 16 #include <linux/gfp.h>
 17 #include <linux/smp.h>
 18 #include <linux/stop_machine.h>
 19 
 20 #include <asm/cputhreads.h>
 21 #include <asm/kvm_ppc.h>
 22 #include <asm/machdep.h>
 23 #include <asm/opal.h>
 24 #include <asm/smp.h>
 25 
 26 #include "subcore.h"
 27 #include "powernv.h"
 28 
 29 
 30 /*
 31  * Split/unsplit procedure:
 32  *
 33  * A core can be in one of three states, unsplit, 2-way split, and 4-way split.
 34  *
 35  * The mapping to subcores_per_core is simple:
 36  *
 37  *  State       | subcores_per_core
 38  *  ------------|------------------
 39  *  Unsplit     |        1
 40  *  2-way split |        2
 41  *  4-way split |        4
 42  *
 43  * The core is split along thread boundaries, the mapping between subcores and
 44  * threads is as follows:
 45  *
 46  *  Unsplit:
 47  *          ----------------------------
 48  *  Subcore |            0             |
 49  *          ----------------------------
 50  *  Thread  |  0  1  2  3  4  5  6  7  |
 51  *          ----------------------------
 52  *
 53  *  2-way split:
 54  *          -------------------------------------
 55  *  Subcore |        0        |        1        |
 56  *          -------------------------------------
 57  *  Thread  |  0   1   2   3  |  4   5   6   7  |
 58  *          -------------------------------------
 59  *
 60  *  4-way split:
 61  *          -----------------------------------------
 62  *  Subcore |    0    |    1    |    2    |    3    |
 63  *          -----------------------------------------
 64  *  Thread  |  0   1  |  2   3  |  4   5  |  6   7  |
 65  *          -----------------------------------------
 66  *
 67  *
 68  * Transitions
 69  * -----------
 70  *
 71  * It is not possible to transition between either of the split states, the
 72  * core must first be unsplit. The legal transitions are:
 73  *
 74  *  -----------          ---------------
 75  *  |         |  <---->  | 2-way split |
 76  *  |         |          ---------------
 77  *  | Unsplit |
 78  *  |         |          ---------------
 79  *  |         |  <---->  | 4-way split |
 80  *  -----------          ---------------
 81  *
 82  * Unsplitting
 83  * -----------
 84  *
 85  * Unsplitting is the simpler procedure. It requires thread 0 to request the
 86  * unsplit while all other threads NAP.
 87  *
 88  * Thread 0 clears HID0_POWER8_DYNLPARDIS (Dynamic LPAR Disable). This tells
 89  * the hardware that if all threads except 0 are napping, the hardware should
 90  * unsplit the core.
 91  *
 92  * Non-zero threads are sent to a NAP loop, they don't exit the loop until they
 93  * see the core unsplit.
 94  *
 95  * Core 0 spins waiting for the hardware to see all the other threads napping
 96  * and perform the unsplit.
 97  *
 98  * Once thread 0 sees the unsplit, it IPIs the secondary threads to wake them
 99  * out of NAP. They will then see the core unsplit and exit the NAP loop.
100  *
101  * Splitting
102  * ---------
103  *
104  * The basic splitting procedure is fairly straight forward. However it is
105  * complicated by the fact that after the split occurs, the newly created
106  * subcores are not in a fully initialised state.
107  *
108  * Most notably the subcores do not have the correct value for SDR1, which
109  * means they must not be running in virtual mode when the split occurs. The
110  * subcores have separate timebases SPRs but these are pre-synchronised by
111  * opal.
112  *
113  * To begin with secondary threads are sent to an assembly routine. There they
114  * switch to real mode, so they are immune to the uninitialised SDR1 value.
115  * Once in real mode they indicate that they are in real mode, and spin waiting
116  * to see the core split.
117  *
118  * Thread 0 waits to see that all secondaries are in real mode, and then begins
119  * the splitting procedure. It firstly sets HID0_POWER8_DYNLPARDIS, which
120  * prevents the hardware from unsplitting. Then it sets the appropriate HID bit
121  * to request the split, and spins waiting to see that the split has happened.
122  *
123  * Concurrently the secondaries will notice the split. When they do they set up
124  * their SPRs, notably SDR1, and then they can return to virtual mode and exit
125  * the procedure.
126  */
127 
128 /* Initialised at boot by subcore_init() */
129 static int subcores_per_core;
130 
131 /*
132  * Used to communicate to offline cpus that we want them to pop out of the
133  * offline loop and do a split or unsplit.
134  *
135  * 0 - no split happening
136  * 1 - unsplit in progress
137  * 2 - split to 2 in progress
138  * 4 - split to 4 in progress
139  */
140 static int new_split_mode;
141 
142 static cpumask_var_t cpu_offline_mask;
143 
144 struct split_state {
145         u8 step;
146         u8 master;
147 };
148 
149 static DEFINE_PER_CPU(struct split_state, split_state);
150 
151 static void wait_for_sync_step(int step)
152 {
153         int i, cpu = smp_processor_id();
154 
155         for (i = cpu + 1; i < cpu + threads_per_core; i++)
156                 while(per_cpu(split_state, i).step < step)
157                         barrier();
158 
159         /* Order the wait loop vs any subsequent loads/stores. */
160         mb();
161 }
162 
163 static void update_hid_in_slw(u64 hid0)
164 {
165         u64 idle_states = pnv_get_supported_cpuidle_states();
166 
167         if (idle_states & OPAL_PM_WINKLE_ENABLED) {
168                 /* OPAL call to patch slw with the new HID0 value */
169                 u64 cpu_pir = hard_smp_processor_id();
170 
171                 opal_slw_set_reg(cpu_pir, SPRN_HID0, hid0);
172         }
173 }
174 
175 static void unsplit_core(void)
176 {
177         u64 hid0, mask;
178         int i, cpu;
179 
180         mask = HID0_POWER8_2LPARMODE | HID0_POWER8_4LPARMODE;
181 
182         cpu = smp_processor_id();
183         if (cpu_thread_in_core(cpu) != 0) {
184                 while (mfspr(SPRN_HID0) & mask)
185                         power7_nap(0);
186 
187                 per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT;
188                 return;
189         }
190 
191         hid0 = mfspr(SPRN_HID0);
192         hid0 &= ~HID0_POWER8_DYNLPARDIS;
193         update_power8_hid0(hid0);
194         update_hid_in_slw(hid0);
195 
196         while (mfspr(SPRN_HID0) & mask)
197                 cpu_relax();
198 
199         /* Wake secondaries out of NAP */
200         for (i = cpu + 1; i < cpu + threads_per_core; i++)
201                 smp_send_reschedule(i);
202 
203         wait_for_sync_step(SYNC_STEP_UNSPLIT);
204 }
205 
206 static void split_core(int new_mode)
207 {
208         struct {  u64 value; u64 mask; } split_parms[2] = {
209                 { HID0_POWER8_1TO2LPAR, HID0_POWER8_2LPARMODE },
210                 { HID0_POWER8_1TO4LPAR, HID0_POWER8_4LPARMODE }
211         };
212         int i, cpu;
213         u64 hid0;
214 
215         /* Convert new_mode (2 or 4) into an index into our parms array */
216         i = (new_mode >> 1) - 1;
217         BUG_ON(i < 0 || i > 1);
218 
219         cpu = smp_processor_id();
220         if (cpu_thread_in_core(cpu) != 0) {
221                 split_core_secondary_loop(&per_cpu(split_state, cpu).step);
222                 return;
223         }
224 
225         wait_for_sync_step(SYNC_STEP_REAL_MODE);
226 
227         /* Write new mode */
228         hid0  = mfspr(SPRN_HID0);
229         hid0 |= HID0_POWER8_DYNLPARDIS | split_parms[i].value;
230         update_power8_hid0(hid0);
231         update_hid_in_slw(hid0);
232 
233         /* Wait for it to happen */
234         while (!(mfspr(SPRN_HID0) & split_parms[i].mask))
235                 cpu_relax();
236 }
237 
238 static void cpu_do_split(int new_mode)
239 {
240         /*
241          * At boot subcores_per_core will be 0, so we will always unsplit at
242          * boot. In the usual case where the core is already unsplit it's a
243          * nop, and this just ensures the kernel's notion of the mode is
244          * consistent with the hardware.
245          */
246         if (subcores_per_core != 1)
247                 unsplit_core();
248 
249         if (new_mode != 1)
250                 split_core(new_mode);
251 
252         mb();
253         per_cpu(split_state, smp_processor_id()).step = SYNC_STEP_FINISHED;
254 }
255 
256 bool cpu_core_split_required(void)
257 {
258         smp_rmb();
259 
260         if (!new_split_mode)
261                 return false;
262 
263         cpu_do_split(new_split_mode);
264 
265         return true;
266 }
267 
268 void update_subcore_sibling_mask(void)
269 {
270         int cpu;
271         /*
272          * sibling mask for the first cpu. Left shift this by required bits
273          * to get sibling mask for the rest of the cpus.
274          */
275         int sibling_mask_first_cpu =  (1 << threads_per_subcore) - 1;
276 
277         for_each_possible_cpu(cpu) {
278                 int tid = cpu_thread_in_core(cpu);
279                 int offset = (tid / threads_per_subcore) * threads_per_subcore;
280                 int mask = sibling_mask_first_cpu << offset;
281 
282                 paca[cpu].subcore_sibling_mask = mask;
283 
284         }
285 }
286 
287 static int cpu_update_split_mode(void *data)
288 {
289         int cpu, new_mode = *(int *)data;
290 
291         if (this_cpu_ptr(&split_state)->master) {
292                 new_split_mode = new_mode;
293                 smp_wmb();
294 
295                 cpumask_andnot(cpu_offline_mask, cpu_present_mask,
296                                cpu_online_mask);
297 
298                 /* This should work even though the cpu is offline */
299                 for_each_cpu(cpu, cpu_offline_mask)
300                         smp_send_reschedule(cpu);
301         }
302 
303         cpu_do_split(new_mode);
304 
305         if (this_cpu_ptr(&split_state)->master) {
306                 /* Wait for all cpus to finish before we touch subcores_per_core */
307                 for_each_present_cpu(cpu) {
308                         if (cpu >= setup_max_cpus)
309                                 break;
310 
311                         while(per_cpu(split_state, cpu).step < SYNC_STEP_FINISHED)
312                                 barrier();
313                 }
314 
315                 new_split_mode = 0;
316 
317                 /* Make the new mode public */
318                 subcores_per_core = new_mode;
319                 threads_per_subcore = threads_per_core / subcores_per_core;
320                 update_subcore_sibling_mask();
321 
322                 /* Make sure the new mode is written before we exit */
323                 mb();
324         }
325 
326         return 0;
327 }
328 
329 static int set_subcores_per_core(int new_mode)
330 {
331         struct split_state *state;
332         int cpu;
333 
334         if (kvm_hv_mode_active()) {
335                 pr_err("Unable to change split core mode while KVM active.\n");
336                 return -EBUSY;
337         }
338 
339         /*
340          * We are only called at boot, or from the sysfs write. If that ever
341          * changes we'll need a lock here.
342          */
343         BUG_ON(new_mode < 1 || new_mode > 4 || new_mode == 3);
344 
345         for_each_present_cpu(cpu) {
346                 state = &per_cpu(split_state, cpu);
347                 state->step = SYNC_STEP_INITIAL;
348                 state->master = 0;
349         }
350 
351         get_online_cpus();
352 
353         /* This cpu will update the globals before exiting stop machine */
354         this_cpu_ptr(&split_state)->master = 1;
355 
356         /* Ensure state is consistent before we call the other cpus */
357         mb();
358 
359         stop_machine(cpu_update_split_mode, &new_mode, cpu_online_mask);
360 
361         put_online_cpus();
362 
363         return 0;
364 }
365 
366 static ssize_t __used store_subcores_per_core(struct device *dev,
367                 struct device_attribute *attr, const char *buf,
368                 size_t count)
369 {
370         unsigned long val;
371         int rc;
372 
373         /* We are serialised by the attribute lock */
374 
375         rc = sscanf(buf, "%lx", &val);
376         if (rc != 1)
377                 return -EINVAL;
378 
379         switch (val) {
380         case 1:
381         case 2:
382         case 4:
383                 if (subcores_per_core == val)
384                         /* Nothing to do */
385                         goto out;
386                 break;
387         default:
388                 return -EINVAL;
389         }
390 
391         rc = set_subcores_per_core(val);
392         if (rc)
393                 return rc;
394 
395 out:
396         return count;
397 }
398 
399 static ssize_t show_subcores_per_core(struct device *dev,
400                 struct device_attribute *attr, char *buf)
401 {
402         return sprintf(buf, "%x\n", subcores_per_core);
403 }
404 
405 static DEVICE_ATTR(subcores_per_core, 0644,
406                 show_subcores_per_core, store_subcores_per_core);
407 
408 static int subcore_init(void)
409 {
410         if (!cpu_has_feature(CPU_FTR_SUBCORE))
411                 return 0;
412 
413         /*
414          * We need all threads in a core to be present to split/unsplit so
415          * continue only if max_cpus are aligned to threads_per_core.
416          */
417         if (setup_max_cpus % threads_per_core)
418                 return 0;
419 
420         BUG_ON(!alloc_cpumask_var(&cpu_offline_mask, GFP_KERNEL));
421 
422         set_subcores_per_core(1);
423 
424         return device_create_file(cpu_subsys.dev_root,
425                                   &dev_attr_subcores_per_core);
426 }
427 machine_device_initcall(powernv, subcore_init);
428 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp