~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/riscv/mm/context.c

Version: ~ [ linux-6.0-rc1 ] ~ [ linux-5.19.1 ] ~ [ linux-5.18.17 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.60 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.136 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.210 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.255 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.290 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.325 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Copyright (C) 2012 Regents of the University of California
  4  * Copyright (C) 2017 SiFive
  5  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
  6  */
  7 
  8 #include <linux/bitops.h>
  9 #include <linux/cpumask.h>
 10 #include <linux/mm.h>
 11 #include <linux/percpu.h>
 12 #include <linux/slab.h>
 13 #include <linux/spinlock.h>
 14 #include <linux/static_key.h>
 15 #include <asm/tlbflush.h>
 16 #include <asm/cacheflush.h>
 17 #include <asm/mmu_context.h>
 18 
 19 #ifdef CONFIG_MMU
 20 
 21 static DEFINE_STATIC_KEY_FALSE(use_asid_allocator);
 22 
 23 static unsigned long asid_bits;
 24 static unsigned long num_asids;
 25 static unsigned long asid_mask;
 26 
 27 static atomic_long_t current_version;
 28 
 29 static DEFINE_RAW_SPINLOCK(context_lock);
 30 static cpumask_t context_tlb_flush_pending;
 31 static unsigned long *context_asid_map;
 32 
 33 static DEFINE_PER_CPU(atomic_long_t, active_context);
 34 static DEFINE_PER_CPU(unsigned long, reserved_context);
 35 
 36 static bool check_update_reserved_context(unsigned long cntx,
 37                                           unsigned long newcntx)
 38 {
 39         int cpu;
 40         bool hit = false;
 41 
 42         /*
 43          * Iterate over the set of reserved CONTEXT looking for a match.
 44          * If we find one, then we can update our mm to use new CONTEXT
 45          * (i.e. the same CONTEXT in the current_version) but we can't
 46          * exit the loop early, since we need to ensure that all copies
 47          * of the old CONTEXT are updated to reflect the mm. Failure to do
 48          * so could result in us missing the reserved CONTEXT in a future
 49          * version.
 50          */
 51         for_each_possible_cpu(cpu) {
 52                 if (per_cpu(reserved_context, cpu) == cntx) {
 53                         hit = true;
 54                         per_cpu(reserved_context, cpu) = newcntx;
 55                 }
 56         }
 57 
 58         return hit;
 59 }
 60 
 61 static void __flush_context(void)
 62 {
 63         int i;
 64         unsigned long cntx;
 65 
 66         /* Must be called with context_lock held */
 67         lockdep_assert_held(&context_lock);
 68 
 69         /* Update the list of reserved ASIDs and the ASID bitmap. */
 70         bitmap_clear(context_asid_map, 0, num_asids);
 71 
 72         /* Mark already active ASIDs as used */
 73         for_each_possible_cpu(i) {
 74                 cntx = atomic_long_xchg_relaxed(&per_cpu(active_context, i), 0);
 75                 /*
 76                  * If this CPU has already been through a rollover, but
 77                  * hasn't run another task in the meantime, we must preserve
 78                  * its reserved CONTEXT, as this is the only trace we have of
 79                  * the process it is still running.
 80                  */
 81                 if (cntx == 0)
 82                         cntx = per_cpu(reserved_context, i);
 83 
 84                 __set_bit(cntx & asid_mask, context_asid_map);
 85                 per_cpu(reserved_context, i) = cntx;
 86         }
 87 
 88         /* Mark ASID #0 as used because it is used at boot-time */
 89         __set_bit(0, context_asid_map);
 90 
 91         /* Queue a TLB invalidation for each CPU on next context-switch */
 92         cpumask_setall(&context_tlb_flush_pending);
 93 }
 94 
 95 static unsigned long __new_context(struct mm_struct *mm)
 96 {
 97         static u32 cur_idx = 1;
 98         unsigned long cntx = atomic_long_read(&mm->context.id);
 99         unsigned long asid, ver = atomic_long_read(&current_version);
100 
101         /* Must be called with context_lock held */
102         lockdep_assert_held(&context_lock);
103 
104         if (cntx != 0) {
105                 unsigned long newcntx = ver | (cntx & asid_mask);
106 
107                 /*
108                  * If our current CONTEXT was active during a rollover, we
109                  * can continue to use it and this was just a false alarm.
110                  */
111                 if (check_update_reserved_context(cntx, newcntx))
112                         return newcntx;
113 
114                 /*
115                  * We had a valid CONTEXT in a previous life, so try to
116                  * re-use it if possible.
117                  */
118                 if (!__test_and_set_bit(cntx & asid_mask, context_asid_map))
119                         return newcntx;
120         }
121 
122         /*
123          * Allocate a free ASID. If we can't find one then increment
124          * current_version and flush all ASIDs.
125          */
126         asid = find_next_zero_bit(context_asid_map, num_asids, cur_idx);
127         if (asid != num_asids)
128                 goto set_asid;
129 
130         /* We're out of ASIDs, so increment current_version */
131         ver = atomic_long_add_return_relaxed(num_asids, &current_version);
132 
133         /* Flush everything  */
134         __flush_context();
135 
136         /* We have more ASIDs than CPUs, so this will always succeed */
137         asid = find_next_zero_bit(context_asid_map, num_asids, 1);
138 
139 set_asid:
140         __set_bit(asid, context_asid_map);
141         cur_idx = asid;
142         return asid | ver;
143 }
144 
145 static void set_mm_asid(struct mm_struct *mm, unsigned int cpu)
146 {
147         unsigned long flags;
148         bool need_flush_tlb = false;
149         unsigned long cntx, old_active_cntx;
150 
151         cntx = atomic_long_read(&mm->context.id);
152 
153         /*
154          * If our active_context is non-zero and the context matches the
155          * current_version, then we update the active_context entry with a
156          * relaxed cmpxchg.
157          *
158          * Following is how we handle racing with a concurrent rollover:
159          *
160          * - We get a zero back from the cmpxchg and end up waiting on the
161          *   lock. Taking the lock synchronises with the rollover and so
162          *   we are forced to see the updated verion.
163          *
164          * - We get a valid context back from the cmpxchg then we continue
165          *   using old ASID because __flush_context() would have marked ASID
166          *   of active_context as used and next context switch we will
167          *   allocate new context.
168          */
169         old_active_cntx = atomic_long_read(&per_cpu(active_context, cpu));
170         if (old_active_cntx &&
171             ((cntx & ~asid_mask) == atomic_long_read(&current_version)) &&
172             atomic_long_cmpxchg_relaxed(&per_cpu(active_context, cpu),
173                                         old_active_cntx, cntx))
174                 goto switch_mm_fast;
175 
176         raw_spin_lock_irqsave(&context_lock, flags);
177 
178         /* Check that our ASID belongs to the current_version. */
179         cntx = atomic_long_read(&mm->context.id);
180         if ((cntx & ~asid_mask) != atomic_long_read(&current_version)) {
181                 cntx = __new_context(mm);
182                 atomic_long_set(&mm->context.id, cntx);
183         }
184 
185         if (cpumask_test_and_clear_cpu(cpu, &context_tlb_flush_pending))
186                 need_flush_tlb = true;
187 
188         atomic_long_set(&per_cpu(active_context, cpu), cntx);
189 
190         raw_spin_unlock_irqrestore(&context_lock, flags);
191 
192 switch_mm_fast:
193         csr_write(CSR_SATP, virt_to_pfn(mm->pgd) |
194                   ((cntx & asid_mask) << SATP_ASID_SHIFT) |
195                   SATP_MODE);
196 
197         if (need_flush_tlb)
198                 local_flush_tlb_all();
199 }
200 
201 static void set_mm_noasid(struct mm_struct *mm)
202 {
203         /* Switch the page table and blindly nuke entire local TLB */
204         csr_write(CSR_SATP, virt_to_pfn(mm->pgd) | SATP_MODE);
205         local_flush_tlb_all();
206 }
207 
208 static inline void set_mm(struct mm_struct *mm, unsigned int cpu)
209 {
210         if (static_branch_unlikely(&use_asid_allocator))
211                 set_mm_asid(mm, cpu);
212         else
213                 set_mm_noasid(mm);
214 }
215 
216 static int asids_init(void)
217 {
218         unsigned long old;
219 
220         /* Figure-out number of ASID bits in HW */
221         old = csr_read(CSR_SATP);
222         asid_bits = old | (SATP_ASID_MASK << SATP_ASID_SHIFT);
223         csr_write(CSR_SATP, asid_bits);
224         asid_bits = (csr_read(CSR_SATP) >> SATP_ASID_SHIFT)  & SATP_ASID_MASK;
225         asid_bits = fls_long(asid_bits);
226         csr_write(CSR_SATP, old);
227 
228         /*
229          * In the process of determining number of ASID bits (above)
230          * we polluted the TLB of current HART so let's do TLB flushed
231          * to remove unwanted TLB enteries.
232          */
233         local_flush_tlb_all();
234 
235         /* Pre-compute ASID details */
236         num_asids = 1 << asid_bits;
237         asid_mask = num_asids - 1;
238 
239         /*
240          * Use ASID allocator only if number of HW ASIDs are
241          * at-least twice more than CPUs
242          */
243         if (num_asids > (2 * num_possible_cpus())) {
244                 atomic_long_set(&current_version, num_asids);
245 
246                 context_asid_map = kcalloc(BITS_TO_LONGS(num_asids),
247                                    sizeof(*context_asid_map), GFP_KERNEL);
248                 if (!context_asid_map)
249                         panic("Failed to allocate bitmap for %lu ASIDs\n",
250                               num_asids);
251 
252                 __set_bit(0, context_asid_map);
253 
254                 static_branch_enable(&use_asid_allocator);
255 
256                 pr_info("ASID allocator using %lu bits (%lu entries)\n",
257                         asid_bits, num_asids);
258         } else {
259                 pr_info("ASID allocator disabled\n");
260         }
261 
262         return 0;
263 }
264 early_initcall(asids_init);
265 #else
266 static inline void set_mm(struct mm_struct *mm, unsigned int cpu)
267 {
268         /* Nothing to do here when there is no MMU */
269 }
270 #endif
271 
272 /*
273  * When necessary, performs a deferred icache flush for the given MM context,
274  * on the local CPU.  RISC-V has no direct mechanism for instruction cache
275  * shoot downs, so instead we send an IPI that informs the remote harts they
276  * need to flush their local instruction caches.  To avoid pathologically slow
277  * behavior in a common case (a bunch of single-hart processes on a many-hart
278  * machine, ie 'make -j') we avoid the IPIs for harts that are not currently
279  * executing a MM context and instead schedule a deferred local instruction
280  * cache flush to be performed before execution resumes on each hart.  This
281  * actually performs that local instruction cache flush, which implicitly only
282  * refers to the current hart.
283  */
284 static inline void flush_icache_deferred(struct mm_struct *mm)
285 {
286 #ifdef CONFIG_SMP
287         unsigned int cpu = smp_processor_id();
288         cpumask_t *mask = &mm->context.icache_stale_mask;
289 
290         if (cpumask_test_cpu(cpu, mask)) {
291                 cpumask_clear_cpu(cpu, mask);
292                 /*
293                  * Ensure the remote hart's writes are visible to this hart.
294                  * This pairs with a barrier in flush_icache_mm.
295                  */
296                 smp_mb();
297                 local_flush_icache_all();
298         }
299 
300 #endif
301 }
302 
303 void switch_mm(struct mm_struct *prev, struct mm_struct *next,
304         struct task_struct *task)
305 {
306         unsigned int cpu;
307 
308         if (unlikely(prev == next))
309                 return;
310 
311         /*
312          * Mark the current MM context as inactive, and the next as
313          * active.  This is at least used by the icache flushing
314          * routines in order to determine who should be flushed.
315          */
316         cpu = smp_processor_id();
317 
318         cpumask_clear_cpu(cpu, mm_cpumask(prev));
319         cpumask_set_cpu(cpu, mm_cpumask(next));
320 
321         set_mm(next, cpu);
322 
323         flush_icache_deferred(next);
324 }
325 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp