~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/csky/mm/asid.c

Version: ~ [ linux-5.15-rc5 ] ~ [ linux-5.14.11 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.72 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.152 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.210 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.250 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.286 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.288 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Generic ASID allocator.
  4  *
  5  * Based on arch/arm/mm/context.c
  6  *
  7  * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
  8  * Copyright (C) 2012 ARM Ltd.
  9  */
 10 
 11 #include <linux/slab.h>
 12 #include <linux/mm_types.h>
 13 
 14 #include <asm/asid.h>
 15 
 16 #define reserved_asid(info, cpu) *per_cpu_ptr((info)->reserved, cpu)
 17 
 18 #define ASID_MASK(info)                 (~GENMASK((info)->bits - 1, 0))
 19 #define ASID_FIRST_VERSION(info)        (1UL << ((info)->bits))
 20 
 21 #define asid2idx(info, asid)            (((asid) & ~ASID_MASK(info)) >> (info)->ctxt_shift)
 22 #define idx2asid(info, idx)             (((idx) << (info)->ctxt_shift) & ~ASID_MASK(info))
 23 
 24 static void flush_context(struct asid_info *info)
 25 {
 26         int i;
 27         u64 asid;
 28 
 29         /* Update the list of reserved ASIDs and the ASID bitmap. */
 30         bitmap_clear(info->map, 0, NUM_CTXT_ASIDS(info));
 31 
 32         for_each_possible_cpu(i) {
 33                 asid = atomic64_xchg_relaxed(&active_asid(info, i), 0);
 34                 /*
 35                  * If this CPU has already been through a
 36                  * rollover, but hasn't run another task in
 37                  * the meantime, we must preserve its reserved
 38                  * ASID, as this is the only trace we have of
 39                  * the process it is still running.
 40                  */
 41                 if (asid == 0)
 42                         asid = reserved_asid(info, i);
 43                 __set_bit(asid2idx(info, asid), info->map);
 44                 reserved_asid(info, i) = asid;
 45         }
 46 
 47         /*
 48          * Queue a TLB invalidation for each CPU to perform on next
 49          * context-switch
 50          */
 51         cpumask_setall(&info->flush_pending);
 52 }
 53 
 54 static bool check_update_reserved_asid(struct asid_info *info, u64 asid,
 55                                        u64 newasid)
 56 {
 57         int cpu;
 58         bool hit = false;
 59 
 60         /*
 61          * Iterate over the set of reserved ASIDs looking for a match.
 62          * If we find one, then we can update our mm to use newasid
 63          * (i.e. the same ASID in the current generation) but we can't
 64          * exit the loop early, since we need to ensure that all copies
 65          * of the old ASID are updated to reflect the mm. Failure to do
 66          * so could result in us missing the reserved ASID in a future
 67          * generation.
 68          */
 69         for_each_possible_cpu(cpu) {
 70                 if (reserved_asid(info, cpu) == asid) {
 71                         hit = true;
 72                         reserved_asid(info, cpu) = newasid;
 73                 }
 74         }
 75 
 76         return hit;
 77 }
 78 
 79 static u64 new_context(struct asid_info *info, atomic64_t *pasid,
 80                        struct mm_struct *mm)
 81 {
 82         static u32 cur_idx = 1;
 83         u64 asid = atomic64_read(pasid);
 84         u64 generation = atomic64_read(&info->generation);
 85 
 86         if (asid != 0) {
 87                 u64 newasid = generation | (asid & ~ASID_MASK(info));
 88 
 89                 /*
 90                  * If our current ASID was active during a rollover, we
 91                  * can continue to use it and this was just a false alarm.
 92                  */
 93                 if (check_update_reserved_asid(info, asid, newasid))
 94                         return newasid;
 95 
 96                 /*
 97                  * We had a valid ASID in a previous life, so try to re-use
 98                  * it if possible.
 99                  */
100                 if (!__test_and_set_bit(asid2idx(info, asid), info->map))
101                         return newasid;
102         }
103 
104         /*
105          * Allocate a free ASID. If we can't find one, take a note of the
106          * currently active ASIDs and mark the TLBs as requiring flushes.  We
107          * always count from ASID #2 (index 1), as we use ASID #0 when setting
108          * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
109          * pairs.
110          */
111         asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), cur_idx);
112         if (asid != NUM_CTXT_ASIDS(info))
113                 goto set_asid;
114 
115         /* We're out of ASIDs, so increment the global generation count */
116         generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION(info),
117                                                  &info->generation);
118         flush_context(info);
119 
120         /* We have more ASIDs than CPUs, so this will always succeed */
121         asid = find_next_zero_bit(info->map, NUM_CTXT_ASIDS(info), 1);
122 
123 set_asid:
124         __set_bit(asid, info->map);
125         cur_idx = asid;
126         cpumask_clear(mm_cpumask(mm));
127         return idx2asid(info, asid) | generation;
128 }
129 
130 /*
131  * Generate a new ASID for the context.
132  *
133  * @pasid: Pointer to the current ASID batch allocated. It will be updated
134  * with the new ASID batch.
135  * @cpu: current CPU ID. Must have been acquired through get_cpu()
136  */
137 void asid_new_context(struct asid_info *info, atomic64_t *pasid,
138                       unsigned int cpu, struct mm_struct *mm)
139 {
140         unsigned long flags;
141         u64 asid;
142 
143         raw_spin_lock_irqsave(&info->lock, flags);
144         /* Check that our ASID belongs to the current generation. */
145         asid = atomic64_read(pasid);
146         if ((asid ^ atomic64_read(&info->generation)) >> info->bits) {
147                 asid = new_context(info, pasid, mm);
148                 atomic64_set(pasid, asid);
149         }
150 
151         if (cpumask_test_and_clear_cpu(cpu, &info->flush_pending))
152                 info->flush_cpu_ctxt_cb();
153 
154         atomic64_set(&active_asid(info, cpu), asid);
155         cpumask_set_cpu(cpu, mm_cpumask(mm));
156         raw_spin_unlock_irqrestore(&info->lock, flags);
157 }
158 
159 /*
160  * Initialize the ASID allocator
161  *
162  * @info: Pointer to the asid allocator structure
163  * @bits: Number of ASIDs available
164  * @asid_per_ctxt: Number of ASIDs to allocate per-context. ASIDs are
165  * allocated contiguously for a given context. This value should be a power of
166  * 2.
167  */
168 int asid_allocator_init(struct asid_info *info,
169                         u32 bits, unsigned int asid_per_ctxt,
170                         void (*flush_cpu_ctxt_cb)(void))
171 {
172         info->bits = bits;
173         info->ctxt_shift = ilog2(asid_per_ctxt);
174         info->flush_cpu_ctxt_cb = flush_cpu_ctxt_cb;
175         /*
176          * Expect allocation after rollover to fail if we don't have at least
177          * one more ASID than CPUs. ASID #0 is always reserved.
178          */
179         WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus());
180         atomic64_set(&info->generation, ASID_FIRST_VERSION(info));
181         info->map = kcalloc(BITS_TO_LONGS(NUM_CTXT_ASIDS(info)),
182                             sizeof(*info->map), GFP_KERNEL);
183         if (!info->map)
184                 return -ENOMEM;
185 
186         raw_spin_lock_init(&info->lock);
187 
188         return 0;
189 }
190 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp