~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/kernel/cpufeature.c

Version: ~ [ linux-5.4-rc3 ] ~ [ linux-5.3.6 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.79 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.149 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.196 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.196 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.75 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Contains CPU feature definitions
  3  *
  4  * Copyright (C) 2015 ARM Ltd.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  *
 10  * This program is distributed in the hope that it will be useful,
 11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  *
 15  * You should have received a copy of the GNU General Public License
 16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17  */
 18 
 19 #define pr_fmt(fmt) "CPU features: " fmt
 20 
 21 #include <linux/bsearch.h>
 22 #include <linux/cpumask.h>
 23 #include <linux/sort.h>
 24 #include <linux/stop_machine.h>
 25 #include <linux/types.h>
 26 #include <asm/cpu.h>
 27 #include <asm/cpufeature.h>
 28 #include <asm/cpu_ops.h>
 29 #include <asm/mmu_context.h>
 30 #include <asm/processor.h>
 31 #include <asm/sysreg.h>
 32 #include <asm/virt.h>
 33 
 34 unsigned long elf_hwcap __read_mostly;
 35 EXPORT_SYMBOL_GPL(elf_hwcap);
 36 
 37 #ifdef CONFIG_COMPAT
 38 #define COMPAT_ELF_HWCAP_DEFAULT        \
 39                                 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
 40                                  COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
 41                                  COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
 42                                  COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
 43                                  COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
 44                                  COMPAT_HWCAP_LPAE)
 45 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
 46 unsigned int compat_elf_hwcap2 __read_mostly;
 47 #endif
 48 
 49 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 50 EXPORT_SYMBOL(cpu_hwcaps);
 51 
 52 DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
 53 EXPORT_SYMBOL(cpu_hwcap_keys);
 54 
 55 #define __ARM64_FTR_BITS(SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
 56         {                                               \
 57                 .sign = SIGNED,                         \
 58                 .strict = STRICT,                       \
 59                 .type = TYPE,                           \
 60                 .shift = SHIFT,                         \
 61                 .width = WIDTH,                         \
 62                 .safe_val = SAFE_VAL,                   \
 63         }
 64 
 65 /* Define a feature with unsigned values */
 66 #define ARM64_FTR_BITS(STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
 67         __ARM64_FTR_BITS(FTR_UNSIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
 68 
 69 /* Define a feature with a signed value */
 70 #define S_ARM64_FTR_BITS(STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
 71         __ARM64_FTR_BITS(FTR_SIGNED, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
 72 
 73 #define ARM64_FTR_END                                   \
 74         {                                               \
 75                 .width = 0,                             \
 76         }
 77 
 78 /* meta feature for alternatives */
 79 static bool __maybe_unused
 80 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
 81 
 82 
 83 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
 84         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
 85         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
 86         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0),
 87         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
 88         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
 89         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
 90         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
 91         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
 92         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* RAZ */
 93         ARM64_FTR_END,
 94 };
 95 
 96 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
 97         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
 98         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0),
 99         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_GIC_SHIFT, 4, 0),
100         S_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
101         S_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
102         /* Linux doesn't care about the EL3 */
103         ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64PFR0_EL3_SHIFT, 4, 0),
104         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL2_SHIFT, 4, 0),
105         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
106         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
107         ARM64_FTR_END,
108 };
109 
110 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
111         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
112         S_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
113         S_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
114         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
115         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
116         /* Linux shouldn't care about secure memory */
117         ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
118         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
119         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
120         /*
121          * Differing PARange is fine as long as all peripherals and memory are mapped
122          * within the minimum PARange of all CPUs
123          */
124         ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
125         ARM64_FTR_END,
126 };
127 
128 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
129         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
130         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
131         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
132         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
133         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
134         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
135         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
136         ARM64_FTR_END,
137 };
138 
139 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
140         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
141         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
142         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
143         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
144         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
145         ARM64_FTR_END,
146 };
147 
148 static const struct arm64_ftr_bits ftr_ctr[] = {
149         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 31, 1, 1),        /* RAO */
150         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 3, 0),
151         ARM64_FTR_BITS(FTR_STRICT, FTR_HIGHER_SAFE, 24, 4, 0),  /* CWG */
152         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),   /* ERG */
153         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 1),   /* DminLine */
154         /*
155          * Linux can handle differing I-cache policies. Userspace JITs will
156          * make use of *minLine.
157          * If we have differing I-cache policies, report it as the weakest - AIVIVT.
158          */
159         ARM64_FTR_BITS(FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_AIVIVT),  /* L1Ip */
160         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 10, 0),        /* RAZ */
161         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),    /* IminLine */
162         ARM64_FTR_END,
163 };
164 
165 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
166         .name           = "SYS_CTR_EL0",
167         .ftr_bits       = ftr_ctr
168 };
169 
170 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
171         S_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 28, 4, 0xf),    /* InnerShr */
172         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 24, 4, 0),        /* FCSE */
173         ARM64_FTR_BITS(FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0),        /* AuxReg */
174         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 4, 0),        /* TCM */
175         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0),        /* ShareLvl */
176         S_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0xf),     /* OuterShr */
177         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0), /* PMSA */
178         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0), /* VMSA */
179         ARM64_FTR_END,
180 };
181 
182 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
183         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 32, 32, 0),
184         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
185         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
186         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
187         S_ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
188         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
189         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
190         ARM64_FTR_END,
191 };
192 
193 static const struct arm64_ftr_bits ftr_mvfr2[] = {
194         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0),        /* RAZ */
195         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0),         /* FPMisc */
196         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0),         /* SIMDMisc */
197         ARM64_FTR_END,
198 };
199 
200 static const struct arm64_ftr_bits ftr_dczid[] = {
201         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 5, 27, 0),        /* RAZ */
202         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 1, 1),         /* DZP */
203         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),    /* BS */
204         ARM64_FTR_END,
205 };
206 
207 
208 static const struct arm64_ftr_bits ftr_id_isar5[] = {
209         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_RDM_SHIFT, 4, 0),
210         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 20, 4, 0),        /* RAZ */
211         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_CRC32_SHIFT, 4, 0),
212         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA2_SHIFT, 4, 0),
213         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SHA1_SHIFT, 4, 0),
214         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_AES_SHIFT, 4, 0),
215         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, ID_ISAR5_SEVL_SHIFT, 4, 0),
216         ARM64_FTR_END,
217 };
218 
219 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
220         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 24, 0),        /* RAZ */
221         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0),         /* ac2 */
222         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0),         /* RAZ */
223         ARM64_FTR_END,
224 };
225 
226 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
227         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 16, 16, 0),       /* RAZ */
228         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 12, 4, 0),        /* State3 */
229         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 8, 4, 0),         /* State2 */
230         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 4, 4, 0),         /* State1 */
231         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 4, 0),         /* State0 */
232         ARM64_FTR_END,
233 };
234 
235 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
236         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
237         S_ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0xf),       /* PerfMon */
238         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
239         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
240         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
241         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
242         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
243         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
244         ARM64_FTR_END,
245 };
246 
247 /*
248  * Common ftr bits for a 32bit register with all hidden, strict
249  * attributes, with 4bit feature fields and a default safe value of
250  * 0. Covers the following 32bit registers:
251  * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
252  */
253 static const struct arm64_ftr_bits ftr_generic_32bits[] = {
254         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
255         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
256         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
257         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
258         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
259         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
260         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
261         ARM64_FTR_BITS(FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
262         ARM64_FTR_END,
263 };
264 
265 static const struct arm64_ftr_bits ftr_generic[] = {
266         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
267         ARM64_FTR_END,
268 };
269 
270 static const struct arm64_ftr_bits ftr_generic32[] = {
271         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 32, 0),
272         ARM64_FTR_END,
273 };
274 
275 static const struct arm64_ftr_bits ftr_aa64raz[] = {
276         ARM64_FTR_BITS(FTR_STRICT, FTR_EXACT, 0, 64, 0),
277         ARM64_FTR_END,
278 };
279 
280 #define ARM64_FTR_REG(id, table) {              \
281         .sys_id = id,                           \
282         .reg =  &(struct arm64_ftr_reg){        \
283                 .name = #id,                    \
284                 .ftr_bits = &((table)[0]),      \
285         }}
286 
287 static const struct __ftr_reg_entry {
288         u32                     sys_id;
289         struct arm64_ftr_reg    *reg;
290 } arm64_ftr_regs[] = {
291 
292         /* Op1 = 0, CRn = 0, CRm = 1 */
293         ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
294         ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
295         ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
296         ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
297         ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
298         ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
299         ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
300 
301         /* Op1 = 0, CRn = 0, CRm = 2 */
302         ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
303         ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
304         ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
305         ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
306         ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
307         ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
308         ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
309 
310         /* Op1 = 0, CRn = 0, CRm = 3 */
311         ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
312         ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
313         ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
314 
315         /* Op1 = 0, CRn = 0, CRm = 4 */
316         ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
317         ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_aa64raz),
318 
319         /* Op1 = 0, CRn = 0, CRm = 5 */
320         ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
321         ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_generic),
322 
323         /* Op1 = 0, CRn = 0, CRm = 6 */
324         ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
325         ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_aa64raz),
326 
327         /* Op1 = 0, CRn = 0, CRm = 7 */
328         ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
329         ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
330         ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
331 
332         /* Op1 = 3, CRn = 0, CRm = 0 */
333         { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
334         ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
335 
336         /* Op1 = 3, CRn = 14, CRm = 0 */
337         ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_generic32),
338 };
339 
340 static int search_cmp_ftr_reg(const void *id, const void *regp)
341 {
342         return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
343 }
344 
345 /*
346  * get_arm64_ftr_reg - Lookup a feature register entry using its
347  * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
348  * ascending order of sys_id , we use binary search to find a matching
349  * entry.
350  *
351  * returns - Upon success,  matching ftr_reg entry for id.
352  *         - NULL on failure. It is upto the caller to decide
353  *           the impact of a failure.
354  */
355 static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
356 {
357         const struct __ftr_reg_entry *ret;
358 
359         ret = bsearch((const void *)(unsigned long)sys_id,
360                         arm64_ftr_regs,
361                         ARRAY_SIZE(arm64_ftr_regs),
362                         sizeof(arm64_ftr_regs[0]),
363                         search_cmp_ftr_reg);
364         if (ret)
365                 return ret->reg;
366         return NULL;
367 }
368 
369 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
370                                s64 ftr_val)
371 {
372         u64 mask = arm64_ftr_mask(ftrp);
373 
374         reg &= ~mask;
375         reg |= (ftr_val << ftrp->shift) & mask;
376         return reg;
377 }
378 
379 static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
380                                 s64 cur)
381 {
382         s64 ret = 0;
383 
384         switch (ftrp->type) {
385         case FTR_EXACT:
386                 ret = ftrp->safe_val;
387                 break;
388         case FTR_LOWER_SAFE:
389                 ret = new < cur ? new : cur;
390                 break;
391         case FTR_HIGHER_SAFE:
392                 ret = new > cur ? new : cur;
393                 break;
394         default:
395                 BUG();
396         }
397 
398         return ret;
399 }
400 
401 static void __init sort_ftr_regs(void)
402 {
403         int i;
404 
405         /* Check that the array is sorted so that we can do the binary search */
406         for (i = 1; i < ARRAY_SIZE(arm64_ftr_regs); i++)
407                 BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id);
408 }
409 
410 /*
411  * Initialise the CPU feature register from Boot CPU values.
412  * Also initiliases the strict_mask for the register.
413  */
414 static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
415 {
416         u64 val = 0;
417         u64 strict_mask = ~0x0ULL;
418         const struct arm64_ftr_bits *ftrp;
419         struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
420 
421         BUG_ON(!reg);
422 
423         for (ftrp  = reg->ftr_bits; ftrp->width; ftrp++) {
424                 s64 ftr_new = arm64_ftr_value(ftrp, new);
425 
426                 val = arm64_ftr_set_value(ftrp, val, ftr_new);
427                 if (!ftrp->strict)
428                         strict_mask &= ~arm64_ftr_mask(ftrp);
429         }
430         reg->sys_val = val;
431         reg->strict_mask = strict_mask;
432 }
433 
434 void __init init_cpu_features(struct cpuinfo_arm64 *info)
435 {
436         /* Before we start using the tables, make sure it is sorted */
437         sort_ftr_regs();
438 
439         init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
440         init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
441         init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
442         init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
443         init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
444         init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
445         init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
446         init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
447         init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
448         init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
449         init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
450         init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
451 
452         if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
453                 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
454                 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
455                 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
456                 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
457                 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
458                 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
459                 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
460                 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
461                 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
462                 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
463                 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
464                 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
465                 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
466                 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
467                 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
468                 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
469         }
470 
471 }
472 
473 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
474 {
475         const struct arm64_ftr_bits *ftrp;
476 
477         for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
478                 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
479                 s64 ftr_new = arm64_ftr_value(ftrp, new);
480 
481                 if (ftr_cur == ftr_new)
482                         continue;
483                 /* Find a safe value */
484                 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
485                 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
486         }
487 
488 }
489 
490 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
491 {
492         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
493 
494         BUG_ON(!regp);
495         update_cpu_ftr_reg(regp, val);
496         if ((boot & regp->strict_mask) == (val & regp->strict_mask))
497                 return 0;
498         pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
499                         regp->name, boot, cpu, val);
500         return 1;
501 }
502 
503 /*
504  * Update system wide CPU feature registers with the values from a
505  * non-boot CPU. Also performs SANITY checks to make sure that there
506  * aren't any insane variations from that of the boot CPU.
507  */
508 void update_cpu_features(int cpu,
509                          struct cpuinfo_arm64 *info,
510                          struct cpuinfo_arm64 *boot)
511 {
512         int taint = 0;
513 
514         /*
515          * The kernel can handle differing I-cache policies, but otherwise
516          * caches should look identical. Userspace JITs will make use of
517          * *minLine.
518          */
519         taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
520                                       info->reg_ctr, boot->reg_ctr);
521 
522         /*
523          * Userspace may perform DC ZVA instructions. Mismatched block sizes
524          * could result in too much or too little memory being zeroed if a
525          * process is preempted and migrated between CPUs.
526          */
527         taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
528                                       info->reg_dczid, boot->reg_dczid);
529 
530         /* If different, timekeeping will be broken (especially with KVM) */
531         taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
532                                       info->reg_cntfrq, boot->reg_cntfrq);
533 
534         /*
535          * The kernel uses self-hosted debug features and expects CPUs to
536          * support identical debug features. We presently need CTX_CMPs, WRPs,
537          * and BRPs to be identical.
538          * ID_AA64DFR1 is currently RES0.
539          */
540         taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
541                                       info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
542         taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
543                                       info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
544         /*
545          * Even in big.LITTLE, processors should be identical instruction-set
546          * wise.
547          */
548         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
549                                       info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
550         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
551                                       info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
552 
553         /*
554          * Differing PARange support is fine as long as all peripherals and
555          * memory are mapped within the minimum PARange of all CPUs.
556          * Linux should not care about secure memory.
557          */
558         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
559                                       info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
560         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
561                                       info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
562         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
563                                       info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
564 
565         /*
566          * EL3 is not our concern.
567          * ID_AA64PFR1 is currently RES0.
568          */
569         taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
570                                       info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
571         taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
572                                       info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
573 
574         /*
575          * If we have AArch32, we care about 32-bit features for compat.
576          * If the system doesn't support AArch32, don't update them.
577          */
578         if (id_aa64pfr0_32bit_el0(read_system_reg(SYS_ID_AA64PFR0_EL1)) &&
579                 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
580 
581                 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
582                                         info->reg_id_dfr0, boot->reg_id_dfr0);
583                 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
584                                         info->reg_id_isar0, boot->reg_id_isar0);
585                 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
586                                         info->reg_id_isar1, boot->reg_id_isar1);
587                 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
588                                         info->reg_id_isar2, boot->reg_id_isar2);
589                 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
590                                         info->reg_id_isar3, boot->reg_id_isar3);
591                 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
592                                         info->reg_id_isar4, boot->reg_id_isar4);
593                 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
594                                         info->reg_id_isar5, boot->reg_id_isar5);
595 
596                 /*
597                  * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
598                  * ACTLR formats could differ across CPUs and therefore would have to
599                  * be trapped for virtualization anyway.
600                  */
601                 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
602                                         info->reg_id_mmfr0, boot->reg_id_mmfr0);
603                 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
604                                         info->reg_id_mmfr1, boot->reg_id_mmfr1);
605                 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
606                                         info->reg_id_mmfr2, boot->reg_id_mmfr2);
607                 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
608                                         info->reg_id_mmfr3, boot->reg_id_mmfr3);
609                 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
610                                         info->reg_id_pfr0, boot->reg_id_pfr0);
611                 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
612                                         info->reg_id_pfr1, boot->reg_id_pfr1);
613                 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
614                                         info->reg_mvfr0, boot->reg_mvfr0);
615                 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
616                                         info->reg_mvfr1, boot->reg_mvfr1);
617                 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
618                                         info->reg_mvfr2, boot->reg_mvfr2);
619         }
620 
621         /*
622          * Mismatched CPU features are a recipe for disaster. Don't even
623          * pretend to support them.
624          */
625         WARN_TAINT_ONCE(taint, TAINT_CPU_OUT_OF_SPEC,
626                         "Unsupported CPU feature variation.\n");
627 }
628 
629 u64 read_system_reg(u32 id)
630 {
631         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
632 
633         /* We shouldn't get a request for an unsupported register */
634         BUG_ON(!regp);
635         return regp->sys_val;
636 }
637 
638 /*
639  * __raw_read_system_reg() - Used by a STARTING cpu before cpuinfo is populated.
640  * Read the system register on the current CPU
641  */
642 static u64 __raw_read_system_reg(u32 sys_id)
643 {
644         switch (sys_id) {
645         case SYS_ID_PFR0_EL1:           return read_cpuid(ID_PFR0_EL1);
646         case SYS_ID_PFR1_EL1:           return read_cpuid(ID_PFR1_EL1);
647         case SYS_ID_DFR0_EL1:           return read_cpuid(ID_DFR0_EL1);
648         case SYS_ID_MMFR0_EL1:          return read_cpuid(ID_MMFR0_EL1);
649         case SYS_ID_MMFR1_EL1:          return read_cpuid(ID_MMFR1_EL1);
650         case SYS_ID_MMFR2_EL1:          return read_cpuid(ID_MMFR2_EL1);
651         case SYS_ID_MMFR3_EL1:          return read_cpuid(ID_MMFR3_EL1);
652         case SYS_ID_ISAR0_EL1:          return read_cpuid(ID_ISAR0_EL1);
653         case SYS_ID_ISAR1_EL1:          return read_cpuid(ID_ISAR1_EL1);
654         case SYS_ID_ISAR2_EL1:          return read_cpuid(ID_ISAR2_EL1);
655         case SYS_ID_ISAR3_EL1:          return read_cpuid(ID_ISAR3_EL1);
656         case SYS_ID_ISAR4_EL1:          return read_cpuid(ID_ISAR4_EL1);
657         case SYS_ID_ISAR5_EL1:          return read_cpuid(ID_ISAR5_EL1);
658         case SYS_MVFR0_EL1:             return read_cpuid(MVFR0_EL1);
659         case SYS_MVFR1_EL1:             return read_cpuid(MVFR1_EL1);
660         case SYS_MVFR2_EL1:             return read_cpuid(MVFR2_EL1);
661 
662         case SYS_ID_AA64PFR0_EL1:       return read_cpuid(ID_AA64PFR0_EL1);
663         case SYS_ID_AA64PFR1_EL1:       return read_cpuid(ID_AA64PFR1_EL1);
664         case SYS_ID_AA64DFR0_EL1:       return read_cpuid(ID_AA64DFR0_EL1);
665         case SYS_ID_AA64DFR1_EL1:       return read_cpuid(ID_AA64DFR1_EL1);
666         case SYS_ID_AA64MMFR0_EL1:      return read_cpuid(ID_AA64MMFR0_EL1);
667         case SYS_ID_AA64MMFR1_EL1:      return read_cpuid(ID_AA64MMFR1_EL1);
668         case SYS_ID_AA64MMFR2_EL1:      return read_cpuid(ID_AA64MMFR2_EL1);
669         case SYS_ID_AA64ISAR0_EL1:      return read_cpuid(ID_AA64ISAR0_EL1);
670         case SYS_ID_AA64ISAR1_EL1:      return read_cpuid(ID_AA64ISAR1_EL1);
671 
672         case SYS_CNTFRQ_EL0:            return read_cpuid(CNTFRQ_EL0);
673         case SYS_CTR_EL0:               return read_cpuid(CTR_EL0);
674         case SYS_DCZID_EL0:             return read_cpuid(DCZID_EL0);
675         default:
676                 BUG();
677                 return 0;
678         }
679 }
680 
681 #include <linux/irqchip/arm-gic-v3.h>
682 
683 static bool
684 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
685 {
686         int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);
687 
688         return val >= entry->min_field_value;
689 }
690 
691 static bool
692 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
693 {
694         u64 val;
695 
696         WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
697         if (scope == SCOPE_SYSTEM)
698                 val = read_system_reg(entry->sys_reg);
699         else
700                 val = __raw_read_system_reg(entry->sys_reg);
701 
702         return feature_matches(val, entry);
703 }
704 
705 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
706 {
707         bool has_sre;
708 
709         if (!has_cpuid_feature(entry, scope))
710                 return false;
711 
712         has_sre = gic_enable_sre();
713         if (!has_sre)
714                 pr_warn_once("%s present but disabled by higher exception level\n",
715                              entry->desc);
716 
717         return has_sre;
718 }
719 
720 static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
721 {
722         u32 midr = read_cpuid_id();
723         u32 rv_min, rv_max;
724 
725         /* Cavium ThunderX pass 1.x and 2.x */
726         rv_min = 0;
727         rv_max = (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK;
728 
729         return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX, rv_min, rv_max);
730 }
731 
732 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
733 {
734         return is_kernel_in_hyp_mode();
735 }
736 
737 static bool hyp_offset_low(const struct arm64_cpu_capabilities *entry,
738                            int __unused)
739 {
740         phys_addr_t idmap_addr = virt_to_phys(__hyp_idmap_text_start);
741 
742         /*
743          * Activate the lower HYP offset only if:
744          * - the idmap doesn't clash with it,
745          * - the kernel is not running at EL2.
746          */
747         return idmap_addr > GENMASK(VA_BITS - 2, 0) && !is_kernel_in_hyp_mode();
748 }
749 
750 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
751 {
752         u64 pfr0 = read_system_reg(SYS_ID_AA64PFR0_EL1);
753 
754         return cpuid_feature_extract_signed_field(pfr0,
755                                         ID_AA64PFR0_FP_SHIFT) < 0;
756 }
757 
758 static const struct arm64_cpu_capabilities arm64_features[] = {
759         {
760                 .desc = "GIC system register CPU interface",
761                 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
762                 .def_scope = SCOPE_SYSTEM,
763                 .matches = has_useable_gicv3_cpuif,
764                 .sys_reg = SYS_ID_AA64PFR0_EL1,
765                 .field_pos = ID_AA64PFR0_GIC_SHIFT,
766                 .sign = FTR_UNSIGNED,
767                 .min_field_value = 1,
768         },
769 #ifdef CONFIG_ARM64_PAN
770         {
771                 .desc = "Privileged Access Never",
772                 .capability = ARM64_HAS_PAN,
773                 .def_scope = SCOPE_SYSTEM,
774                 .matches = has_cpuid_feature,
775                 .sys_reg = SYS_ID_AA64MMFR1_EL1,
776                 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
777                 .sign = FTR_UNSIGNED,
778                 .min_field_value = 1,
779                 .enable = cpu_enable_pan,
780         },
781 #endif /* CONFIG_ARM64_PAN */
782 #if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
783         {
784                 .desc = "LSE atomic instructions",
785                 .capability = ARM64_HAS_LSE_ATOMICS,
786                 .def_scope = SCOPE_SYSTEM,
787                 .matches = has_cpuid_feature,
788                 .sys_reg = SYS_ID_AA64ISAR0_EL1,
789                 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
790                 .sign = FTR_UNSIGNED,
791                 .min_field_value = 2,
792         },
793 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
794         {
795                 .desc = "Software prefetching using PRFM",
796                 .capability = ARM64_HAS_NO_HW_PREFETCH,
797                 .def_scope = SCOPE_SYSTEM,
798                 .matches = has_no_hw_prefetch,
799         },
800 #ifdef CONFIG_ARM64_UAO
801         {
802                 .desc = "User Access Override",
803                 .capability = ARM64_HAS_UAO,
804                 .def_scope = SCOPE_SYSTEM,
805                 .matches = has_cpuid_feature,
806                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
807                 .field_pos = ID_AA64MMFR2_UAO_SHIFT,
808                 .min_field_value = 1,
809                 .enable = cpu_enable_uao,
810         },
811 #endif /* CONFIG_ARM64_UAO */
812 #ifdef CONFIG_ARM64_PAN
813         {
814                 .capability = ARM64_ALT_PAN_NOT_UAO,
815                 .def_scope = SCOPE_SYSTEM,
816                 .matches = cpufeature_pan_not_uao,
817         },
818 #endif /* CONFIG_ARM64_PAN */
819         {
820                 .desc = "Virtualization Host Extensions",
821                 .capability = ARM64_HAS_VIRT_HOST_EXTN,
822                 .def_scope = SCOPE_SYSTEM,
823                 .matches = runs_at_el2,
824         },
825         {
826                 .desc = "32-bit EL0 Support",
827                 .capability = ARM64_HAS_32BIT_EL0,
828                 .def_scope = SCOPE_SYSTEM,
829                 .matches = has_cpuid_feature,
830                 .sys_reg = SYS_ID_AA64PFR0_EL1,
831                 .sign = FTR_UNSIGNED,
832                 .field_pos = ID_AA64PFR0_EL0_SHIFT,
833                 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
834         },
835         {
836                 .desc = "Reduced HYP mapping offset",
837                 .capability = ARM64_HYP_OFFSET_LOW,
838                 .def_scope = SCOPE_SYSTEM,
839                 .matches = hyp_offset_low,
840         },
841         {
842                 /* FP/SIMD is not implemented */
843                 .capability = ARM64_HAS_NO_FPSIMD,
844                 .def_scope = SCOPE_SYSTEM,
845                 .min_field_value = 0,
846                 .matches = has_no_fpsimd,
847         },
848         {},
849 };
850 
851 #define HWCAP_CAP(reg, field, s, min_value, type, cap)  \
852         {                                                       \
853                 .desc = #cap,                                   \
854                 .def_scope = SCOPE_SYSTEM,                      \
855                 .matches = has_cpuid_feature,                   \
856                 .sys_reg = reg,                                 \
857                 .field_pos = field,                             \
858                 .sign = s,                                      \
859                 .min_field_value = min_value,                   \
860                 .hwcap_type = type,                             \
861                 .hwcap = cap,                                   \
862         }
863 
864 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
865         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL),
866         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
867         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
868         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
869         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
870         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
871         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
872         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
873         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
874         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
875         {},
876 };
877 
878 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
879 #ifdef CONFIG_COMPAT
880         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
881         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
882         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
883         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
884         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
885 #endif
886         {},
887 };
888 
889 static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
890 {
891         switch (cap->hwcap_type) {
892         case CAP_HWCAP:
893                 elf_hwcap |= cap->hwcap;
894                 break;
895 #ifdef CONFIG_COMPAT
896         case CAP_COMPAT_HWCAP:
897                 compat_elf_hwcap |= (u32)cap->hwcap;
898                 break;
899         case CAP_COMPAT_HWCAP2:
900                 compat_elf_hwcap2 |= (u32)cap->hwcap;
901                 break;
902 #endif
903         default:
904                 WARN_ON(1);
905                 break;
906         }
907 }
908 
909 /* Check if we have a particular HWCAP enabled */
910 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
911 {
912         bool rc;
913 
914         switch (cap->hwcap_type) {
915         case CAP_HWCAP:
916                 rc = (elf_hwcap & cap->hwcap) != 0;
917                 break;
918 #ifdef CONFIG_COMPAT
919         case CAP_COMPAT_HWCAP:
920                 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
921                 break;
922         case CAP_COMPAT_HWCAP2:
923                 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
924                 break;
925 #endif
926         default:
927                 WARN_ON(1);
928                 rc = false;
929         }
930 
931         return rc;
932 }
933 
934 static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
935 {
936         for (; hwcaps->matches; hwcaps++)
937                 if (hwcaps->matches(hwcaps, hwcaps->def_scope))
938                         cap_set_elf_hwcap(hwcaps);
939 }
940 
941 void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
942                             const char *info)
943 {
944         for (; caps->matches; caps++) {
945                 if (!caps->matches(caps, caps->def_scope))
946                         continue;
947 
948                 if (!cpus_have_cap(caps->capability) && caps->desc)
949                         pr_info("%s %s\n", info, caps->desc);
950                 cpus_set_cap(caps->capability);
951         }
952 }
953 
954 /*
955  * Run through the enabled capabilities and enable() it on all active
956  * CPUs
957  */
958 void __init enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps)
959 {
960         for (; caps->matches; caps++)
961                 if (caps->enable && cpus_have_cap(caps->capability))
962                         /*
963                          * Use stop_machine() as it schedules the work allowing
964                          * us to modify PSTATE, instead of on_each_cpu() which
965                          * uses an IPI, giving us a PSTATE that disappears when
966                          * we return.
967                          */
968                         stop_machine(caps->enable, NULL, cpu_online_mask);
969 }
970 
971 /*
972  * Flag to indicate if we have computed the system wide
973  * capabilities based on the boot time active CPUs. This
974  * will be used to determine if a new booting CPU should
975  * go through the verification process to make sure that it
976  * supports the system capabilities, without using a hotplug
977  * notifier.
978  */
979 static bool sys_caps_initialised;
980 
981 static inline void set_sys_caps_initialised(void)
982 {
983         sys_caps_initialised = true;
984 }
985 
986 /*
987  * Check for CPU features that are used in early boot
988  * based on the Boot CPU value.
989  */
990 static void check_early_cpu_features(void)
991 {
992         verify_cpu_run_el();
993         verify_cpu_asid_bits();
994 }
995 
996 static void
997 verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
998 {
999 
1000         for (; caps->matches; caps++)
1001                 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
1002                         pr_crit("CPU%d: missing HWCAP: %s\n",
1003                                         smp_processor_id(), caps->desc);
1004                         cpu_die_early();
1005                 }
1006 }
1007 
1008 static void
1009 verify_local_cpu_features(const struct arm64_cpu_capabilities *caps)
1010 {
1011         for (; caps->matches; caps++) {
1012                 if (!cpus_have_cap(caps->capability))
1013                         continue;
1014                 /*
1015                  * If the new CPU misses an advertised feature, we cannot proceed
1016                  * further, park the cpu.
1017                  */
1018                 if (!caps->matches(caps, SCOPE_LOCAL_CPU)) {
1019                         pr_crit("CPU%d: missing feature: %s\n",
1020                                         smp_processor_id(), caps->desc);
1021                         cpu_die_early();
1022                 }
1023                 if (caps->enable)
1024                         caps->enable(NULL);
1025         }
1026 }
1027 
1028 /*
1029  * Run through the enabled system capabilities and enable() it on this CPU.
1030  * The capabilities were decided based on the available CPUs at the boot time.
1031  * Any new CPU should match the system wide status of the capability. If the
1032  * new CPU doesn't have a capability which the system now has enabled, we
1033  * cannot do anything to fix it up and could cause unexpected failures. So
1034  * we park the CPU.
1035  */
1036 static void verify_local_cpu_capabilities(void)
1037 {
1038         verify_local_cpu_errata_workarounds();
1039         verify_local_cpu_features(arm64_features);
1040         verify_local_elf_hwcaps(arm64_elf_hwcaps);
1041         if (system_supports_32bit_el0())
1042                 verify_local_elf_hwcaps(compat_elf_hwcaps);
1043 }
1044 
1045 void check_local_cpu_capabilities(void)
1046 {
1047         /*
1048          * All secondary CPUs should conform to the early CPU features
1049          * in use by the kernel based on boot CPU.
1050          */
1051         check_early_cpu_features();
1052 
1053         /*
1054          * If we haven't finalised the system capabilities, this CPU gets
1055          * a chance to update the errata work arounds.
1056          * Otherwise, this CPU should verify that it has all the system
1057          * advertised capabilities.
1058          */
1059         if (!sys_caps_initialised)
1060                 update_cpu_errata_workarounds();
1061         else
1062                 verify_local_cpu_capabilities();
1063 }
1064 
1065 static void __init setup_feature_capabilities(void)
1066 {
1067         update_cpu_capabilities(arm64_features, "detected feature:");
1068         enable_cpu_capabilities(arm64_features);
1069 }
1070 
1071 /*
1072  * Check if the current CPU has a given feature capability.
1073  * Should be called from non-preemptible context.
1074  */
1075 bool this_cpu_has_cap(unsigned int cap)
1076 {
1077         const struct arm64_cpu_capabilities *caps;
1078 
1079         if (WARN_ON(preemptible()))
1080                 return false;
1081 
1082         for (caps = arm64_features; caps->desc; caps++)
1083                 if (caps->capability == cap && caps->matches)
1084                         return caps->matches(caps, SCOPE_LOCAL_CPU);
1085 
1086         return false;
1087 }
1088 
1089 void __init setup_cpu_features(void)
1090 {
1091         u32 cwg;
1092         int cls;
1093 
1094         /* Set the CPU feature capabilies */
1095         setup_feature_capabilities();
1096         enable_errata_workarounds();
1097         setup_elf_hwcaps(arm64_elf_hwcaps);
1098 
1099         if (system_supports_32bit_el0())
1100                 setup_elf_hwcaps(compat_elf_hwcaps);
1101 
1102         /* Advertise that we have computed the system capabilities */
1103         set_sys_caps_initialised();
1104 
1105         /*
1106          * Check for sane CTR_EL0.CWG value.
1107          */
1108         cwg = cache_type_cwg();
1109         cls = cache_line_size();
1110         if (!cwg)
1111                 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
1112                         cls);
1113         if (L1_CACHE_BYTES < cls)
1114                 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
1115                         L1_CACHE_BYTES, cls);
1116 }
1117 
1118 static bool __maybe_unused
1119 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
1120 {
1121         return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
1122 }
1123 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp