~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/kernel/cpufeature.c

Version: ~ [ linux-6.0-rc6 ] ~ [ linux-5.19.10 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.69 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.144 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.214 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.259 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.294 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.329 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Contains CPU feature definitions
  3  *
  4  * Copyright (C) 2015 ARM Ltd.
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  *
 10  * This program is distributed in the hope that it will be useful,
 11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 13  * GNU General Public License for more details.
 14  *
 15  * You should have received a copy of the GNU General Public License
 16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 17  */
 18 
 19 #define pr_fmt(fmt) "CPU features: " fmt
 20 
 21 #include <linux/bsearch.h>
 22 #include <linux/cpumask.h>
 23 #include <linux/crash_dump.h>
 24 #include <linux/sort.h>
 25 #include <linux/stop_machine.h>
 26 #include <linux/types.h>
 27 #include <linux/mm.h>
 28 #include <asm/cpu.h>
 29 #include <asm/cpufeature.h>
 30 #include <asm/cpu_ops.h>
 31 #include <asm/fpsimd.h>
 32 #include <asm/mmu_context.h>
 33 #include <asm/processor.h>
 34 #include <asm/sysreg.h>
 35 #include <asm/traps.h>
 36 #include <asm/virt.h>
 37 
 38 unsigned long elf_hwcap __read_mostly;
 39 EXPORT_SYMBOL_GPL(elf_hwcap);
 40 
 41 #ifdef CONFIG_COMPAT
 42 #define COMPAT_ELF_HWCAP_DEFAULT        \
 43                                 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
 44                                  COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
 45                                  COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
 46                                  COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
 47                                  COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
 48                                  COMPAT_HWCAP_LPAE)
 49 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
 50 unsigned int compat_elf_hwcap2 __read_mostly;
 51 #endif
 52 
 53 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 54 EXPORT_SYMBOL(cpu_hwcaps);
 55 static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS];
 56 
 57 /* Need also bit for ARM64_CB_PATCH */
 58 DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
 59 
 60 /*
 61  * Flag to indicate if we have computed the system wide
 62  * capabilities based on the boot time active CPUs. This
 63  * will be used to determine if a new booting CPU should
 64  * go through the verification process to make sure that it
 65  * supports the system capabilities, without using a hotplug
 66  * notifier.
 67  */
 68 static bool sys_caps_initialised;
 69 
 70 static inline void set_sys_caps_initialised(void)
 71 {
 72         sys_caps_initialised = true;
 73 }
 74 
 75 static int dump_cpu_hwcaps(struct notifier_block *self, unsigned long v, void *p)
 76 {
 77         /* file-wide pr_fmt adds "CPU features: " prefix */
 78         pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps);
 79         return 0;
 80 }
 81 
 82 static struct notifier_block cpu_hwcaps_notifier = {
 83         .notifier_call = dump_cpu_hwcaps
 84 };
 85 
 86 static int __init register_cpu_hwcaps_dumper(void)
 87 {
 88         atomic_notifier_chain_register(&panic_notifier_list,
 89                                        &cpu_hwcaps_notifier);
 90         return 0;
 91 }
 92 __initcall(register_cpu_hwcaps_dumper);
 93 
 94 DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
 95 EXPORT_SYMBOL(cpu_hwcap_keys);
 96 
 97 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
 98         {                                               \
 99                 .sign = SIGNED,                         \
100                 .visible = VISIBLE,                     \
101                 .strict = STRICT,                       \
102                 .type = TYPE,                           \
103                 .shift = SHIFT,                         \
104                 .width = WIDTH,                         \
105                 .safe_val = SAFE_VAL,                   \
106         }
107 
108 /* Define a feature with unsigned values */
109 #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
110         __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
111 
112 /* Define a feature with a signed value */
113 #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
114         __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
115 
116 #define ARM64_FTR_END                                   \
117         {                                               \
118                 .width = 0,                             \
119         }
120 
121 /* meta feature for alternatives */
122 static bool __maybe_unused
123 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
124 
125 static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
126 
127 /*
128  * NOTE: Any changes to the visibility of features should be kept in
129  * sync with the documentation of the CPU feature register ABI.
130  */
131 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
132         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_TS_SHIFT, 4, 0),
133         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_FHM_SHIFT, 4, 0),
134         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_DP_SHIFT, 4, 0),
135         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM4_SHIFT, 4, 0),
136         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SM3_SHIFT, 4, 0),
137         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA3_SHIFT, 4, 0),
138         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_RDM_SHIFT, 4, 0),
139         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_ATOMICS_SHIFT, 4, 0),
140         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_CRC32_SHIFT, 4, 0),
141         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA2_SHIFT, 4, 0),
142         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_SHA1_SHIFT, 4, 0),
143         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_AES_SHIFT, 4, 0),
144         ARM64_FTR_END,
145 };
146 
147 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
148         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SB_SHIFT, 4, 0),
149         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
150                        FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPI_SHIFT, 4, 0),
151         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
152                        FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPA_SHIFT, 4, 0),
153         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
154         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
155         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
156         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
157                        FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_API_SHIFT, 4, 0),
158         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
159                        FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_APA_SHIFT, 4, 0),
160         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
161         ARM64_FTR_END,
162 };
163 
164 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
165         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
166         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
167         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0),
168         ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
169                                    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
170         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
171         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
172         S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
173         S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
174         /* Linux doesn't care about the EL3 */
175         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
176         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
177         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_EL1_64BIT_ONLY),
178         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_EL0_64BIT_ONLY),
179         ARM64_FTR_END,
180 };
181 
182 static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
183         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
184         ARM64_FTR_END,
185 };
186 
187 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
188         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
189         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
190         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
191         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
192         /* Linux shouldn't care about secure memory */
193         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
194         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
195         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
196         /*
197          * Differing PARange is fine as long as all peripherals and memory are mapped
198          * within the minimum PARange of all CPUs
199          */
200         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
201         ARM64_FTR_END,
202 };
203 
204 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
205         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
206         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
207         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
208         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
209         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
210         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
211         ARM64_FTR_END,
212 };
213 
214 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
215         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0),
216         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0),
217         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
218         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
219         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
220         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
221         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
222         ARM64_FTR_END,
223 };
224 
225 static const struct arm64_ftr_bits ftr_ctr[] = {
226         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
227         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DIC_SHIFT, 1, 1),
228         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IDC_SHIFT, 1, 1),
229         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_CWG_SHIFT, 4, 0),
230         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_SAFE, CTR_ERG_SHIFT, 4, 0),
231         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_DMINLINE_SHIFT, 4, 1),
232         /*
233          * Linux can handle differing I-cache policies. Userspace JITs will
234          * make use of *minLine.
235          * If we have differing I-cache policies, report it as the weakest - VIPT.
236          */
237         ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT),       /* L1Ip */
238         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IMINLINE_SHIFT, 4, 0),
239         ARM64_FTR_END,
240 };
241 
242 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
243         .name           = "SYS_CTR_EL0",
244         .ftr_bits       = ftr_ctr
245 };
246 
247 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
248         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0xf),   /* InnerShr */
249         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),       /* FCSE */
250         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, 20, 4, 0),    /* AuxReg */
251         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),       /* TCM */
252         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),       /* ShareLvl */
253         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0xf),    /* OuterShr */
254         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),        /* PMSA */
255         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),        /* VMSA */
256         ARM64_FTR_END,
257 };
258 
259 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
260         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 36, 28, 0),
261         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0),
262         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
263         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
264         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
265         /*
266          * We can instantiate multiple PMU instances with different levels
267          * of support.
268          */
269         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
270         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_TRACEVER_SHIFT, 4, 0),
271         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
272         ARM64_FTR_END,
273 };
274 
275 static const struct arm64_ftr_bits ftr_mvfr2[] = {
276         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),                /* FPMisc */
277         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),                /* SIMDMisc */
278         ARM64_FTR_END,
279 };
280 
281 static const struct arm64_ftr_bits ftr_dczid[] = {
282         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 4, 1, 1),            /* DZP */
283         ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),       /* BS */
284         ARM64_FTR_END,
285 };
286 
287 
288 static const struct arm64_ftr_bits ftr_id_isar5[] = {
289         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_RDM_SHIFT, 4, 0),
290         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_CRC32_SHIFT, 4, 0),
291         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA2_SHIFT, 4, 0),
292         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SHA1_SHIFT, 4, 0),
293         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_AES_SHIFT, 4, 0),
294         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_SEVL_SHIFT, 4, 0),
295         ARM64_FTR_END,
296 };
297 
298 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
299         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),        /* ac2 */
300         ARM64_FTR_END,
301 };
302 
303 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
304         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),               /* State3 */
305         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),                /* State2 */
306         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),                /* State1 */
307         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),                /* State0 */
308         ARM64_FTR_END,
309 };
310 
311 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
312         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
313         S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0xf),   /* PerfMon */
314         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
315         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
316         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
317         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
318         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
319         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
320         ARM64_FTR_END,
321 };
322 
323 static const struct arm64_ftr_bits ftr_zcr[] = {
324         ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE,
325                 ZCR_ELx_LEN_SHIFT, ZCR_ELx_LEN_SIZE, 0),        /* LEN */
326         ARM64_FTR_END,
327 };
328 
329 /*
330  * Common ftr bits for a 32bit register with all hidden, strict
331  * attributes, with 4bit feature fields and a default safe value of
332  * 0. Covers the following 32bit registers:
333  * id_isar[0-4], id_mmfr[1-3], id_pfr1, mvfr[0-1]
334  */
335 static const struct arm64_ftr_bits ftr_generic_32bits[] = {
336         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
337         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
338         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
339         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
340         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
341         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
342         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
343         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
344         ARM64_FTR_END,
345 };
346 
347 /* Table for a single 32bit feature value */
348 static const struct arm64_ftr_bits ftr_single32[] = {
349         ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
350         ARM64_FTR_END,
351 };
352 
353 static const struct arm64_ftr_bits ftr_raz[] = {
354         ARM64_FTR_END,
355 };
356 
357 #define ARM64_FTR_REG(id, table) {              \
358         .sys_id = id,                           \
359         .reg =  &(struct arm64_ftr_reg){        \
360                 .name = #id,                    \
361                 .ftr_bits = &((table)[0]),      \
362         }}
363 
364 static const struct __ftr_reg_entry {
365         u32                     sys_id;
366         struct arm64_ftr_reg    *reg;
367 } arm64_ftr_regs[] = {
368 
369         /* Op1 = 0, CRn = 0, CRm = 1 */
370         ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
371         ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_generic_32bits),
372         ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
373         ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
374         ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
375         ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
376         ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
377 
378         /* Op1 = 0, CRn = 0, CRm = 2 */
379         ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_generic_32bits),
380         ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
381         ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
382         ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
383         ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_generic_32bits),
384         ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
385         ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
386 
387         /* Op1 = 0, CRn = 0, CRm = 3 */
388         ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_generic_32bits),
389         ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_generic_32bits),
390         ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
391 
392         /* Op1 = 0, CRn = 0, CRm = 4 */
393         ARM64_FTR_REG(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0),
394         ARM64_FTR_REG(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1),
395         ARM64_FTR_REG(SYS_ID_AA64ZFR0_EL1, ftr_raz),
396 
397         /* Op1 = 0, CRn = 0, CRm = 5 */
398         ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
399         ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
400 
401         /* Op1 = 0, CRn = 0, CRm = 6 */
402         ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
403         ARM64_FTR_REG(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1),
404 
405         /* Op1 = 0, CRn = 0, CRm = 7 */
406         ARM64_FTR_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0),
407         ARM64_FTR_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1),
408         ARM64_FTR_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2),
409 
410         /* Op1 = 0, CRn = 1, CRm = 2 */
411         ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
412 
413         /* Op1 = 3, CRn = 0, CRm = 0 */
414         { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
415         ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
416 
417         /* Op1 = 3, CRn = 14, CRm = 0 */
418         ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
419 };
420 
421 static int search_cmp_ftr_reg(const void *id, const void *regp)
422 {
423         return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
424 }
425 
426 /*
427  * get_arm64_ftr_reg - Lookup a feature register entry using its
428  * sys_reg() encoding. With the array arm64_ftr_regs sorted in the
429  * ascending order of sys_id , we use binary search to find a matching
430  * entry.
431  *
432  * returns - Upon success,  matching ftr_reg entry for id.
433  *         - NULL on failure. It is upto the caller to decide
434  *           the impact of a failure.
435  */
436 static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
437 {
438         const struct __ftr_reg_entry *ret;
439 
440         ret = bsearch((const void *)(unsigned long)sys_id,
441                         arm64_ftr_regs,
442                         ARRAY_SIZE(arm64_ftr_regs),
443                         sizeof(arm64_ftr_regs[0]),
444                         search_cmp_ftr_reg);
445         if (ret)
446                 return ret->reg;
447         return NULL;
448 }
449 
450 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
451                                s64 ftr_val)
452 {
453         u64 mask = arm64_ftr_mask(ftrp);
454 
455         reg &= ~mask;
456         reg |= (ftr_val << ftrp->shift) & mask;
457         return reg;
458 }
459 
460 static s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
461                                 s64 cur)
462 {
463         s64 ret = 0;
464 
465         switch (ftrp->type) {
466         case FTR_EXACT:
467                 ret = ftrp->safe_val;
468                 break;
469         case FTR_LOWER_SAFE:
470                 ret = new < cur ? new : cur;
471                 break;
472         case FTR_HIGHER_SAFE:
473                 ret = new > cur ? new : cur;
474                 break;
475         default:
476                 BUG();
477         }
478 
479         return ret;
480 }
481 
482 static void __init sort_ftr_regs(void)
483 {
484         int i;
485 
486         /* Check that the array is sorted so that we can do the binary search */
487         for (i = 1; i < ARRAY_SIZE(arm64_ftr_regs); i++)
488                 BUG_ON(arm64_ftr_regs[i].sys_id < arm64_ftr_regs[i - 1].sys_id);
489 }
490 
491 /*
492  * Initialise the CPU feature register from Boot CPU values.
493  * Also initiliases the strict_mask for the register.
494  * Any bits that are not covered by an arm64_ftr_bits entry are considered
495  * RES0 for the system-wide value, and must strictly match.
496  */
497 static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
498 {
499         u64 val = 0;
500         u64 strict_mask = ~0x0ULL;
501         u64 user_mask = 0;
502         u64 valid_mask = 0;
503 
504         const struct arm64_ftr_bits *ftrp;
505         struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
506 
507         BUG_ON(!reg);
508 
509         for (ftrp  = reg->ftr_bits; ftrp->width; ftrp++) {
510                 u64 ftr_mask = arm64_ftr_mask(ftrp);
511                 s64 ftr_new = arm64_ftr_value(ftrp, new);
512 
513                 val = arm64_ftr_set_value(ftrp, val, ftr_new);
514 
515                 valid_mask |= ftr_mask;
516                 if (!ftrp->strict)
517                         strict_mask &= ~ftr_mask;
518                 if (ftrp->visible)
519                         user_mask |= ftr_mask;
520                 else
521                         reg->user_val = arm64_ftr_set_value(ftrp,
522                                                             reg->user_val,
523                                                             ftrp->safe_val);
524         }
525 
526         val &= valid_mask;
527 
528         reg->sys_val = val;
529         reg->strict_mask = strict_mask;
530         reg->user_mask = user_mask;
531 }
532 
533 extern const struct arm64_cpu_capabilities arm64_errata[];
534 static const struct arm64_cpu_capabilities arm64_features[];
535 
536 static void __init
537 init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
538 {
539         for (; caps->matches; caps++) {
540                 if (WARN(caps->capability >= ARM64_NCAPS,
541                         "Invalid capability %d\n", caps->capability))
542                         continue;
543                 if (WARN(cpu_hwcaps_ptrs[caps->capability],
544                         "Duplicate entry for capability %d\n",
545                         caps->capability))
546                         continue;
547                 cpu_hwcaps_ptrs[caps->capability] = caps;
548         }
549 }
550 
551 static void __init init_cpu_hwcaps_indirect_list(void)
552 {
553         init_cpu_hwcaps_indirect_list_from_array(arm64_features);
554         init_cpu_hwcaps_indirect_list_from_array(arm64_errata);
555 }
556 
557 static void __init setup_boot_cpu_capabilities(void);
558 
559 void __init init_cpu_features(struct cpuinfo_arm64 *info)
560 {
561         /* Before we start using the tables, make sure it is sorted */
562         sort_ftr_regs();
563 
564         init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
565         init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
566         init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
567         init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
568         init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
569         init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
570         init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
571         init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
572         init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
573         init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
574         init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
575         init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
576         init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
577 
578         if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
579                 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
580                 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
581                 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
582                 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
583                 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
584                 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
585                 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
586                 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
587                 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
588                 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
589                 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
590                 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
591                 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
592                 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
593                 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
594                 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
595         }
596 
597         if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
598                 init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
599                 sve_init_vq_map();
600         }
601 
602         /*
603          * Initialize the indirect array of CPU hwcaps capabilities pointers
604          * before we handle the boot CPU below.
605          */
606         init_cpu_hwcaps_indirect_list();
607 
608         /*
609          * Detect and enable early CPU capabilities based on the boot CPU,
610          * after we have initialised the CPU feature infrastructure.
611          */
612         setup_boot_cpu_capabilities();
613 }
614 
615 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
616 {
617         const struct arm64_ftr_bits *ftrp;
618 
619         for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
620                 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
621                 s64 ftr_new = arm64_ftr_value(ftrp, new);
622 
623                 if (ftr_cur == ftr_new)
624                         continue;
625                 /* Find a safe value */
626                 ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
627                 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
628         }
629 
630 }
631 
632 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
633 {
634         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
635 
636         BUG_ON(!regp);
637         update_cpu_ftr_reg(regp, val);
638         if ((boot & regp->strict_mask) == (val & regp->strict_mask))
639                 return 0;
640         pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
641                         regp->name, boot, cpu, val);
642         return 1;
643 }
644 
645 /*
646  * Update system wide CPU feature registers with the values from a
647  * non-boot CPU. Also performs SANITY checks to make sure that there
648  * aren't any insane variations from that of the boot CPU.
649  */
650 void update_cpu_features(int cpu,
651                          struct cpuinfo_arm64 *info,
652                          struct cpuinfo_arm64 *boot)
653 {
654         int taint = 0;
655 
656         /*
657          * The kernel can handle differing I-cache policies, but otherwise
658          * caches should look identical. Userspace JITs will make use of
659          * *minLine.
660          */
661         taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
662                                       info->reg_ctr, boot->reg_ctr);
663 
664         /*
665          * Userspace may perform DC ZVA instructions. Mismatched block sizes
666          * could result in too much or too little memory being zeroed if a
667          * process is preempted and migrated between CPUs.
668          */
669         taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
670                                       info->reg_dczid, boot->reg_dczid);
671 
672         /* If different, timekeeping will be broken (especially with KVM) */
673         taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
674                                       info->reg_cntfrq, boot->reg_cntfrq);
675 
676         /*
677          * The kernel uses self-hosted debug features and expects CPUs to
678          * support identical debug features. We presently need CTX_CMPs, WRPs,
679          * and BRPs to be identical.
680          * ID_AA64DFR1 is currently RES0.
681          */
682         taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
683                                       info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
684         taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
685                                       info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
686         /*
687          * Even in big.LITTLE, processors should be identical instruction-set
688          * wise.
689          */
690         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
691                                       info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
692         taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
693                                       info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
694 
695         /*
696          * Differing PARange support is fine as long as all peripherals and
697          * memory are mapped within the minimum PARange of all CPUs.
698          * Linux should not care about secure memory.
699          */
700         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
701                                       info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
702         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
703                                       info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
704         taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
705                                       info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
706 
707         /*
708          * EL3 is not our concern.
709          */
710         taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
711                                       info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
712         taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
713                                       info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
714 
715         taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
716                                       info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
717 
718         /*
719          * If we have AArch32, we care about 32-bit features for compat.
720          * If the system doesn't support AArch32, don't update them.
721          */
722         if (id_aa64pfr0_32bit_el0(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
723                 id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
724 
725                 taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
726                                         info->reg_id_dfr0, boot->reg_id_dfr0);
727                 taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
728                                         info->reg_id_isar0, boot->reg_id_isar0);
729                 taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
730                                         info->reg_id_isar1, boot->reg_id_isar1);
731                 taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
732                                         info->reg_id_isar2, boot->reg_id_isar2);
733                 taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
734                                         info->reg_id_isar3, boot->reg_id_isar3);
735                 taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
736                                         info->reg_id_isar4, boot->reg_id_isar4);
737                 taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
738                                         info->reg_id_isar5, boot->reg_id_isar5);
739 
740                 /*
741                  * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
742                  * ACTLR formats could differ across CPUs and therefore would have to
743                  * be trapped for virtualization anyway.
744                  */
745                 taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
746                                         info->reg_id_mmfr0, boot->reg_id_mmfr0);
747                 taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
748                                         info->reg_id_mmfr1, boot->reg_id_mmfr1);
749                 taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
750                                         info->reg_id_mmfr2, boot->reg_id_mmfr2);
751                 taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
752                                         info->reg_id_mmfr3, boot->reg_id_mmfr3);
753                 taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
754                                         info->reg_id_pfr0, boot->reg_id_pfr0);
755                 taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
756                                         info->reg_id_pfr1, boot->reg_id_pfr1);
757                 taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
758                                         info->reg_mvfr0, boot->reg_mvfr0);
759                 taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
760                                         info->reg_mvfr1, boot->reg_mvfr1);
761                 taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
762                                         info->reg_mvfr2, boot->reg_mvfr2);
763         }
764 
765         if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
766                 taint |= check_update_ftr_reg(SYS_ZCR_EL1, cpu,
767                                         info->reg_zcr, boot->reg_zcr);
768 
769                 /* Probe vector lengths, unless we already gave up on SVE */
770                 if (id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1)) &&
771                     !sys_caps_initialised)
772                         sve_update_vq_map();
773         }
774 
775         /*
776          * Mismatched CPU features are a recipe for disaster. Don't even
777          * pretend to support them.
778          */
779         if (taint) {
780                 pr_warn_once("Unsupported CPU feature variation detected.\n");
781                 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
782         }
783 }
784 
785 u64 read_sanitised_ftr_reg(u32 id)
786 {
787         struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
788 
789         /* We shouldn't get a request for an unsupported register */
790         BUG_ON(!regp);
791         return regp->sys_val;
792 }
793 
794 #define read_sysreg_case(r)     \
795         case r:         return read_sysreg_s(r)
796 
797 /*
798  * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
799  * Read the system register on the current CPU
800  */
801 static u64 __read_sysreg_by_encoding(u32 sys_id)
802 {
803         switch (sys_id) {
804         read_sysreg_case(SYS_ID_PFR0_EL1);
805         read_sysreg_case(SYS_ID_PFR1_EL1);
806         read_sysreg_case(SYS_ID_DFR0_EL1);
807         read_sysreg_case(SYS_ID_MMFR0_EL1);
808         read_sysreg_case(SYS_ID_MMFR1_EL1);
809         read_sysreg_case(SYS_ID_MMFR2_EL1);
810         read_sysreg_case(SYS_ID_MMFR3_EL1);
811         read_sysreg_case(SYS_ID_ISAR0_EL1);
812         read_sysreg_case(SYS_ID_ISAR1_EL1);
813         read_sysreg_case(SYS_ID_ISAR2_EL1);
814         read_sysreg_case(SYS_ID_ISAR3_EL1);
815         read_sysreg_case(SYS_ID_ISAR4_EL1);
816         read_sysreg_case(SYS_ID_ISAR5_EL1);
817         read_sysreg_case(SYS_MVFR0_EL1);
818         read_sysreg_case(SYS_MVFR1_EL1);
819         read_sysreg_case(SYS_MVFR2_EL1);
820 
821         read_sysreg_case(SYS_ID_AA64PFR0_EL1);
822         read_sysreg_case(SYS_ID_AA64PFR1_EL1);
823         read_sysreg_case(SYS_ID_AA64DFR0_EL1);
824         read_sysreg_case(SYS_ID_AA64DFR1_EL1);
825         read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
826         read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
827         read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
828         read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
829         read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
830 
831         read_sysreg_case(SYS_CNTFRQ_EL0);
832         read_sysreg_case(SYS_CTR_EL0);
833         read_sysreg_case(SYS_DCZID_EL0);
834 
835         default:
836                 BUG();
837                 return 0;
838         }
839 }
840 
841 #include <linux/irqchip/arm-gic-v3.h>
842 
843 static bool
844 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
845 {
846         int val = cpuid_feature_extract_field(reg, entry->field_pos, entry->sign);
847 
848         return val >= entry->min_field_value;
849 }
850 
851 static bool
852 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
853 {
854         u64 val;
855 
856         WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
857         if (scope == SCOPE_SYSTEM)
858                 val = read_sanitised_ftr_reg(entry->sys_reg);
859         else
860                 val = __read_sysreg_by_encoding(entry->sys_reg);
861 
862         return feature_matches(val, entry);
863 }
864 
865 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
866 {
867         bool has_sre;
868 
869         if (!has_cpuid_feature(entry, scope))
870                 return false;
871 
872         has_sre = gic_enable_sre();
873         if (!has_sre)
874                 pr_warn_once("%s present but disabled by higher exception level\n",
875                              entry->desc);
876 
877         return has_sre;
878 }
879 
880 static bool has_no_hw_prefetch(const struct arm64_cpu_capabilities *entry, int __unused)
881 {
882         u32 midr = read_cpuid_id();
883 
884         /* Cavium ThunderX pass 1.x and 2.x */
885         return MIDR_IS_CPU_MODEL_RANGE(midr, MIDR_THUNDERX,
886                 MIDR_CPU_VAR_REV(0, 0),
887                 MIDR_CPU_VAR_REV(1, MIDR_REVISION_MASK));
888 }
889 
890 static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unused)
891 {
892         u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
893 
894         return cpuid_feature_extract_signed_field(pfr0,
895                                         ID_AA64PFR0_FP_SHIFT) < 0;
896 }
897 
898 static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
899                           int scope)
900 {
901         u64 ctr;
902 
903         if (scope == SCOPE_SYSTEM)
904                 ctr = arm64_ftr_reg_ctrel0.sys_val;
905         else
906                 ctr = read_cpuid_effective_cachetype();
907 
908         return ctr & BIT(CTR_IDC_SHIFT);
909 }
910 
911 static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
912 {
913         /*
914          * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
915          * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
916          * to the CTR_EL0 on this CPU and emulate it with the real/safe
917          * value.
918          */
919         if (!(read_cpuid_cachetype() & BIT(CTR_IDC_SHIFT)))
920                 sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
921 }
922 
923 static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
924                           int scope)
925 {
926         u64 ctr;
927 
928         if (scope == SCOPE_SYSTEM)
929                 ctr = arm64_ftr_reg_ctrel0.sys_val;
930         else
931                 ctr = read_cpuid_cachetype();
932 
933         return ctr & BIT(CTR_DIC_SHIFT);
934 }
935 
936 static bool __maybe_unused
937 has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
938 {
939         /*
940          * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
941          * may share TLB entries with a CPU stuck in the crashed
942          * kernel.
943          */
944          if (is_kdump_kernel())
945                 return false;
946 
947         return has_cpuid_feature(entry, scope);
948 }
949 
950 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
951 static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
952 
953 static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
954                                 int scope)
955 {
956         /* List of CPUs that are not vulnerable and don't need KPTI */
957         static const struct midr_range kpti_safe_list[] = {
958                 MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
959                 MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
960                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
961                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
962                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
963                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
964                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
965                 MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
966                 MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
967                 { /* sentinel */ }
968         };
969         char const *str = "command line option";
970 
971         /*
972          * For reasons that aren't entirely clear, enabling KPTI on Cavium
973          * ThunderX leads to apparent I-cache corruption of kernel text, which
974          * ends as well as you might imagine. Don't even try.
975          */
976         if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
977                 str = "ARM64_WORKAROUND_CAVIUM_27456";
978                 __kpti_forced = -1;
979         }
980 
981         /* Forced? */
982         if (__kpti_forced) {
983                 pr_info_once("kernel page table isolation forced %s by %s\n",
984                              __kpti_forced > 0 ? "ON" : "OFF", str);
985                 return __kpti_forced > 0;
986         }
987 
988         /* Useful for KASLR robustness */
989         if (IS_ENABLED(CONFIG_RANDOMIZE_BASE))
990                 return kaslr_offset() > 0;
991 
992         /* Don't force KPTI for CPUs that are not vulnerable */
993         if (is_midr_in_range_list(read_cpuid_id(), kpti_safe_list))
994                 return false;
995 
996         /* Defer to CPU feature registers */
997         return !has_cpuid_feature(entry, scope);
998 }
999 
1000 static void
1001 kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
1002 {
1003         typedef void (kpti_remap_fn)(int, int, phys_addr_t);
1004         extern kpti_remap_fn idmap_kpti_install_ng_mappings;
1005         kpti_remap_fn *remap_fn;
1006 
1007         static bool kpti_applied = false;
1008         int cpu = smp_processor_id();
1009 
1010         /*
1011          * We don't need to rewrite the page-tables if either we've done
1012          * it already or we have KASLR enabled and therefore have not
1013          * created any global mappings at all.
1014          */
1015         if (kpti_applied || kaslr_offset() > 0)
1016                 return;
1017 
1018         remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
1019 
1020         cpu_install_idmap();
1021         remap_fn(cpu, num_online_cpus(), __pa_symbol(swapper_pg_dir));
1022         cpu_uninstall_idmap();
1023 
1024         if (!cpu)
1025                 kpti_applied = true;
1026 
1027         return;
1028 }
1029 
1030 static int __init parse_kpti(char *str)
1031 {
1032         bool enabled;
1033         int ret = strtobool(str, &enabled);
1034 
1035         if (ret)
1036                 return ret;
1037 
1038         __kpti_forced = enabled ? 1 : -1;
1039         return 0;
1040 }
1041 early_param("kpti", parse_kpti);
1042 #endif  /* CONFIG_UNMAP_KERNEL_AT_EL0 */
1043 
1044 #ifdef CONFIG_ARM64_HW_AFDBM
1045 static inline void __cpu_enable_hw_dbm(void)
1046 {
1047         u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
1048 
1049         write_sysreg(tcr, tcr_el1);
1050         isb();
1051 }
1052 
1053 static bool cpu_has_broken_dbm(void)
1054 {
1055         /* List of CPUs which have broken DBM support. */
1056         static const struct midr_range cpus[] = {
1057 #ifdef CONFIG_ARM64_ERRATUM_1024718
1058                 MIDR_RANGE(MIDR_CORTEX_A55, 0, 0, 1, 0),  // A55 r0p0 -r1p0
1059 #endif
1060                 {},
1061         };
1062 
1063         return is_midr_in_range_list(read_cpuid_id(), cpus);
1064 }
1065 
1066 static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
1067 {
1068         return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
1069                !cpu_has_broken_dbm();
1070 }
1071 
1072 static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
1073 {
1074         if (cpu_can_use_dbm(cap))
1075                 __cpu_enable_hw_dbm();
1076 }
1077 
1078 static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
1079                        int __unused)
1080 {
1081         static bool detected = false;
1082         /*
1083          * DBM is a non-conflicting feature. i.e, the kernel can safely
1084          * run a mix of CPUs with and without the feature. So, we
1085          * unconditionally enable the capability to allow any late CPU
1086          * to use the feature. We only enable the control bits on the
1087          * CPU, if it actually supports.
1088          *
1089          * We have to make sure we print the "feature" detection only
1090          * when at least one CPU actually uses it. So check if this CPU
1091          * can actually use it and print the message exactly once.
1092          *
1093          * This is safe as all CPUs (including secondary CPUs - due to the
1094          * LOCAL_CPU scope - and the hotplugged CPUs - via verification)
1095          * goes through the "matches" check exactly once. Also if a CPU
1096          * matches the criteria, it is guaranteed that the CPU will turn
1097          * the DBM on, as the capability is unconditionally enabled.
1098          */
1099         if (!detected && cpu_can_use_dbm(cap)) {
1100                 detected = true;
1101                 pr_info("detected: Hardware dirty bit management\n");
1102         }
1103 
1104         return true;
1105 }
1106 
1107 #endif
1108 
1109 #ifdef CONFIG_ARM64_VHE
1110 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
1111 {
1112         return is_kernel_in_hyp_mode();
1113 }
1114 
1115 static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
1116 {
1117         /*
1118          * Copy register values that aren't redirected by hardware.
1119          *
1120          * Before code patching, we only set tpidr_el1, all CPUs need to copy
1121          * this value to tpidr_el2 before we patch the code. Once we've done
1122          * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
1123          * do anything here.
1124          */
1125         if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
1126                 write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
1127 }
1128 #endif
1129 
1130 static void cpu_has_fwb(const struct arm64_cpu_capabilities *__unused)
1131 {
1132         u64 val = read_sysreg_s(SYS_CLIDR_EL1);
1133 
1134         /* Check that CLIDR_EL1.LOU{U,IS} are both 0 */
1135         WARN_ON(val & (7 << 27 | 7 << 21));
1136 }
1137 
1138 #ifdef CONFIG_ARM64_SSBD
1139 static int ssbs_emulation_handler(struct pt_regs *regs, u32 instr)
1140 {
1141         if (user_mode(regs))
1142                 return 1;
1143 
1144         if (instr & BIT(PSTATE_Imm_shift))
1145                 regs->pstate |= PSR_SSBS_BIT;
1146         else
1147                 regs->pstate &= ~PSR_SSBS_BIT;
1148 
1149         arm64_skip_faulting_instruction(regs, 4);
1150         return 0;
1151 }
1152 
1153 static struct undef_hook ssbs_emulation_hook = {
1154         .instr_mask     = ~(1U << PSTATE_Imm_shift),
1155         .instr_val      = 0xd500401f | PSTATE_SSBS,
1156         .fn             = ssbs_emulation_handler,
1157 };
1158 
1159 static void cpu_enable_ssbs(const struct arm64_cpu_capabilities *__unused)
1160 {
1161         static bool undef_hook_registered = false;
1162         static DEFINE_SPINLOCK(hook_lock);
1163 
1164         spin_lock(&hook_lock);
1165         if (!undef_hook_registered) {
1166                 register_undef_hook(&ssbs_emulation_hook);
1167                 undef_hook_registered = true;
1168         }
1169         spin_unlock(&hook_lock);
1170 
1171         if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE) {
1172                 sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
1173                 arm64_set_ssbd_mitigation(false);
1174         } else {
1175                 arm64_set_ssbd_mitigation(true);
1176         }
1177 }
1178 #endif /* CONFIG_ARM64_SSBD */
1179 
1180 #ifdef CONFIG_ARM64_PAN
1181 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
1182 {
1183         /*
1184          * We modify PSTATE. This won't work from irq context as the PSTATE
1185          * is discarded once we return from the exception.
1186          */
1187         WARN_ON_ONCE(in_interrupt());
1188 
1189         sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
1190         asm(SET_PSTATE_PAN(1));
1191 }
1192 #endif /* CONFIG_ARM64_PAN */
1193 
1194 #ifdef CONFIG_ARM64_RAS_EXTN
1195 static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
1196 {
1197         /* Firmware may have left a deferred SError in this register. */
1198         write_sysreg_s(0, SYS_DISR_EL1);
1199 }
1200 #endif /* CONFIG_ARM64_RAS_EXTN */
1201 
1202 #ifdef CONFIG_ARM64_PTR_AUTH
1203 static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap)
1204 {
1205         sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA | SCTLR_ELx_ENIB |
1206                                        SCTLR_ELx_ENDA | SCTLR_ELx_ENDB);
1207 }
1208 #endif /* CONFIG_ARM64_PTR_AUTH */
1209 
1210 #ifdef CONFIG_ARM64_PSEUDO_NMI
1211 static bool enable_pseudo_nmi;
1212 
1213 static int __init early_enable_pseudo_nmi(char *p)
1214 {
1215         return strtobool(p, &enable_pseudo_nmi);
1216 }
1217 early_param("irqchip.gicv3_pseudo_nmi", early_enable_pseudo_nmi);
1218 
1219 static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
1220                                    int scope)
1221 {
1222         return enable_pseudo_nmi && has_useable_gicv3_cpuif(entry, scope);
1223 }
1224 #endif
1225 
1226 static const struct arm64_cpu_capabilities arm64_features[] = {
1227         {
1228                 .desc = "GIC system register CPU interface",
1229                 .capability = ARM64_HAS_SYSREG_GIC_CPUIF,
1230                 .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
1231                 .matches = has_useable_gicv3_cpuif,
1232                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1233                 .field_pos = ID_AA64PFR0_GIC_SHIFT,
1234                 .sign = FTR_UNSIGNED,
1235                 .min_field_value = 1,
1236         },
1237 #ifdef CONFIG_ARM64_PAN
1238         {
1239                 .desc = "Privileged Access Never",
1240                 .capability = ARM64_HAS_PAN,
1241                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1242                 .matches = has_cpuid_feature,
1243                 .sys_reg = SYS_ID_AA64MMFR1_EL1,
1244                 .field_pos = ID_AA64MMFR1_PAN_SHIFT,
1245                 .sign = FTR_UNSIGNED,
1246                 .min_field_value = 1,
1247                 .cpu_enable = cpu_enable_pan,
1248         },
1249 #endif /* CONFIG_ARM64_PAN */
1250 #if defined(CONFIG_AS_LSE) && defined(CONFIG_ARM64_LSE_ATOMICS)
1251         {
1252                 .desc = "LSE atomic instructions",
1253                 .capability = ARM64_HAS_LSE_ATOMICS,
1254                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1255                 .matches = has_cpuid_feature,
1256                 .sys_reg = SYS_ID_AA64ISAR0_EL1,
1257                 .field_pos = ID_AA64ISAR0_ATOMICS_SHIFT,
1258                 .sign = FTR_UNSIGNED,
1259                 .min_field_value = 2,
1260         },
1261 #endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
1262         {
1263                 .desc = "Software prefetching using PRFM",
1264                 .capability = ARM64_HAS_NO_HW_PREFETCH,
1265                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1266                 .matches = has_no_hw_prefetch,
1267         },
1268 #ifdef CONFIG_ARM64_UAO
1269         {
1270                 .desc = "User Access Override",
1271                 .capability = ARM64_HAS_UAO,
1272                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1273                 .matches = has_cpuid_feature,
1274                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1275                 .field_pos = ID_AA64MMFR2_UAO_SHIFT,
1276                 .min_field_value = 1,
1277                 /*
1278                  * We rely on stop_machine() calling uao_thread_switch() to set
1279                  * UAO immediately after patching.
1280                  */
1281         },
1282 #endif /* CONFIG_ARM64_UAO */
1283 #ifdef CONFIG_ARM64_PAN
1284         {
1285                 .capability = ARM64_ALT_PAN_NOT_UAO,
1286                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1287                 .matches = cpufeature_pan_not_uao,
1288         },
1289 #endif /* CONFIG_ARM64_PAN */
1290 #ifdef CONFIG_ARM64_VHE
1291         {
1292                 .desc = "Virtualization Host Extensions",
1293                 .capability = ARM64_HAS_VIRT_HOST_EXTN,
1294                 .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
1295                 .matches = runs_at_el2,
1296                 .cpu_enable = cpu_copy_el2regs,
1297         },
1298 #endif  /* CONFIG_ARM64_VHE */
1299         {
1300                 .desc = "32-bit EL0 Support",
1301                 .capability = ARM64_HAS_32BIT_EL0,
1302                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1303                 .matches = has_cpuid_feature,
1304                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1305                 .sign = FTR_UNSIGNED,
1306                 .field_pos = ID_AA64PFR0_EL0_SHIFT,
1307                 .min_field_value = ID_AA64PFR0_EL0_32BIT_64BIT,
1308         },
1309 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1310         {
1311                 .desc = "Kernel page table isolation (KPTI)",
1312                 .capability = ARM64_UNMAP_KERNEL_AT_EL0,
1313                 .type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
1314                 /*
1315                  * The ID feature fields below are used to indicate that
1316                  * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
1317                  * more details.
1318                  */
1319                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1320                 .field_pos = ID_AA64PFR0_CSV3_SHIFT,
1321                 .min_field_value = 1,
1322                 .matches = unmap_kernel_at_el0,
1323                 .cpu_enable = kpti_install_ng_mappings,
1324         },
1325 #endif
1326         {
1327                 /* FP/SIMD is not implemented */
1328                 .capability = ARM64_HAS_NO_FPSIMD,
1329                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1330                 .min_field_value = 0,
1331                 .matches = has_no_fpsimd,
1332         },
1333 #ifdef CONFIG_ARM64_PMEM
1334         {
1335                 .desc = "Data cache clean to Point of Persistence",
1336                 .capability = ARM64_HAS_DCPOP,
1337                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1338                 .matches = has_cpuid_feature,
1339                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1340                 .field_pos = ID_AA64ISAR1_DPB_SHIFT,
1341                 .min_field_value = 1,
1342         },
1343 #endif
1344 #ifdef CONFIG_ARM64_SVE
1345         {
1346                 .desc = "Scalable Vector Extension",
1347                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1348                 .capability = ARM64_SVE,
1349                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1350                 .sign = FTR_UNSIGNED,
1351                 .field_pos = ID_AA64PFR0_SVE_SHIFT,
1352                 .min_field_value = ID_AA64PFR0_SVE,
1353                 .matches = has_cpuid_feature,
1354                 .cpu_enable = sve_kernel_enable,
1355         },
1356 #endif /* CONFIG_ARM64_SVE */
1357 #ifdef CONFIG_ARM64_RAS_EXTN
1358         {
1359                 .desc = "RAS Extension Support",
1360                 .capability = ARM64_HAS_RAS_EXTN,
1361                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1362                 .matches = has_cpuid_feature,
1363                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1364                 .sign = FTR_UNSIGNED,
1365                 .field_pos = ID_AA64PFR0_RAS_SHIFT,
1366                 .min_field_value = ID_AA64PFR0_RAS_V1,
1367                 .cpu_enable = cpu_clear_disr,
1368         },
1369 #endif /* CONFIG_ARM64_RAS_EXTN */
1370         {
1371                 .desc = "Data cache clean to the PoU not required for I/D coherence",
1372                 .capability = ARM64_HAS_CACHE_IDC,
1373                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1374                 .matches = has_cache_idc,
1375                 .cpu_enable = cpu_emulate_effective_ctr,
1376         },
1377         {
1378                 .desc = "Instruction cache invalidation not required for I/D coherence",
1379                 .capability = ARM64_HAS_CACHE_DIC,
1380                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1381                 .matches = has_cache_dic,
1382         },
1383         {
1384                 .desc = "Stage-2 Force Write-Back",
1385                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1386                 .capability = ARM64_HAS_STAGE2_FWB,
1387                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1388                 .sign = FTR_UNSIGNED,
1389                 .field_pos = ID_AA64MMFR2_FWB_SHIFT,
1390                 .min_field_value = 1,
1391                 .matches = has_cpuid_feature,
1392                 .cpu_enable = cpu_has_fwb,
1393         },
1394 #ifdef CONFIG_ARM64_HW_AFDBM
1395         {
1396                 /*
1397                  * Since we turn this on always, we don't want the user to
1398                  * think that the feature is available when it may not be.
1399                  * So hide the description.
1400                  *
1401                  * .desc = "Hardware pagetable Dirty Bit Management",
1402                  *
1403                  */
1404                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1405                 .capability = ARM64_HW_DBM,
1406                 .sys_reg = SYS_ID_AA64MMFR1_EL1,
1407                 .sign = FTR_UNSIGNED,
1408                 .field_pos = ID_AA64MMFR1_HADBS_SHIFT,
1409                 .min_field_value = 2,
1410                 .matches = has_hw_dbm,
1411                 .cpu_enable = cpu_enable_hw_dbm,
1412         },
1413 #endif
1414         {
1415                 .desc = "CRC32 instructions",
1416                 .capability = ARM64_HAS_CRC32,
1417                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1418                 .matches = has_cpuid_feature,
1419                 .sys_reg = SYS_ID_AA64ISAR0_EL1,
1420                 .field_pos = ID_AA64ISAR0_CRC32_SHIFT,
1421                 .min_field_value = 1,
1422         },
1423 #ifdef CONFIG_ARM64_SSBD
1424         {
1425                 .desc = "Speculative Store Bypassing Safe (SSBS)",
1426                 .capability = ARM64_SSBS,
1427                 .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1428                 .matches = has_cpuid_feature,
1429                 .sys_reg = SYS_ID_AA64PFR1_EL1,
1430                 .field_pos = ID_AA64PFR1_SSBS_SHIFT,
1431                 .sign = FTR_UNSIGNED,
1432                 .min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
1433                 .cpu_enable = cpu_enable_ssbs,
1434         },
1435 #endif
1436 #ifdef CONFIG_ARM64_CNP
1437         {
1438                 .desc = "Common not Private translations",
1439                 .capability = ARM64_HAS_CNP,
1440                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1441                 .matches = has_useable_cnp,
1442                 .sys_reg = SYS_ID_AA64MMFR2_EL1,
1443                 .sign = FTR_UNSIGNED,
1444                 .field_pos = ID_AA64MMFR2_CNP_SHIFT,
1445                 .min_field_value = 1,
1446                 .cpu_enable = cpu_enable_cnp,
1447         },
1448 #endif
1449         {
1450                 .desc = "Speculation barrier (SB)",
1451                 .capability = ARM64_HAS_SB,
1452                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1453                 .matches = has_cpuid_feature,
1454                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1455                 .field_pos = ID_AA64ISAR1_SB_SHIFT,
1456                 .sign = FTR_UNSIGNED,
1457                 .min_field_value = 1,
1458         },
1459 #ifdef CONFIG_ARM64_PTR_AUTH
1460         {
1461                 .desc = "Address authentication (architected algorithm)",
1462                 .capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
1463                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1464                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1465                 .sign = FTR_UNSIGNED,
1466                 .field_pos = ID_AA64ISAR1_APA_SHIFT,
1467                 .min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
1468                 .matches = has_cpuid_feature,
1469                 .cpu_enable = cpu_enable_address_auth,
1470         },
1471         {
1472                 .desc = "Address authentication (IMP DEF algorithm)",
1473                 .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
1474                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1475                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1476                 .sign = FTR_UNSIGNED,
1477                 .field_pos = ID_AA64ISAR1_API_SHIFT,
1478                 .min_field_value = ID_AA64ISAR1_API_IMP_DEF,
1479                 .matches = has_cpuid_feature,
1480                 .cpu_enable = cpu_enable_address_auth,
1481         },
1482         {
1483                 .desc = "Generic authentication (architected algorithm)",
1484                 .capability = ARM64_HAS_GENERIC_AUTH_ARCH,
1485                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1486                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1487                 .sign = FTR_UNSIGNED,
1488                 .field_pos = ID_AA64ISAR1_GPA_SHIFT,
1489                 .min_field_value = ID_AA64ISAR1_GPA_ARCHITECTED,
1490                 .matches = has_cpuid_feature,
1491         },
1492         {
1493                 .desc = "Generic authentication (IMP DEF algorithm)",
1494                 .capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
1495                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1496                 .sys_reg = SYS_ID_AA64ISAR1_EL1,
1497                 .sign = FTR_UNSIGNED,
1498                 .field_pos = ID_AA64ISAR1_GPI_SHIFT,
1499                 .min_field_value = ID_AA64ISAR1_GPI_IMP_DEF,
1500                 .matches = has_cpuid_feature,
1501         },
1502 #endif /* CONFIG_ARM64_PTR_AUTH */
1503 #ifdef CONFIG_ARM64_PSEUDO_NMI
1504         {
1505                 /*
1506                  * Depends on having GICv3
1507                  */
1508                 .desc = "IRQ priority masking",
1509                 .capability = ARM64_HAS_IRQ_PRIO_MASKING,
1510                 .type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
1511                 .matches = can_use_gic_priorities,
1512                 .sys_reg = SYS_ID_AA64PFR0_EL1,
1513                 .field_pos = ID_AA64PFR0_GIC_SHIFT,
1514                 .sign = FTR_UNSIGNED,
1515                 .min_field_value = 1,
1516         },
1517 #endif
1518         {},
1519 };
1520 
1521 #define HWCAP_CPUID_MATCH(reg, field, s, min_value)                             \
1522                 .matches = has_cpuid_feature,                                   \
1523                 .sys_reg = reg,                                                 \
1524                 .field_pos = field,                                             \
1525                 .sign = s,                                                      \
1526                 .min_field_value = min_value,
1527 
1528 #define __HWCAP_CAP(name, cap_type, cap)                                        \
1529                 .desc = name,                                                   \
1530                 .type = ARM64_CPUCAP_SYSTEM_FEATURE,                            \
1531                 .hwcap_type = cap_type,                                         \
1532                 .hwcap = cap,                                                   \
1533 
1534 #define HWCAP_CAP(reg, field, s, min_value, cap_type, cap)                      \
1535         {                                                                       \
1536                 __HWCAP_CAP(#cap, cap_type, cap)                                \
1537                 HWCAP_CPUID_MATCH(reg, field, s, min_value)                     \
1538         }
1539 
1540 #define HWCAP_MULTI_CAP(list, cap_type, cap)                                    \
1541         {                                                                       \
1542                 __HWCAP_CAP(#cap, cap_type, cap)                                \
1543                 .matches = cpucap_multi_entry_cap_matches,                      \
1544                 .match_list = list,                                             \
1545         }
1546 
1547 #ifdef CONFIG_ARM64_PTR_AUTH
1548 static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = {
1549         {
1550                 HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_APA_SHIFT,
1551                                   FTR_UNSIGNED, ID_AA64ISAR1_APA_ARCHITECTED)
1552         },
1553         {
1554                 HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_SHIFT,
1555                                   FTR_UNSIGNED, ID_AA64ISAR1_API_IMP_DEF)
1556         },
1557         {},
1558 };
1559 
1560 static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = {
1561         {
1562                 HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPA_SHIFT,
1563                                   FTR_UNSIGNED, ID_AA64ISAR1_GPA_ARCHITECTED)
1564         },
1565         {
1566                 HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPI_SHIFT,
1567                                   FTR_UNSIGNED, ID_AA64ISAR1_GPI_IMP_DEF)
1568         },
1569         {},
1570 };
1571 #endif
1572 
1573 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
1574         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL),
1575         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
1576         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA1),
1577         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA2),
1578         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA2_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_SHA512),
1579         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_CRC32),
1580         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_ATOMICS_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ATOMICS),
1581         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_RDM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDRDM),
1582         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SHA3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SHA3),
1583         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM3_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM3),
1584         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_SM4_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SM4),
1585         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_DP_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDDP),
1586         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_FHM_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_ASIMDFHM),
1587         HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_TS_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FLAGM),
1588         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_FP),
1589         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
1590         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
1591         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
1592         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_DIT),
1593         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_DPB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_DCPOP),
1594         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_JSCVT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_JSCVT),
1595         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
1596         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
1597         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ILRCPC),
1598         HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SB),
1599         HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_USCAT),
1600 #ifdef CONFIG_ARM64_SVE
1601         HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
1602 #endif
1603         HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS),
1604 #ifdef CONFIG_ARM64_PTR_AUTH
1605         HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, HWCAP_PACA),
1606         HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, HWCAP_PACG),
1607 #endif
1608         {},
1609 };
1610 
1611 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
1612 #ifdef CONFIG_COMPAT
1613         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 2, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
1614         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_AES_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
1615         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA1_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
1616         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_SHA2_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
1617         HWCAP_CAP(SYS_ID_ISAR5_EL1, ID_ISAR5_CRC32_SHIFT, FTR_UNSIGNED, 1, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
1618 #endif
1619         {},
1620 };
1621 
1622 static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
1623 {
1624         switch (cap->hwcap_type) {
1625         case CAP_HWCAP:
1626                 elf_hwcap |= cap->hwcap;
1627                 break;
1628 #ifdef CONFIG_COMPAT
1629         case CAP_COMPAT_HWCAP:
1630                 compat_elf_hwcap |= (u32)cap->hwcap;
1631                 break;
1632         case CAP_COMPAT_HWCAP2:
1633                 compat_elf_hwcap2 |= (u32)cap->hwcap;
1634                 break;
1635 #endif
1636         default:
1637                 WARN_ON(1);
1638                 break;
1639         }
1640 }
1641 
1642 /* Check if we have a particular HWCAP enabled */
1643 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
1644 {
1645         bool rc;
1646 
1647         switch (cap->hwcap_type) {
1648         case CAP_HWCAP:
1649                 rc = (elf_hwcap & cap->hwcap) != 0;
1650                 break;
1651 #ifdef CONFIG_COMPAT
1652         case CAP_COMPAT_HWCAP:
1653                 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
1654                 break;
1655         case CAP_COMPAT_HWCAP2:
1656                 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
1657                 break;
1658 #endif
1659         default:
1660                 WARN_ON(1);
1661                 rc = false;
1662         }
1663 
1664         return rc;
1665 }
1666 
1667 static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
1668 {
1669         /* We support emulation of accesses to CPU ID feature registers */
1670         elf_hwcap |= HWCAP_CPUID;
1671         for (; hwcaps->matches; hwcaps++)
1672                 if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
1673                         cap_set_elf_hwcap(hwcaps);
1674 }
1675 
1676 static void update_cpu_capabilities(u16 scope_mask)
1677 {
1678         int i;
1679         const struct arm64_cpu_capabilities *caps;
1680 
1681         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1682         for (i = 0; i < ARM64_NCAPS; i++) {
1683                 caps = cpu_hwcaps_ptrs[i];
1684                 if (!caps || !(caps->type & scope_mask) ||
1685                     cpus_have_cap(caps->capability) ||
1686                     !caps->matches(caps, cpucap_default_scope(caps)))
1687                         continue;
1688 
1689                 if (caps->desc)
1690                         pr_info("detected: %s\n", caps->desc);
1691                 cpus_set_cap(caps->capability);
1692 
1693                 if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU))
1694                         set_bit(caps->capability, boot_capabilities);
1695         }
1696 }
1697 
1698 /*
1699  * Enable all the available capabilities on this CPU. The capabilities
1700  * with BOOT_CPU scope are handled separately and hence skipped here.
1701  */
1702 static int cpu_enable_non_boot_scope_capabilities(void *__unused)
1703 {
1704         int i;
1705         u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
1706 
1707         for_each_available_cap(i) {
1708                 const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[i];
1709 
1710                 if (WARN_ON(!cap))
1711                         continue;
1712 
1713                 if (!(cap->type & non_boot_scope))
1714                         continue;
1715 
1716                 if (cap->cpu_enable)
1717                         cap->cpu_enable(cap);
1718         }
1719         return 0;
1720 }
1721 
1722 /*
1723  * Run through the enabled capabilities and enable() it on all active
1724  * CPUs
1725  */
1726 static void __init enable_cpu_capabilities(u16 scope_mask)
1727 {
1728         int i;
1729         const struct arm64_cpu_capabilities *caps;
1730         bool boot_scope;
1731 
1732         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1733         boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
1734 
1735         for (i = 0; i < ARM64_NCAPS; i++) {
1736                 unsigned int num;
1737 
1738                 caps = cpu_hwcaps_ptrs[i];
1739                 if (!caps || !(caps->type & scope_mask))
1740                         continue;
1741                 num = caps->capability;
1742                 if (!cpus_have_cap(num))
1743                         continue;
1744 
1745                 /* Ensure cpus_have_const_cap(num) works */
1746                 static_branch_enable(&cpu_hwcap_keys[num]);
1747 
1748                 if (boot_scope && caps->cpu_enable)
1749                         /*
1750                          * Capabilities with SCOPE_BOOT_CPU scope are finalised
1751                          * before any secondary CPU boots. Thus, each secondary
1752                          * will enable the capability as appropriate via
1753                          * check_local_cpu_capabilities(). The only exception is
1754                          * the boot CPU, for which the capability must be
1755                          * enabled here. This approach avoids costly
1756                          * stop_machine() calls for this case.
1757                          */
1758                         caps->cpu_enable(caps);
1759         }
1760 
1761         /*
1762          * For all non-boot scope capabilities, use stop_machine()
1763          * as it schedules the work allowing us to modify PSTATE,
1764          * instead of on_each_cpu() which uses an IPI, giving us a
1765          * PSTATE that disappears when we return.
1766          */
1767         if (!boot_scope)
1768                 stop_machine(cpu_enable_non_boot_scope_capabilities,
1769                              NULL, cpu_online_mask);
1770 }
1771 
1772 /*
1773  * Run through the list of capabilities to check for conflicts.
1774  * If the system has already detected a capability, take necessary
1775  * action on this CPU.
1776  *
1777  * Returns "false" on conflicts.
1778  */
1779 static bool verify_local_cpu_caps(u16 scope_mask)
1780 {
1781         int i;
1782         bool cpu_has_cap, system_has_cap;
1783         const struct arm64_cpu_capabilities *caps;
1784 
1785         scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
1786 
1787         for (i = 0; i < ARM64_NCAPS; i++) {
1788                 caps = cpu_hwcaps_ptrs[i];
1789                 if (!caps || !(caps->type & scope_mask))
1790                         continue;
1791 
1792                 cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
1793                 system_has_cap = cpus_have_cap(caps->capability);
1794 
1795                 if (system_has_cap) {
1796                         /*
1797                          * Check if the new CPU misses an advertised feature,
1798                          * which is not safe to miss.
1799                          */
1800                         if (!cpu_has_cap && !cpucap_late_cpu_optional(caps))
1801                                 break;
1802                         /*
1803                          * We have to issue cpu_enable() irrespective of
1804                          * whether the CPU has it or not, as it is enabeld
1805                          * system wide. It is upto the call back to take
1806                          * appropriate action on this CPU.
1807                          */
1808                         if (caps->cpu_enable)
1809                                 caps->cpu_enable(caps);
1810                 } else {
1811                         /*
1812                          * Check if the CPU has this capability if it isn't
1813                          * safe to have when the system doesn't.
1814                          */
1815                         if (cpu_has_cap && !cpucap_late_cpu_permitted(caps))
1816                                 break;
1817                 }
1818         }
1819 
1820         if (i < ARM64_NCAPS) {
1821                 pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
1822                         smp_processor_id(), caps->capability,
1823                         caps->desc, system_has_cap, cpu_has_cap);
1824                 return false;
1825         }
1826 
1827         return true;
1828 }
1829 
1830 /*
1831  * Check for CPU features that are used in early boot
1832  * based on the Boot CPU value.
1833  */
1834 static void check_early_cpu_features(void)
1835 {
1836         verify_cpu_asid_bits();
1837         /*
1838          * Early features are used by the kernel already. If there
1839          * is a conflict, we cannot proceed further.
1840          */
1841         if (!verify_local_cpu_caps(SCOPE_BOOT_CPU))
1842                 cpu_panic_kernel();
1843 }
1844 
1845 static void
1846 verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
1847 {
1848 
1849         for (; caps->matches; caps++)
1850                 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
1851                         pr_crit("CPU%d: missing HWCAP: %s\n",
1852                                         smp_processor_id(), caps->desc);
1853                         cpu_die_early();
1854                 }
1855 }
1856 
1857 static void verify_sve_features(void)
1858 {
1859         u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
1860         u64 zcr = read_zcr_features();
1861 
1862         unsigned int safe_len = safe_zcr & ZCR_ELx_LEN_MASK;
1863         unsigned int len = zcr & ZCR_ELx_LEN_MASK;
1864 
1865         if (len < safe_len || sve_verify_vq_map()) {
1866                 pr_crit("CPU%d: SVE: required vector length(s) missing\n",
1867                         smp_processor_id());
1868                 cpu_die_early();
1869         }
1870 
1871         /* Add checks on other ZCR bits here if necessary */
1872 }
1873 
1874 
1875 /*
1876  * Run through the enabled system capabilities and enable() it on this CPU.
1877  * The capabilities were decided based on the available CPUs at the boot time.
1878  * Any new CPU should match the system wide status of the capability. If the
1879  * new CPU doesn't have a capability which the system now has enabled, we
1880  * cannot do anything to fix it up and could cause unexpected failures. So
1881  * we park the CPU.
1882  */
1883 static void verify_local_cpu_capabilities(void)
1884 {
1885         /*
1886          * The capabilities with SCOPE_BOOT_CPU are checked from
1887          * check_early_cpu_features(), as they need to be verified
1888          * on all secondary CPUs.
1889          */
1890         if (!verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU))
1891                 cpu_die_early();
1892 
1893         verify_local_elf_hwcaps(arm64_elf_hwcaps);
1894 
1895         if (system_supports_32bit_el0())
1896                 verify_local_elf_hwcaps(compat_elf_hwcaps);
1897 
1898         if (system_supports_sve())
1899                 verify_sve_features();
1900 }
1901 
1902 void check_local_cpu_capabilities(void)
1903 {
1904         /*
1905          * All secondary CPUs should conform to the early CPU features
1906          * in use by the kernel based on boot CPU.
1907          */
1908         check_early_cpu_features();
1909 
1910         /*
1911          * If we haven't finalised the system capabilities, this CPU gets
1912          * a chance to update the errata work arounds and local features.
1913          * Otherwise, this CPU should verify that it has all the system
1914          * advertised capabilities.
1915          */
1916         if (!sys_caps_initialised)
1917                 update_cpu_capabilities(SCOPE_LOCAL_CPU);
1918         else
1919                 verify_local_cpu_capabilities();
1920 }
1921 
1922 static void __init setup_boot_cpu_capabilities(void)
1923 {
1924         /* Detect capabilities with either SCOPE_BOOT_CPU or SCOPE_LOCAL_CPU */
1925         update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU);
1926         /* Enable the SCOPE_BOOT_CPU capabilities alone right away */
1927         enable_cpu_capabilities(SCOPE_BOOT_CPU);
1928 }
1929 
1930 DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
1931 EXPORT_SYMBOL(arm64_const_caps_ready);
1932 
1933 static void __init mark_const_caps_ready(void)
1934 {
1935         static_branch_enable(&arm64_const_caps_ready);
1936 }
1937 
1938 bool this_cpu_has_cap(unsigned int n)
1939 {
1940         if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) {
1941                 const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
1942 
1943                 if (cap)
1944                         return cap->matches(cap, SCOPE_LOCAL_CPU);
1945         }
1946 
1947         return false;
1948 }
1949 
1950 static void __init setup_system_capabilities(void)
1951 {
1952         /*
1953          * We have finalised the system-wide safe feature
1954          * registers, finalise the capabilities that depend
1955          * on it. Also enable all the available capabilities,
1956          * that are not enabled already.
1957          */
1958         update_cpu_capabilities(SCOPE_SYSTEM);
1959         enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
1960 }
1961 
1962 void __init setup_cpu_features(void)
1963 {
1964         u32 cwg;
1965 
1966         setup_system_capabilities();
1967         mark_const_caps_ready();
1968         setup_elf_hwcaps(arm64_elf_hwcaps);
1969 
1970         if (system_supports_32bit_el0())
1971                 setup_elf_hwcaps(compat_elf_hwcaps);
1972 
1973         if (system_uses_ttbr0_pan())
1974                 pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
1975 
1976         sve_setup();
1977         minsigstksz_setup();
1978 
1979         /* Advertise that we have computed the system capabilities */
1980         set_sys_caps_initialised();
1981 
1982         /*
1983          * Check for sane CTR_EL0.CWG value.
1984          */
1985         cwg = cache_type_cwg();
1986         if (!cwg)
1987                 pr_warn("No Cache Writeback Granule information, assuming %d\n",
1988                         ARCH_DMA_MINALIGN);
1989 }
1990 
1991 static bool __maybe_unused
1992 cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
1993 {
1994         return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
1995 }
1996 
1997 static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
1998 {
1999         cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
2000 }
2001 
2002 /*
2003  * We emulate only the following system register space.
2004  * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 4 - 7]
2005  * See Table C5-6 System instruction encodings for System register accesses,
2006  * ARMv8 ARM(ARM DDI 0487A.f) for more details.
2007  */
2008 static inline bool __attribute_const__ is_emulated(u32 id)
2009 {
2010         return (sys_reg_Op0(id) == 0x3 &&
2011                 sys_reg_CRn(id) == 0x0 &&
2012                 sys_reg_Op1(id) == 0x0 &&
2013                 (sys_reg_CRm(id) == 0 ||
2014                  ((sys_reg_CRm(id) >= 4) && (sys_reg_CRm(id) <= 7))));
2015 }
2016 
2017 /*
2018  * With CRm == 0, reg should be one of :
2019  * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
2020  */
2021 static inline int emulate_id_reg(u32 id, u64 *valp)
2022 {
2023         switch (id) {
2024         case SYS_MIDR_EL1:
2025                 *valp = read_cpuid_id();
2026                 break;
2027         case SYS_MPIDR_EL1:
2028                 *valp = SYS_MPIDR_SAFE_VAL;
2029                 break;
2030         case SYS_REVIDR_EL1:
2031                 /* IMPLEMENTATION DEFINED values are emulated with 0 */
2032                 *valp = 0;
2033                 break;
2034         default:
2035                 return -EINVAL;
2036         }
2037 
2038         return 0;
2039 }
2040 
2041 static int emulate_sys_reg(u32 id, u64 *valp)
2042 {
2043         struct arm64_ftr_reg *regp;
2044 
2045         if (!is_emulated(id))
2046                 return -EINVAL;
2047 
2048         if (sys_reg_CRm(id) == 0)
2049                 return emulate_id_reg(id, valp);
2050 
2051         regp = get_arm64_ftr_reg(id);
2052         if (regp)
2053                 *valp = arm64_ftr_reg_user_value(regp);
2054         else
2055                 /*
2056                  * The untracked registers are either IMPLEMENTATION DEFINED
2057                  * (e.g, ID_AFR0_EL1) or reserved RAZ.
2058                  */
2059                 *valp = 0;
2060         return 0;
2061 }
2062 
2063 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
2064 {
2065         int rc;
2066         u64 val;
2067 
2068         rc = emulate_sys_reg(sys_reg, &val);
2069         if (!rc) {
2070                 pt_regs_write_reg(regs, rt, val);
2071                 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
2072         }
2073         return rc;
2074 }
2075 
2076 static int emulate_mrs(struct pt_regs *regs, u32 insn)
2077 {
2078         u32 sys_reg, rt;
2079 
2080         /*
2081          * sys_reg values are defined as used in mrs/msr instruction.
2082          * shift the imm value to get the encoding.
2083          */
2084         sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
2085         rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
2086         return do_emulate_mrs(regs, sys_reg, rt);
2087 }
2088 
2089 static struct undef_hook mrs_hook = {
2090         .instr_mask = 0xfff00000,
2091         .instr_val  = 0xd5300000,
2092         .pstate_mask = PSR_AA32_MODE_MASK,
2093         .pstate_val = PSR_MODE_EL0t,
2094         .fn = emulate_mrs,
2095 };
2096 
2097 static int __init enable_mrs_emulation(void)
2098 {
2099         register_undef_hook(&mrs_hook);
2100         return 0;
2101 }
2102 
2103 core_initcall(enable_mrs_emulation);
2104 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp