~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/crypto/crc32-ce-glue.c

Version: ~ [ linux-5.8-rc3 ] ~ [ linux-5.7.5 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.48 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.129 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.185 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.228 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.228 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
  3  *
  4  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  */
 10 
 11 #include <linux/cpufeature.h>
 12 #include <linux/crc32.h>
 13 #include <linux/init.h>
 14 #include <linux/kernel.h>
 15 #include <linux/module.h>
 16 #include <linux/string.h>
 17 
 18 #include <crypto/internal/hash.h>
 19 
 20 #include <asm/hwcap.h>
 21 #include <asm/neon.h>
 22 #include <asm/simd.h>
 23 #include <asm/unaligned.h>
 24 
 25 #define PMULL_MIN_LEN           64L     /* minimum size of buffer
 26                                          * for crc32_pmull_le_16 */
 27 #define SCALE_F                 16L     /* size of NEON register */
 28 
 29 asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
 30 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
 31 
 32 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
 33 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
 34 
 35 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
 36 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
 37 
 38 static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
 39 {
 40         u32 *key = crypto_tfm_ctx(tfm);
 41 
 42         *key = 0;
 43         return 0;
 44 }
 45 
 46 static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
 47 {
 48         u32 *key = crypto_tfm_ctx(tfm);
 49 
 50         *key = ~0;
 51         return 0;
 52 }
 53 
 54 static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
 55                               unsigned int keylen)
 56 {
 57         u32 *mctx = crypto_shash_ctx(hash);
 58 
 59         if (keylen != sizeof(u32)) {
 60                 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
 61                 return -EINVAL;
 62         }
 63         *mctx = le32_to_cpup((__le32 *)key);
 64         return 0;
 65 }
 66 
 67 static int crc32_pmull_init(struct shash_desc *desc)
 68 {
 69         u32 *mctx = crypto_shash_ctx(desc->tfm);
 70         u32 *crc = shash_desc_ctx(desc);
 71 
 72         *crc = *mctx;
 73         return 0;
 74 }
 75 
 76 static int crc32_update(struct shash_desc *desc, const u8 *data,
 77                         unsigned int length)
 78 {
 79         u32 *crc = shash_desc_ctx(desc);
 80 
 81         *crc = crc32_armv8_le(*crc, data, length);
 82         return 0;
 83 }
 84 
 85 static int crc32c_update(struct shash_desc *desc, const u8 *data,
 86                          unsigned int length)
 87 {
 88         u32 *crc = shash_desc_ctx(desc);
 89 
 90         *crc = crc32c_armv8_le(*crc, data, length);
 91         return 0;
 92 }
 93 
 94 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
 95                          unsigned int length)
 96 {
 97         u32 *crc = shash_desc_ctx(desc);
 98         unsigned int l;
 99 
100         if ((u64)data % SCALE_F) {
101                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
102 
103                 *crc = fallback_crc32(*crc, data, l);
104 
105                 data += l;
106                 length -= l;
107         }
108 
109         if (length >= PMULL_MIN_LEN && may_use_simd()) {
110                 l = round_down(length, SCALE_F);
111 
112                 kernel_neon_begin();
113                 *crc = crc32_pmull_le(data, l, *crc);
114                 kernel_neon_end();
115 
116                 data += l;
117                 length -= l;
118         }
119 
120         if (length > 0)
121                 *crc = fallback_crc32(*crc, data, length);
122 
123         return 0;
124 }
125 
126 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
127                          unsigned int length)
128 {
129         u32 *crc = shash_desc_ctx(desc);
130         unsigned int l;
131 
132         if ((u64)data % SCALE_F) {
133                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
134 
135                 *crc = fallback_crc32c(*crc, data, l);
136 
137                 data += l;
138                 length -= l;
139         }
140 
141         if (length >= PMULL_MIN_LEN && may_use_simd()) {
142                 l = round_down(length, SCALE_F);
143 
144                 kernel_neon_begin();
145                 *crc = crc32c_pmull_le(data, l, *crc);
146                 kernel_neon_end();
147 
148                 data += l;
149                 length -= l;
150         }
151 
152         if (length > 0) {
153                 *crc = fallback_crc32c(*crc, data, length);
154         }
155 
156         return 0;
157 }
158 
159 static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
160 {
161         u32 *crc = shash_desc_ctx(desc);
162 
163         put_unaligned_le32(*crc, out);
164         return 0;
165 }
166 
167 static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
168 {
169         u32 *crc = shash_desc_ctx(desc);
170 
171         put_unaligned_le32(~*crc, out);
172         return 0;
173 }
174 
175 static struct shash_alg crc32_pmull_algs[] = { {
176         .setkey                 = crc32_pmull_setkey,
177         .init                   = crc32_pmull_init,
178         .update                 = crc32_update,
179         .final                  = crc32_pmull_final,
180         .descsize               = sizeof(u32),
181         .digestsize             = sizeof(u32),
182 
183         .base.cra_ctxsize       = sizeof(u32),
184         .base.cra_init          = crc32_pmull_cra_init,
185         .base.cra_name          = "crc32",
186         .base.cra_driver_name   = "crc32-arm64-ce",
187         .base.cra_priority      = 200,
188         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
189         .base.cra_blocksize     = 1,
190         .base.cra_module        = THIS_MODULE,
191 }, {
192         .setkey                 = crc32_pmull_setkey,
193         .init                   = crc32_pmull_init,
194         .update                 = crc32c_update,
195         .final                  = crc32c_pmull_final,
196         .descsize               = sizeof(u32),
197         .digestsize             = sizeof(u32),
198 
199         .base.cra_ctxsize       = sizeof(u32),
200         .base.cra_init          = crc32c_pmull_cra_init,
201         .base.cra_name          = "crc32c",
202         .base.cra_driver_name   = "crc32c-arm64-ce",
203         .base.cra_priority      = 200,
204         .base.cra_flags         = CRYPTO_ALG_OPTIONAL_KEY,
205         .base.cra_blocksize     = 1,
206         .base.cra_module        = THIS_MODULE,
207 } };
208 
209 static int __init crc32_pmull_mod_init(void)
210 {
211         if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
212                 crc32_pmull_algs[0].update = crc32_pmull_update;
213                 crc32_pmull_algs[1].update = crc32c_pmull_update;
214 
215                 if (elf_hwcap & HWCAP_CRC32) {
216                         fallback_crc32 = crc32_armv8_le;
217                         fallback_crc32c = crc32c_armv8_le;
218                 } else {
219                         fallback_crc32 = crc32_le;
220                         fallback_crc32c = __crc32c_le;
221                 }
222         } else if (!(elf_hwcap & HWCAP_CRC32)) {
223                 return -ENODEV;
224         }
225         return crypto_register_shashes(crc32_pmull_algs,
226                                        ARRAY_SIZE(crc32_pmull_algs));
227 }
228 
229 static void __exit crc32_pmull_mod_exit(void)
230 {
231         crypto_unregister_shashes(crc32_pmull_algs,
232                                   ARRAY_SIZE(crc32_pmull_algs));
233 }
234 
235 static const struct cpu_feature crc32_cpu_feature[] = {
236         { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
237 };
238 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
239 
240 module_init(crc32_pmull_mod_init);
241 module_exit(crc32_pmull_mod_exit);
242 
243 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
244 MODULE_LICENSE("GPL v2");
245 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp