~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/crypto/aes-glue.c

Version: ~ [ linux-5.1-rc5 ] ~ [ linux-5.0.7 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.34 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.111 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.168 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.178 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.138 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.65 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
  3  *
  4  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License version 2 as
  8  * published by the Free Software Foundation.
  9  */
 10 
 11 #include <asm/neon.h>
 12 #include <asm/hwcap.h>
 13 #include <asm/simd.h>
 14 #include <crypto/aes.h>
 15 #include <crypto/internal/hash.h>
 16 #include <crypto/internal/simd.h>
 17 #include <crypto/internal/skcipher.h>
 18 #include <crypto/scatterwalk.h>
 19 #include <linux/module.h>
 20 #include <linux/cpufeature.h>
 21 #include <crypto/xts.h>
 22 
 23 #include "aes-ce-setkey.h"
 24 #include "aes-ctr-fallback.h"
 25 
 26 #ifdef USE_V8_CRYPTO_EXTENSIONS
 27 #define MODE                    "ce"
 28 #define PRIO                    300
 29 #define aes_setkey              ce_aes_setkey
 30 #define aes_expandkey           ce_aes_expandkey
 31 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
 32 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
 33 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
 34 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
 35 #define aes_cbc_cts_encrypt     ce_aes_cbc_cts_encrypt
 36 #define aes_cbc_cts_decrypt     ce_aes_cbc_cts_decrypt
 37 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
 38 #define aes_xts_encrypt         ce_aes_xts_encrypt
 39 #define aes_xts_decrypt         ce_aes_xts_decrypt
 40 #define aes_mac_update          ce_aes_mac_update
 41 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
 42 #else
 43 #define MODE                    "neon"
 44 #define PRIO                    200
 45 #define aes_setkey              crypto_aes_set_key
 46 #define aes_expandkey           crypto_aes_expand_key
 47 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
 48 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
 49 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
 50 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
 51 #define aes_cbc_cts_encrypt     neon_aes_cbc_cts_encrypt
 52 #define aes_cbc_cts_decrypt     neon_aes_cbc_cts_decrypt
 53 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
 54 #define aes_xts_encrypt         neon_aes_xts_encrypt
 55 #define aes_xts_decrypt         neon_aes_xts_decrypt
 56 #define aes_mac_update          neon_aes_mac_update
 57 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
 58 MODULE_ALIAS_CRYPTO("ecb(aes)");
 59 MODULE_ALIAS_CRYPTO("cbc(aes)");
 60 MODULE_ALIAS_CRYPTO("ctr(aes)");
 61 MODULE_ALIAS_CRYPTO("xts(aes)");
 62 MODULE_ALIAS_CRYPTO("cmac(aes)");
 63 MODULE_ALIAS_CRYPTO("xcbc(aes)");
 64 MODULE_ALIAS_CRYPTO("cbcmac(aes)");
 65 #endif
 66 
 67 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 68 MODULE_LICENSE("GPL v2");
 69 
 70 /* defined in aes-modes.S */
 71 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
 72                                 int rounds, int blocks);
 73 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u32 const rk[],
 74                                 int rounds, int blocks);
 75 
 76 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
 77                                 int rounds, int blocks, u8 iv[]);
 78 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u32 const rk[],
 79                                 int rounds, int blocks, u8 iv[]);
 80 
 81 asmlinkage void aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[],
 82                                 int rounds, int bytes, u8 const iv[]);
 83 asmlinkage void aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[],
 84                                 int rounds, int bytes, u8 const iv[]);
 85 
 86 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u32 const rk[],
 87                                 int rounds, int blocks, u8 ctr[]);
 88 
 89 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u32 const rk1[],
 90                                 int rounds, int blocks, u32 const rk2[], u8 iv[],
 91                                 int first);
 92 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u32 const rk1[],
 93                                 int rounds, int blocks, u32 const rk2[], u8 iv[],
 94                                 int first);
 95 
 96 asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
 97                                int blocks, u8 dg[], int enc_before,
 98                                int enc_after);
 99 
100 struct cts_cbc_req_ctx {
101         struct scatterlist sg_src[2];
102         struct scatterlist sg_dst[2];
103         struct skcipher_request subreq;
104 };
105 
106 struct crypto_aes_xts_ctx {
107         struct crypto_aes_ctx key1;
108         struct crypto_aes_ctx __aligned(8) key2;
109 };
110 
111 struct mac_tfm_ctx {
112         struct crypto_aes_ctx key;
113         u8 __aligned(8) consts[];
114 };
115 
116 struct mac_desc_ctx {
117         unsigned int len;
118         u8 dg[AES_BLOCK_SIZE];
119 };
120 
121 static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
122                                unsigned int key_len)
123 {
124         return aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
125 }
126 
127 static int xts_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
128                        unsigned int key_len)
129 {
130         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
131         int ret;
132 
133         ret = xts_verify_key(tfm, in_key, key_len);
134         if (ret)
135                 return ret;
136 
137         ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
138         if (!ret)
139                 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
140                                     key_len / 2);
141         if (!ret)
142                 return 0;
143 
144         crypto_skcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
145         return -EINVAL;
146 }
147 
148 static int ecb_encrypt(struct skcipher_request *req)
149 {
150         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
151         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
152         int err, rounds = 6 + ctx->key_length / 4;
153         struct skcipher_walk walk;
154         unsigned int blocks;
155 
156         err = skcipher_walk_virt(&walk, req, false);
157 
158         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
159                 kernel_neon_begin();
160                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
161                                 ctx->key_enc, rounds, blocks);
162                 kernel_neon_end();
163                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
164         }
165         return err;
166 }
167 
168 static int ecb_decrypt(struct skcipher_request *req)
169 {
170         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
171         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
172         int err, rounds = 6 + ctx->key_length / 4;
173         struct skcipher_walk walk;
174         unsigned int blocks;
175 
176         err = skcipher_walk_virt(&walk, req, false);
177 
178         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
179                 kernel_neon_begin();
180                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
181                                 ctx->key_dec, rounds, blocks);
182                 kernel_neon_end();
183                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
184         }
185         return err;
186 }
187 
188 static int cbc_encrypt(struct skcipher_request *req)
189 {
190         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
191         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
192         int err, rounds = 6 + ctx->key_length / 4;
193         struct skcipher_walk walk;
194         unsigned int blocks;
195 
196         err = skcipher_walk_virt(&walk, req, false);
197 
198         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
199                 kernel_neon_begin();
200                 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
201                                 ctx->key_enc, rounds, blocks, walk.iv);
202                 kernel_neon_end();
203                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
204         }
205         return err;
206 }
207 
208 static int cbc_decrypt(struct skcipher_request *req)
209 {
210         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
211         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
212         int err, rounds = 6 + ctx->key_length / 4;
213         struct skcipher_walk walk;
214         unsigned int blocks;
215 
216         err = skcipher_walk_virt(&walk, req, false);
217 
218         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
219                 kernel_neon_begin();
220                 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
221                                 ctx->key_dec, rounds, blocks, walk.iv);
222                 kernel_neon_end();
223                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
224         }
225         return err;
226 }
227 
228 static int cts_cbc_init_tfm(struct crypto_skcipher *tfm)
229 {
230         crypto_skcipher_set_reqsize(tfm, sizeof(struct cts_cbc_req_ctx));
231         return 0;
232 }
233 
234 static int cts_cbc_encrypt(struct skcipher_request *req)
235 {
236         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
237         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
238         struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
239         int err, rounds = 6 + ctx->key_length / 4;
240         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
241         struct scatterlist *src = req->src, *dst = req->dst;
242         struct skcipher_walk walk;
243 
244         skcipher_request_set_tfm(&rctx->subreq, tfm);
245 
246         if (req->cryptlen <= AES_BLOCK_SIZE) {
247                 if (req->cryptlen < AES_BLOCK_SIZE)
248                         return -EINVAL;
249                 cbc_blocks = 1;
250         }
251 
252         if (cbc_blocks > 0) {
253                 unsigned int blocks;
254 
255                 skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
256                                            cbc_blocks * AES_BLOCK_SIZE,
257                                            req->iv);
258 
259                 err = skcipher_walk_virt(&walk, &rctx->subreq, false);
260 
261                 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
262                         kernel_neon_begin();
263                         aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
264                                         ctx->key_enc, rounds, blocks, walk.iv);
265                         kernel_neon_end();
266                         err = skcipher_walk_done(&walk,
267                                                  walk.nbytes % AES_BLOCK_SIZE);
268                 }
269                 if (err)
270                         return err;
271 
272                 if (req->cryptlen == AES_BLOCK_SIZE)
273                         return 0;
274 
275                 dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
276                                              rctx->subreq.cryptlen);
277                 if (req->dst != req->src)
278                         dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
279                                                rctx->subreq.cryptlen);
280         }
281 
282         /* handle ciphertext stealing */
283         skcipher_request_set_crypt(&rctx->subreq, src, dst,
284                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
285                                    req->iv);
286 
287         err = skcipher_walk_virt(&walk, &rctx->subreq, false);
288         if (err)
289                 return err;
290 
291         kernel_neon_begin();
292         aes_cbc_cts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
293                             ctx->key_enc, rounds, walk.nbytes, walk.iv);
294         kernel_neon_end();
295 
296         return skcipher_walk_done(&walk, 0);
297 }
298 
299 static int cts_cbc_decrypt(struct skcipher_request *req)
300 {
301         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
302         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
303         struct cts_cbc_req_ctx *rctx = skcipher_request_ctx(req);
304         int err, rounds = 6 + ctx->key_length / 4;
305         int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
306         struct scatterlist *src = req->src, *dst = req->dst;
307         struct skcipher_walk walk;
308 
309         skcipher_request_set_tfm(&rctx->subreq, tfm);
310 
311         if (req->cryptlen <= AES_BLOCK_SIZE) {
312                 if (req->cryptlen < AES_BLOCK_SIZE)
313                         return -EINVAL;
314                 cbc_blocks = 1;
315         }
316 
317         if (cbc_blocks > 0) {
318                 unsigned int blocks;
319 
320                 skcipher_request_set_crypt(&rctx->subreq, req->src, req->dst,
321                                            cbc_blocks * AES_BLOCK_SIZE,
322                                            req->iv);
323 
324                 err = skcipher_walk_virt(&walk, &rctx->subreq, false);
325 
326                 while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
327                         kernel_neon_begin();
328                         aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
329                                         ctx->key_dec, rounds, blocks, walk.iv);
330                         kernel_neon_end();
331                         err = skcipher_walk_done(&walk,
332                                                  walk.nbytes % AES_BLOCK_SIZE);
333                 }
334                 if (err)
335                         return err;
336 
337                 if (req->cryptlen == AES_BLOCK_SIZE)
338                         return 0;
339 
340                 dst = src = scatterwalk_ffwd(rctx->sg_src, req->src,
341                                              rctx->subreq.cryptlen);
342                 if (req->dst != req->src)
343                         dst = scatterwalk_ffwd(rctx->sg_dst, req->dst,
344                                                rctx->subreq.cryptlen);
345         }
346 
347         /* handle ciphertext stealing */
348         skcipher_request_set_crypt(&rctx->subreq, src, dst,
349                                    req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
350                                    req->iv);
351 
352         err = skcipher_walk_virt(&walk, &rctx->subreq, false);
353         if (err)
354                 return err;
355 
356         kernel_neon_begin();
357         aes_cbc_cts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
358                             ctx->key_dec, rounds, walk.nbytes, walk.iv);
359         kernel_neon_end();
360 
361         return skcipher_walk_done(&walk, 0);
362 }
363 
364 static int ctr_encrypt(struct skcipher_request *req)
365 {
366         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
367         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
368         int err, rounds = 6 + ctx->key_length / 4;
369         struct skcipher_walk walk;
370         int blocks;
371 
372         err = skcipher_walk_virt(&walk, req, false);
373 
374         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
375                 kernel_neon_begin();
376                 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
377                                 ctx->key_enc, rounds, blocks, walk.iv);
378                 kernel_neon_end();
379                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
380         }
381         if (walk.nbytes) {
382                 u8 __aligned(8) tail[AES_BLOCK_SIZE];
383                 unsigned int nbytes = walk.nbytes;
384                 u8 *tdst = walk.dst.virt.addr;
385                 u8 *tsrc = walk.src.virt.addr;
386 
387                 /*
388                  * Tell aes_ctr_encrypt() to process a tail block.
389                  */
390                 blocks = -1;
391 
392                 kernel_neon_begin();
393                 aes_ctr_encrypt(tail, NULL, ctx->key_enc, rounds,
394                                 blocks, walk.iv);
395                 kernel_neon_end();
396                 crypto_xor_cpy(tdst, tsrc, tail, nbytes);
397                 err = skcipher_walk_done(&walk, 0);
398         }
399 
400         return err;
401 }
402 
403 static int ctr_encrypt_sync(struct skcipher_request *req)
404 {
405         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
406         struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
407 
408         if (!may_use_simd())
409                 return aes_ctr_encrypt_fallback(ctx, req);
410 
411         return ctr_encrypt(req);
412 }
413 
414 static int xts_encrypt(struct skcipher_request *req)
415 {
416         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
417         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
418         int err, first, rounds = 6 + ctx->key1.key_length / 4;
419         struct skcipher_walk walk;
420         unsigned int blocks;
421 
422         err = skcipher_walk_virt(&walk, req, false);
423 
424         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
425                 kernel_neon_begin();
426                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
427                                 ctx->key1.key_enc, rounds, blocks,
428                                 ctx->key2.key_enc, walk.iv, first);
429                 kernel_neon_end();
430                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
431         }
432 
433         return err;
434 }
435 
436 static int xts_decrypt(struct skcipher_request *req)
437 {
438         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
439         struct crypto_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
440         int err, first, rounds = 6 + ctx->key1.key_length / 4;
441         struct skcipher_walk walk;
442         unsigned int blocks;
443 
444         err = skcipher_walk_virt(&walk, req, false);
445 
446         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
447                 kernel_neon_begin();
448                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
449                                 ctx->key1.key_dec, rounds, blocks,
450                                 ctx->key2.key_enc, walk.iv, first);
451                 kernel_neon_end();
452                 err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
453         }
454 
455         return err;
456 }
457 
458 static struct skcipher_alg aes_algs[] = { {
459         .base = {
460                 .cra_name               = "__ecb(aes)",
461                 .cra_driver_name        = "__ecb-aes-" MODE,
462                 .cra_priority           = PRIO,
463                 .cra_flags              = CRYPTO_ALG_INTERNAL,
464                 .cra_blocksize          = AES_BLOCK_SIZE,
465                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
466                 .cra_module             = THIS_MODULE,
467         },
468         .min_keysize    = AES_MIN_KEY_SIZE,
469         .max_keysize    = AES_MAX_KEY_SIZE,
470         .setkey         = skcipher_aes_setkey,
471         .encrypt        = ecb_encrypt,
472         .decrypt        = ecb_decrypt,
473 }, {
474         .base = {
475                 .cra_name               = "__cbc(aes)",
476                 .cra_driver_name        = "__cbc-aes-" MODE,
477                 .cra_priority           = PRIO,
478                 .cra_flags              = CRYPTO_ALG_INTERNAL,
479                 .cra_blocksize          = AES_BLOCK_SIZE,
480                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
481                 .cra_module             = THIS_MODULE,
482         },
483         .min_keysize    = AES_MIN_KEY_SIZE,
484         .max_keysize    = AES_MAX_KEY_SIZE,
485         .ivsize         = AES_BLOCK_SIZE,
486         .setkey         = skcipher_aes_setkey,
487         .encrypt        = cbc_encrypt,
488         .decrypt        = cbc_decrypt,
489 }, {
490         .base = {
491                 .cra_name               = "__cts(cbc(aes))",
492                 .cra_driver_name        = "__cts-cbc-aes-" MODE,
493                 .cra_priority           = PRIO,
494                 .cra_flags              = CRYPTO_ALG_INTERNAL,
495                 .cra_blocksize          = AES_BLOCK_SIZE,
496                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
497                 .cra_module             = THIS_MODULE,
498         },
499         .min_keysize    = AES_MIN_KEY_SIZE,
500         .max_keysize    = AES_MAX_KEY_SIZE,
501         .ivsize         = AES_BLOCK_SIZE,
502         .walksize       = 2 * AES_BLOCK_SIZE,
503         .setkey         = skcipher_aes_setkey,
504         .encrypt        = cts_cbc_encrypt,
505         .decrypt        = cts_cbc_decrypt,
506         .init           = cts_cbc_init_tfm,
507 }, {
508         .base = {
509                 .cra_name               = "__ctr(aes)",
510                 .cra_driver_name        = "__ctr-aes-" MODE,
511                 .cra_priority           = PRIO,
512                 .cra_flags              = CRYPTO_ALG_INTERNAL,
513                 .cra_blocksize          = 1,
514                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
515                 .cra_module             = THIS_MODULE,
516         },
517         .min_keysize    = AES_MIN_KEY_SIZE,
518         .max_keysize    = AES_MAX_KEY_SIZE,
519         .ivsize         = AES_BLOCK_SIZE,
520         .chunksize      = AES_BLOCK_SIZE,
521         .setkey         = skcipher_aes_setkey,
522         .encrypt        = ctr_encrypt,
523         .decrypt        = ctr_encrypt,
524 }, {
525         .base = {
526                 .cra_name               = "ctr(aes)",
527                 .cra_driver_name        = "ctr-aes-" MODE,
528                 .cra_priority           = PRIO - 1,
529                 .cra_blocksize          = 1,
530                 .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
531                 .cra_module             = THIS_MODULE,
532         },
533         .min_keysize    = AES_MIN_KEY_SIZE,
534         .max_keysize    = AES_MAX_KEY_SIZE,
535         .ivsize         = AES_BLOCK_SIZE,
536         .chunksize      = AES_BLOCK_SIZE,
537         .setkey         = skcipher_aes_setkey,
538         .encrypt        = ctr_encrypt_sync,
539         .decrypt        = ctr_encrypt_sync,
540 }, {
541         .base = {
542                 .cra_name               = "__xts(aes)",
543                 .cra_driver_name        = "__xts-aes-" MODE,
544                 .cra_priority           = PRIO,
545                 .cra_flags              = CRYPTO_ALG_INTERNAL,
546                 .cra_blocksize          = AES_BLOCK_SIZE,
547                 .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
548                 .cra_module             = THIS_MODULE,
549         },
550         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
551         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
552         .ivsize         = AES_BLOCK_SIZE,
553         .setkey         = xts_set_key,
554         .encrypt        = xts_encrypt,
555         .decrypt        = xts_decrypt,
556 } };
557 
558 static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
559                          unsigned int key_len)
560 {
561         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
562         int err;
563 
564         err = aes_expandkey(&ctx->key, in_key, key_len);
565         if (err)
566                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
567 
568         return err;
569 }
570 
571 static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
572 {
573         u64 a = be64_to_cpu(x->a);
574         u64 b = be64_to_cpu(x->b);
575 
576         y->a = cpu_to_be64((a << 1) | (b >> 63));
577         y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
578 }
579 
580 static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
581                        unsigned int key_len)
582 {
583         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
584         be128 *consts = (be128 *)ctx->consts;
585         int rounds = 6 + key_len / 4;
586         int err;
587 
588         err = cbcmac_setkey(tfm, in_key, key_len);
589         if (err)
590                 return err;
591 
592         /* encrypt the zero vector */
593         kernel_neon_begin();
594         aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, ctx->key.key_enc,
595                         rounds, 1);
596         kernel_neon_end();
597 
598         cmac_gf128_mul_by_x(consts, consts);
599         cmac_gf128_mul_by_x(consts + 1, consts);
600 
601         return 0;
602 }
603 
604 static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
605                        unsigned int key_len)
606 {
607         static u8 const ks[3][AES_BLOCK_SIZE] = {
608                 { [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
609                 { [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
610                 { [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
611         };
612 
613         struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
614         int rounds = 6 + key_len / 4;
615         u8 key[AES_BLOCK_SIZE];
616         int err;
617 
618         err = cbcmac_setkey(tfm, in_key, key_len);
619         if (err)
620                 return err;
621 
622         kernel_neon_begin();
623         aes_ecb_encrypt(key, ks[0], ctx->key.key_enc, rounds, 1);
624         aes_ecb_encrypt(ctx->consts, ks[1], ctx->key.key_enc, rounds, 2);
625         kernel_neon_end();
626 
627         return cbcmac_setkey(tfm, key, sizeof(key));
628 }
629 
630 static int mac_init(struct shash_desc *desc)
631 {
632         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
633 
634         memset(ctx->dg, 0, AES_BLOCK_SIZE);
635         ctx->len = 0;
636 
637         return 0;
638 }
639 
640 static void mac_do_update(struct crypto_aes_ctx *ctx, u8 const in[], int blocks,
641                           u8 dg[], int enc_before, int enc_after)
642 {
643         int rounds = 6 + ctx->key_length / 4;
644 
645         if (may_use_simd()) {
646                 kernel_neon_begin();
647                 aes_mac_update(in, ctx->key_enc, rounds, blocks, dg, enc_before,
648                                enc_after);
649                 kernel_neon_end();
650         } else {
651                 if (enc_before)
652                         __aes_arm64_encrypt(ctx->key_enc, dg, dg, rounds);
653 
654                 while (blocks--) {
655                         crypto_xor(dg, in, AES_BLOCK_SIZE);
656                         in += AES_BLOCK_SIZE;
657 
658                         if (blocks || enc_after)
659                                 __aes_arm64_encrypt(ctx->key_enc, dg, dg,
660                                                     rounds);
661                 }
662         }
663 }
664 
665 static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
666 {
667         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
668         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
669 
670         while (len > 0) {
671                 unsigned int l;
672 
673                 if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
674                     (ctx->len + len) > AES_BLOCK_SIZE) {
675 
676                         int blocks = len / AES_BLOCK_SIZE;
677 
678                         len %= AES_BLOCK_SIZE;
679 
680                         mac_do_update(&tctx->key, p, blocks, ctx->dg,
681                                       (ctx->len != 0), (len != 0));
682 
683                         p += blocks * AES_BLOCK_SIZE;
684 
685                         if (!len) {
686                                 ctx->len = AES_BLOCK_SIZE;
687                                 break;
688                         }
689                         ctx->len = 0;
690                 }
691 
692                 l = min(len, AES_BLOCK_SIZE - ctx->len);
693 
694                 if (l <= AES_BLOCK_SIZE) {
695                         crypto_xor(ctx->dg + ctx->len, p, l);
696                         ctx->len += l;
697                         len -= l;
698                         p += l;
699                 }
700         }
701 
702         return 0;
703 }
704 
705 static int cbcmac_final(struct shash_desc *desc, u8 *out)
706 {
707         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
708         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
709 
710         mac_do_update(&tctx->key, NULL, 0, ctx->dg, 1, 0);
711 
712         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
713 
714         return 0;
715 }
716 
717 static int cmac_final(struct shash_desc *desc, u8 *out)
718 {
719         struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
720         struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
721         u8 *consts = tctx->consts;
722 
723         if (ctx->len != AES_BLOCK_SIZE) {
724                 ctx->dg[ctx->len] ^= 0x80;
725                 consts += AES_BLOCK_SIZE;
726         }
727 
728         mac_do_update(&tctx->key, consts, 1, ctx->dg, 0, 1);
729 
730         memcpy(out, ctx->dg, AES_BLOCK_SIZE);
731 
732         return 0;
733 }
734 
735 static struct shash_alg mac_algs[] = { {
736         .base.cra_name          = "cmac(aes)",
737         .base.cra_driver_name   = "cmac-aes-" MODE,
738         .base.cra_priority      = PRIO,
739         .base.cra_blocksize     = AES_BLOCK_SIZE,
740         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
741                                   2 * AES_BLOCK_SIZE,
742         .base.cra_module        = THIS_MODULE,
743 
744         .digestsize             = AES_BLOCK_SIZE,
745         .init                   = mac_init,
746         .update                 = mac_update,
747         .final                  = cmac_final,
748         .setkey                 = cmac_setkey,
749         .descsize               = sizeof(struct mac_desc_ctx),
750 }, {
751         .base.cra_name          = "xcbc(aes)",
752         .base.cra_driver_name   = "xcbc-aes-" MODE,
753         .base.cra_priority      = PRIO,
754         .base.cra_blocksize     = AES_BLOCK_SIZE,
755         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx) +
756                                   2 * AES_BLOCK_SIZE,
757         .base.cra_module        = THIS_MODULE,
758 
759         .digestsize             = AES_BLOCK_SIZE,
760         .init                   = mac_init,
761         .update                 = mac_update,
762         .final                  = cmac_final,
763         .setkey                 = xcbc_setkey,
764         .descsize               = sizeof(struct mac_desc_ctx),
765 }, {
766         .base.cra_name          = "cbcmac(aes)",
767         .base.cra_driver_name   = "cbcmac-aes-" MODE,
768         .base.cra_priority      = PRIO,
769         .base.cra_blocksize     = 1,
770         .base.cra_ctxsize       = sizeof(struct mac_tfm_ctx),
771         .base.cra_module        = THIS_MODULE,
772 
773         .digestsize             = AES_BLOCK_SIZE,
774         .init                   = mac_init,
775         .update                 = mac_update,
776         .final                  = cbcmac_final,
777         .setkey                 = cbcmac_setkey,
778         .descsize               = sizeof(struct mac_desc_ctx),
779 } };
780 
781 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
782 
783 static void aes_exit(void)
784 {
785         int i;
786 
787         for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
788                 if (aes_simd_algs[i])
789                         simd_skcipher_free(aes_simd_algs[i]);
790 
791         crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
792         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
793 }
794 
795 static int __init aes_init(void)
796 {
797         struct simd_skcipher_alg *simd;
798         const char *basename;
799         const char *algname;
800         const char *drvname;
801         int err;
802         int i;
803 
804         err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
805         if (err)
806                 return err;
807 
808         err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
809         if (err)
810                 goto unregister_ciphers;
811 
812         for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
813                 if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
814                         continue;
815 
816                 algname = aes_algs[i].base.cra_name + 2;
817                 drvname = aes_algs[i].base.cra_driver_name + 2;
818                 basename = aes_algs[i].base.cra_driver_name;
819                 simd = simd_skcipher_create_compat(algname, drvname, basename);
820                 err = PTR_ERR(simd);
821                 if (IS_ERR(simd))
822                         goto unregister_simds;
823 
824                 aes_simd_algs[i] = simd;
825         }
826 
827         return 0;
828 
829 unregister_simds:
830         aes_exit();
831         return err;
832 unregister_ciphers:
833         crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
834         return err;
835 }
836 
837 #ifdef USE_V8_CRYPTO_EXTENSIONS
838 module_cpu_feature_match(AES, aes_init);
839 #else
840 module_init(aes_init);
841 EXPORT_SYMBOL(neon_aes_ecb_encrypt);
842 EXPORT_SYMBOL(neon_aes_cbc_encrypt);
843 #endif
844 module_exit(aes_exit);
845 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp