~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/crypto/aesni-intel_glue.c

Version: ~ [ linux-5.4 ] ~ [ linux-5.3.13 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.86 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.156 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.203 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.202 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.78 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Support for Intel AES-NI instructions. This file contains glue
  3  * code, the real AES implementation is in intel-aes_asm.S.
  4  *
  5  * Copyright (C) 2008, Intel Corp.
  6  *    Author: Huang Ying <ying.huang@intel.com>
  7  *
  8  * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  9  * interface for 64-bit kernels.
 10  *    Authors: Adrian Hoban <adrian.hoban@intel.com>
 11  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
 12  *             Tadeusz Struk (tadeusz.struk@intel.com)
 13  *             Aidan O'Mahony (aidan.o.mahony@intel.com)
 14  *    Copyright (c) 2010, Intel Corporation.
 15  *
 16  * This program is free software; you can redistribute it and/or modify
 17  * it under the terms of the GNU General Public License as published by
 18  * the Free Software Foundation; either version 2 of the License, or
 19  * (at your option) any later version.
 20  */
 21 
 22 #include <linux/hardirq.h>
 23 #include <linux/types.h>
 24 #include <linux/crypto.h>
 25 #include <linux/module.h>
 26 #include <linux/err.h>
 27 #include <crypto/algapi.h>
 28 #include <crypto/aes.h>
 29 #include <crypto/cryptd.h>
 30 #include <crypto/ctr.h>
 31 #include <crypto/b128ops.h>
 32 #include <crypto/lrw.h>
 33 #include <crypto/xts.h>
 34 #include <asm/cpu_device_id.h>
 35 #include <asm/i387.h>
 36 #include <asm/crypto/aes.h>
 37 #include <asm/crypto/ablk_helper.h>
 38 #include <crypto/scatterwalk.h>
 39 #include <crypto/internal/aead.h>
 40 #include <linux/workqueue.h>
 41 #include <linux/spinlock.h>
 42 
 43 #if defined(CONFIG_CRYPTO_CTR) || defined(CONFIG_CRYPTO_CTR_MODULE)
 44 #define HAS_CTR
 45 #endif
 46 
 47 #if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
 48 #define HAS_PCBC
 49 #endif
 50 
 51 /* This data is stored at the end of the crypto_tfm struct.
 52  * It's a type of per "session" data storage location.
 53  * This needs to be 16 byte aligned.
 54  */
 55 struct aesni_rfc4106_gcm_ctx {
 56         u8 hash_subkey[16];
 57         struct crypto_aes_ctx aes_key_expanded;
 58         u8 nonce[4];
 59         struct cryptd_aead *cryptd_tfm;
 60 };
 61 
 62 struct aesni_gcm_set_hash_subkey_result {
 63         int err;
 64         struct completion completion;
 65 };
 66 
 67 struct aesni_hash_subkey_req_data {
 68         u8 iv[16];
 69         struct aesni_gcm_set_hash_subkey_result result;
 70         struct scatterlist sg;
 71 };
 72 
 73 #define AESNI_ALIGN     (16)
 74 #define AES_BLOCK_MASK  (~(AES_BLOCK_SIZE-1))
 75 #define RFC4106_HASH_SUBKEY_SIZE 16
 76 
 77 struct aesni_lrw_ctx {
 78         struct lrw_table_ctx lrw_table;
 79         u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
 80 };
 81 
 82 struct aesni_xts_ctx {
 83         u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
 84         u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
 85 };
 86 
 87 asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
 88                              unsigned int key_len);
 89 asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
 90                           const u8 *in);
 91 asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out,
 92                           const u8 *in);
 93 asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
 94                               const u8 *in, unsigned int len);
 95 asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
 96                               const u8 *in, unsigned int len);
 97 asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
 98                               const u8 *in, unsigned int len, u8 *iv);
 99 asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
100                               const u8 *in, unsigned int len, u8 *iv);
101 
102 int crypto_fpu_init(void);
103 void crypto_fpu_exit(void);
104 
105 #ifdef CONFIG_X86_64
106 asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
107                               const u8 *in, unsigned int len, u8 *iv);
108 
109 /* asmlinkage void aesni_gcm_enc()
110  * void *ctx,  AES Key schedule. Starts on a 16 byte boundary.
111  * u8 *out, Ciphertext output. Encrypt in-place is allowed.
112  * const u8 *in, Plaintext input
113  * unsigned long plaintext_len, Length of data in bytes for encryption.
114  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
115  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
116  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
117  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
118  * const u8 *aad, Additional Authentication Data (AAD)
119  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this
120  *          is going to be 8 or 12 bytes
121  * u8 *auth_tag, Authenticated Tag output.
122  * unsigned long auth_tag_len), Authenticated Tag Length in bytes.
123  *          Valid values are 16 (most likely), 12 or 8.
124  */
125 asmlinkage void aesni_gcm_enc(void *ctx, u8 *out,
126                         const u8 *in, unsigned long plaintext_len, u8 *iv,
127                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
128                         u8 *auth_tag, unsigned long auth_tag_len);
129 
130 /* asmlinkage void aesni_gcm_dec()
131  * void *ctx, AES Key schedule. Starts on a 16 byte boundary.
132  * u8 *out, Plaintext output. Decrypt in-place is allowed.
133  * const u8 *in, Ciphertext input
134  * unsigned long ciphertext_len, Length of data in bytes for decryption.
135  * u8 *iv, Pre-counter block j0: 4 byte salt (from Security Association)
136  *         concatenated with 8 byte Initialisation Vector (from IPSec ESP
137  *         Payload) concatenated with 0x00000001. 16-byte aligned pointer.
138  * u8 *hash_subkey, the Hash sub key input. Data starts on a 16-byte boundary.
139  * const u8 *aad, Additional Authentication Data (AAD)
140  * unsigned long aad_len, Length of AAD in bytes. With RFC4106 this is going
141  * to be 8 or 12 bytes
142  * u8 *auth_tag, Authenticated Tag output.
143  * unsigned long auth_tag_len) Authenticated Tag Length in bytes.
144  * Valid values are 16 (most likely), 12 or 8.
145  */
146 asmlinkage void aesni_gcm_dec(void *ctx, u8 *out,
147                         const u8 *in, unsigned long ciphertext_len, u8 *iv,
148                         u8 *hash_subkey, const u8 *aad, unsigned long aad_len,
149                         u8 *auth_tag, unsigned long auth_tag_len);
150 
151 static inline struct
152 aesni_rfc4106_gcm_ctx *aesni_rfc4106_gcm_ctx_get(struct crypto_aead *tfm)
153 {
154         return
155                 (struct aesni_rfc4106_gcm_ctx *)
156                 PTR_ALIGN((u8 *)
157                 crypto_tfm_ctx(crypto_aead_tfm(tfm)), AESNI_ALIGN);
158 }
159 #endif
160 
161 static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
162 {
163         unsigned long addr = (unsigned long)raw_ctx;
164         unsigned long align = AESNI_ALIGN;
165 
166         if (align <= crypto_tfm_ctx_alignment())
167                 align = 1;
168         return (struct crypto_aes_ctx *)ALIGN(addr, align);
169 }
170 
171 static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx,
172                               const u8 *in_key, unsigned int key_len)
173 {
174         struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx);
175         u32 *flags = &tfm->crt_flags;
176         int err;
177 
178         if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 &&
179             key_len != AES_KEYSIZE_256) {
180                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
181                 return -EINVAL;
182         }
183 
184         if (!irq_fpu_usable())
185                 err = crypto_aes_expand_key(ctx, in_key, key_len);
186         else {
187                 kernel_fpu_begin();
188                 err = aesni_set_key(ctx, in_key, key_len);
189                 kernel_fpu_end();
190         }
191 
192         return err;
193 }
194 
195 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
196                        unsigned int key_len)
197 {
198         return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len);
199 }
200 
201 static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
202 {
203         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
204 
205         if (!irq_fpu_usable())
206                 crypto_aes_encrypt_x86(ctx, dst, src);
207         else {
208                 kernel_fpu_begin();
209                 aesni_enc(ctx, dst, src);
210                 kernel_fpu_end();
211         }
212 }
213 
214 static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
215 {
216         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
217 
218         if (!irq_fpu_usable())
219                 crypto_aes_decrypt_x86(ctx, dst, src);
220         else {
221                 kernel_fpu_begin();
222                 aesni_dec(ctx, dst, src);
223                 kernel_fpu_end();
224         }
225 }
226 
227 static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
228 {
229         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
230 
231         aesni_enc(ctx, dst, src);
232 }
233 
234 static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
235 {
236         struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm));
237 
238         aesni_dec(ctx, dst, src);
239 }
240 
241 static int ecb_encrypt(struct blkcipher_desc *desc,
242                        struct scatterlist *dst, struct scatterlist *src,
243                        unsigned int nbytes)
244 {
245         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
246         struct blkcipher_walk walk;
247         int err;
248 
249         blkcipher_walk_init(&walk, dst, src, nbytes);
250         err = blkcipher_walk_virt(desc, &walk);
251         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
252 
253         kernel_fpu_begin();
254         while ((nbytes = walk.nbytes)) {
255                 aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
256                               nbytes & AES_BLOCK_MASK);
257                 nbytes &= AES_BLOCK_SIZE - 1;
258                 err = blkcipher_walk_done(desc, &walk, nbytes);
259         }
260         kernel_fpu_end();
261 
262         return err;
263 }
264 
265 static int ecb_decrypt(struct blkcipher_desc *desc,
266                        struct scatterlist *dst, struct scatterlist *src,
267                        unsigned int nbytes)
268 {
269         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
270         struct blkcipher_walk walk;
271         int err;
272 
273         blkcipher_walk_init(&walk, dst, src, nbytes);
274         err = blkcipher_walk_virt(desc, &walk);
275         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
276 
277         kernel_fpu_begin();
278         while ((nbytes = walk.nbytes)) {
279                 aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
280                               nbytes & AES_BLOCK_MASK);
281                 nbytes &= AES_BLOCK_SIZE - 1;
282                 err = blkcipher_walk_done(desc, &walk, nbytes);
283         }
284         kernel_fpu_end();
285 
286         return err;
287 }
288 
289 static int cbc_encrypt(struct blkcipher_desc *desc,
290                        struct scatterlist *dst, struct scatterlist *src,
291                        unsigned int nbytes)
292 {
293         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
294         struct blkcipher_walk walk;
295         int err;
296 
297         blkcipher_walk_init(&walk, dst, src, nbytes);
298         err = blkcipher_walk_virt(desc, &walk);
299         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
300 
301         kernel_fpu_begin();
302         while ((nbytes = walk.nbytes)) {
303                 aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
304                               nbytes & AES_BLOCK_MASK, walk.iv);
305                 nbytes &= AES_BLOCK_SIZE - 1;
306                 err = blkcipher_walk_done(desc, &walk, nbytes);
307         }
308         kernel_fpu_end();
309 
310         return err;
311 }
312 
313 static int cbc_decrypt(struct blkcipher_desc *desc,
314                        struct scatterlist *dst, struct scatterlist *src,
315                        unsigned int nbytes)
316 {
317         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
318         struct blkcipher_walk walk;
319         int err;
320 
321         blkcipher_walk_init(&walk, dst, src, nbytes);
322         err = blkcipher_walk_virt(desc, &walk);
323         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
324 
325         kernel_fpu_begin();
326         while ((nbytes = walk.nbytes)) {
327                 aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr,
328                               nbytes & AES_BLOCK_MASK, walk.iv);
329                 nbytes &= AES_BLOCK_SIZE - 1;
330                 err = blkcipher_walk_done(desc, &walk, nbytes);
331         }
332         kernel_fpu_end();
333 
334         return err;
335 }
336 
337 #ifdef CONFIG_X86_64
338 static void ctr_crypt_final(struct crypto_aes_ctx *ctx,
339                             struct blkcipher_walk *walk)
340 {
341         u8 *ctrblk = walk->iv;
342         u8 keystream[AES_BLOCK_SIZE];
343         u8 *src = walk->src.virt.addr;
344         u8 *dst = walk->dst.virt.addr;
345         unsigned int nbytes = walk->nbytes;
346 
347         aesni_enc(ctx, keystream, ctrblk);
348         crypto_xor(keystream, src, nbytes);
349         memcpy(dst, keystream, nbytes);
350         crypto_inc(ctrblk, AES_BLOCK_SIZE);
351 }
352 
353 static int ctr_crypt(struct blkcipher_desc *desc,
354                      struct scatterlist *dst, struct scatterlist *src,
355                      unsigned int nbytes)
356 {
357         struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm));
358         struct blkcipher_walk walk;
359         int err;
360 
361         blkcipher_walk_init(&walk, dst, src, nbytes);
362         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
363         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
364 
365         kernel_fpu_begin();
366         while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
367                 aesni_ctr_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr,
368                               nbytes & AES_BLOCK_MASK, walk.iv);
369                 nbytes &= AES_BLOCK_SIZE - 1;
370                 err = blkcipher_walk_done(desc, &walk, nbytes);
371         }
372         if (walk.nbytes) {
373                 ctr_crypt_final(ctx, &walk);
374                 err = blkcipher_walk_done(desc, &walk, 0);
375         }
376         kernel_fpu_end();
377 
378         return err;
379 }
380 #endif
381 
382 static int ablk_ecb_init(struct crypto_tfm *tfm)
383 {
384         return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
385 }
386 
387 static int ablk_cbc_init(struct crypto_tfm *tfm)
388 {
389         return ablk_init_common(tfm, "__driver-cbc-aes-aesni");
390 }
391 
392 #ifdef CONFIG_X86_64
393 static int ablk_ctr_init(struct crypto_tfm *tfm)
394 {
395         return ablk_init_common(tfm, "__driver-ctr-aes-aesni");
396 }
397 
398 #ifdef HAS_CTR
399 static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
400 {
401         return ablk_init_common(tfm, "rfc3686(__driver-ctr-aes-aesni)");
402 }
403 #endif
404 #endif
405 
406 #ifdef HAS_PCBC
407 static int ablk_pcbc_init(struct crypto_tfm *tfm)
408 {
409         return ablk_init_common(tfm, "fpu(pcbc(__driver-aes-aesni))");
410 }
411 #endif
412 
413 static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
414 {
415         aesni_ecb_enc(ctx, blks, blks, nbytes);
416 }
417 
418 static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
419 {
420         aesni_ecb_dec(ctx, blks, blks, nbytes);
421 }
422 
423 static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
424                             unsigned int keylen)
425 {
426         struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
427         int err;
428 
429         err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
430                                  keylen - AES_BLOCK_SIZE);
431         if (err)
432                 return err;
433 
434         return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
435 }
436 
437 static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
438 {
439         struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
440 
441         lrw_free_table(&ctx->lrw_table);
442 }
443 
444 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
445                        struct scatterlist *src, unsigned int nbytes)
446 {
447         struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
448         be128 buf[8];
449         struct lrw_crypt_req req = {
450                 .tbuf = buf,
451                 .tbuflen = sizeof(buf),
452 
453                 .table_ctx = &ctx->lrw_table,
454                 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
455                 .crypt_fn = lrw_xts_encrypt_callback,
456         };
457         int ret;
458 
459         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
460 
461         kernel_fpu_begin();
462         ret = lrw_crypt(desc, dst, src, nbytes, &req);
463         kernel_fpu_end();
464 
465         return ret;
466 }
467 
468 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
469                        struct scatterlist *src, unsigned int nbytes)
470 {
471         struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
472         be128 buf[8];
473         struct lrw_crypt_req req = {
474                 .tbuf = buf,
475                 .tbuflen = sizeof(buf),
476 
477                 .table_ctx = &ctx->lrw_table,
478                 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
479                 .crypt_fn = lrw_xts_decrypt_callback,
480         };
481         int ret;
482 
483         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
484 
485         kernel_fpu_begin();
486         ret = lrw_crypt(desc, dst, src, nbytes, &req);
487         kernel_fpu_end();
488 
489         return ret;
490 }
491 
492 static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
493                             unsigned int keylen)
494 {
495         struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
496         u32 *flags = &tfm->crt_flags;
497         int err;
498 
499         /* key consists of keys of equal size concatenated, therefore
500          * the length must be even
501          */
502         if (keylen % 2) {
503                 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
504                 return -EINVAL;
505         }
506 
507         /* first half of xts-key is for crypt */
508         err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
509         if (err)
510                 return err;
511 
512         /* second half of xts-key is for tweak */
513         return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
514                                   keylen / 2);
515 }
516 
517 
518 static void aesni_xts_tweak(void *ctx, u8 *out, const u8 *in)
519 {
520         aesni_enc(ctx, out, in);
521 }
522 
523 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
524                        struct scatterlist *src, unsigned int nbytes)
525 {
526         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
527         be128 buf[8];
528         struct xts_crypt_req req = {
529                 .tbuf = buf,
530                 .tbuflen = sizeof(buf),
531 
532                 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
533                 .tweak_fn = aesni_xts_tweak,
534                 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
535                 .crypt_fn = lrw_xts_encrypt_callback,
536         };
537         int ret;
538 
539         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
540 
541         kernel_fpu_begin();
542         ret = xts_crypt(desc, dst, src, nbytes, &req);
543         kernel_fpu_end();
544 
545         return ret;
546 }
547 
548 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
549                        struct scatterlist *src, unsigned int nbytes)
550 {
551         struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
552         be128 buf[8];
553         struct xts_crypt_req req = {
554                 .tbuf = buf,
555                 .tbuflen = sizeof(buf),
556 
557                 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
558                 .tweak_fn = aesni_xts_tweak,
559                 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
560                 .crypt_fn = lrw_xts_decrypt_callback,
561         };
562         int ret;
563 
564         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
565 
566         kernel_fpu_begin();
567         ret = xts_crypt(desc, dst, src, nbytes, &req);
568         kernel_fpu_end();
569 
570         return ret;
571 }
572 
573 #ifdef CONFIG_X86_64
574 static int rfc4106_init(struct crypto_tfm *tfm)
575 {
576         struct cryptd_aead *cryptd_tfm;
577         struct aesni_rfc4106_gcm_ctx *ctx = (struct aesni_rfc4106_gcm_ctx *)
578                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
579         struct crypto_aead *cryptd_child;
580         struct aesni_rfc4106_gcm_ctx *child_ctx;
581         cryptd_tfm = cryptd_alloc_aead("__driver-gcm-aes-aesni", 0, 0);
582         if (IS_ERR(cryptd_tfm))
583                 return PTR_ERR(cryptd_tfm);
584 
585         cryptd_child = cryptd_aead_child(cryptd_tfm);
586         child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child);
587         memcpy(child_ctx, ctx, sizeof(*ctx));
588         ctx->cryptd_tfm = cryptd_tfm;
589         tfm->crt_aead.reqsize = sizeof(struct aead_request)
590                 + crypto_aead_reqsize(&cryptd_tfm->base);
591         return 0;
592 }
593 
594 static void rfc4106_exit(struct crypto_tfm *tfm)
595 {
596         struct aesni_rfc4106_gcm_ctx *ctx =
597                 (struct aesni_rfc4106_gcm_ctx *)
598                 PTR_ALIGN((u8 *)crypto_tfm_ctx(tfm), AESNI_ALIGN);
599         if (!IS_ERR(ctx->cryptd_tfm))
600                 cryptd_free_aead(ctx->cryptd_tfm);
601         return;
602 }
603 
604 static void
605 rfc4106_set_hash_subkey_done(struct crypto_async_request *req, int err)
606 {
607         struct aesni_gcm_set_hash_subkey_result *result = req->data;
608 
609         if (err == -EINPROGRESS)
610                 return;
611         result->err = err;
612         complete(&result->completion);
613 }
614 
615 static int
616 rfc4106_set_hash_subkey(u8 *hash_subkey, const u8 *key, unsigned int key_len)
617 {
618         struct crypto_ablkcipher *ctr_tfm;
619         struct ablkcipher_request *req;
620         int ret = -EINVAL;
621         struct aesni_hash_subkey_req_data *req_data;
622 
623         ctr_tfm = crypto_alloc_ablkcipher("ctr(aes)", 0, 0);
624         if (IS_ERR(ctr_tfm))
625                 return PTR_ERR(ctr_tfm);
626 
627         crypto_ablkcipher_clear_flags(ctr_tfm, ~0);
628 
629         ret = crypto_ablkcipher_setkey(ctr_tfm, key, key_len);
630         if (ret)
631                 goto out_free_ablkcipher;
632 
633         ret = -ENOMEM;
634         req = ablkcipher_request_alloc(ctr_tfm, GFP_KERNEL);
635         if (!req)
636                 goto out_free_ablkcipher;
637 
638         req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
639         if (!req_data)
640                 goto out_free_request;
641 
642         memset(req_data->iv, 0, sizeof(req_data->iv));
643 
644         /* Clear the data in the hash sub key container to zero.*/
645         /* We want to cipher all zeros to create the hash sub key. */
646         memset(hash_subkey, 0, RFC4106_HASH_SUBKEY_SIZE);
647 
648         init_completion(&req_data->result.completion);
649         sg_init_one(&req_data->sg, hash_subkey, RFC4106_HASH_SUBKEY_SIZE);
650         ablkcipher_request_set_tfm(req, ctr_tfm);
651         ablkcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
652                                         CRYPTO_TFM_REQ_MAY_BACKLOG,
653                                         rfc4106_set_hash_subkey_done,
654                                         &req_data->result);
655 
656         ablkcipher_request_set_crypt(req, &req_data->sg,
657                 &req_data->sg, RFC4106_HASH_SUBKEY_SIZE, req_data->iv);
658 
659         ret = crypto_ablkcipher_encrypt(req);
660         if (ret == -EINPROGRESS || ret == -EBUSY) {
661                 ret = wait_for_completion_interruptible
662                         (&req_data->result.completion);
663                 if (!ret)
664                         ret = req_data->result.err;
665         }
666         kfree(req_data);
667 out_free_request:
668         ablkcipher_request_free(req);
669 out_free_ablkcipher:
670         crypto_free_ablkcipher(ctr_tfm);
671         return ret;
672 }
673 
674 static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
675                                                    unsigned int key_len)
676 {
677         int ret = 0;
678         struct crypto_tfm *tfm = crypto_aead_tfm(parent);
679         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
680         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
681         struct aesni_rfc4106_gcm_ctx *child_ctx =
682                                  aesni_rfc4106_gcm_ctx_get(cryptd_child);
683         u8 *new_key_align, *new_key_mem = NULL;
684 
685         if (key_len < 4) {
686                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
687                 return -EINVAL;
688         }
689         /*Account for 4 byte nonce at the end.*/
690         key_len -= 4;
691         if (key_len != AES_KEYSIZE_128) {
692                 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
693                 return -EINVAL;
694         }
695 
696         memcpy(ctx->nonce, key + key_len, sizeof(ctx->nonce));
697         /*This must be on a 16 byte boundary!*/
698         if ((unsigned long)(&(ctx->aes_key_expanded.key_enc[0])) % AESNI_ALIGN)
699                 return -EINVAL;
700 
701         if ((unsigned long)key % AESNI_ALIGN) {
702                 /*key is not aligned: use an auxuliar aligned pointer*/
703                 new_key_mem = kmalloc(key_len+AESNI_ALIGN, GFP_KERNEL);
704                 if (!new_key_mem)
705                         return -ENOMEM;
706 
707                 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
708                 memcpy(new_key_align, key, key_len);
709                 key = new_key_align;
710         }
711 
712         if (!irq_fpu_usable())
713                 ret = crypto_aes_expand_key(&(ctx->aes_key_expanded),
714                 key, key_len);
715         else {
716                 kernel_fpu_begin();
717                 ret = aesni_set_key(&(ctx->aes_key_expanded), key, key_len);
718                 kernel_fpu_end();
719         }
720         /*This must be on a 16 byte boundary!*/
721         if ((unsigned long)(&(ctx->hash_subkey[0])) % AESNI_ALIGN) {
722                 ret = -EINVAL;
723                 goto exit;
724         }
725         ret = rfc4106_set_hash_subkey(ctx->hash_subkey, key, key_len);
726         memcpy(child_ctx, ctx, sizeof(*ctx));
727 exit:
728         kfree(new_key_mem);
729         return ret;
730 }
731 
732 /* This is the Integrity Check Value (aka the authentication tag length and can
733  * be 8, 12 or 16 bytes long. */
734 static int rfc4106_set_authsize(struct crypto_aead *parent,
735                                 unsigned int authsize)
736 {
737         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(parent);
738         struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
739 
740         switch (authsize) {
741         case 8:
742         case 12:
743         case 16:
744                 break;
745         default:
746                 return -EINVAL;
747         }
748         crypto_aead_crt(parent)->authsize = authsize;
749         crypto_aead_crt(cryptd_child)->authsize = authsize;
750         return 0;
751 }
752 
753 static int rfc4106_encrypt(struct aead_request *req)
754 {
755         int ret;
756         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
757         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
758 
759         if (!irq_fpu_usable()) {
760                 struct aead_request *cryptd_req =
761                         (struct aead_request *) aead_request_ctx(req);
762                 memcpy(cryptd_req, req, sizeof(*req));
763                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
764                 return crypto_aead_encrypt(cryptd_req);
765         } else {
766                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
767                 kernel_fpu_begin();
768                 ret = cryptd_child->base.crt_aead.encrypt(req);
769                 kernel_fpu_end();
770                 return ret;
771         }
772 }
773 
774 static int rfc4106_decrypt(struct aead_request *req)
775 {
776         int ret;
777         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
778         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
779 
780         if (!irq_fpu_usable()) {
781                 struct aead_request *cryptd_req =
782                         (struct aead_request *) aead_request_ctx(req);
783                 memcpy(cryptd_req, req, sizeof(*req));
784                 aead_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
785                 return crypto_aead_decrypt(cryptd_req);
786         } else {
787                 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
788                 kernel_fpu_begin();
789                 ret = cryptd_child->base.crt_aead.decrypt(req);
790                 kernel_fpu_end();
791                 return ret;
792         }
793 }
794 
795 static int __driver_rfc4106_encrypt(struct aead_request *req)
796 {
797         u8 one_entry_in_sg = 0;
798         u8 *src, *dst, *assoc;
799         __be32 counter = cpu_to_be32(1);
800         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
801         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
802         void *aes_ctx = &(ctx->aes_key_expanded);
803         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
804         u8 iv_tab[16+AESNI_ALIGN];
805         u8* iv = (u8 *) PTR_ALIGN((u8 *)iv_tab, AESNI_ALIGN);
806         struct scatter_walk src_sg_walk;
807         struct scatter_walk assoc_sg_walk;
808         struct scatter_walk dst_sg_walk;
809         unsigned int i;
810 
811         /* Assuming we are supporting rfc4106 64-bit extended */
812         /* sequence numbers We need to have the AAD length equal */
813         /* to 8 or 12 bytes */
814         if (unlikely(req->assoclen != 8 && req->assoclen != 12))
815                 return -EINVAL;
816         /* IV below built */
817         for (i = 0; i < 4; i++)
818                 *(iv+i) = ctx->nonce[i];
819         for (i = 0; i < 8; i++)
820                 *(iv+4+i) = req->iv[i];
821         *((__be32 *)(iv+12)) = counter;
822 
823         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
824                 one_entry_in_sg = 1;
825                 scatterwalk_start(&src_sg_walk, req->src);
826                 scatterwalk_start(&assoc_sg_walk, req->assoc);
827                 src = scatterwalk_map(&src_sg_walk);
828                 assoc = scatterwalk_map(&assoc_sg_walk);
829                 dst = src;
830                 if (unlikely(req->src != req->dst)) {
831                         scatterwalk_start(&dst_sg_walk, req->dst);
832                         dst = scatterwalk_map(&dst_sg_walk);
833                 }
834 
835         } else {
836                 /* Allocate memory for src, dst, assoc */
837                 src = kmalloc(req->cryptlen + auth_tag_len + req->assoclen,
838                         GFP_ATOMIC);
839                 if (unlikely(!src))
840                         return -ENOMEM;
841                 assoc = (src + req->cryptlen + auth_tag_len);
842                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
843                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
844                                         req->assoclen, 0);
845                 dst = src;
846         }
847 
848         aesni_gcm_enc(aes_ctx, dst, src, (unsigned long)req->cryptlen, iv,
849                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen, dst
850                 + ((unsigned long)req->cryptlen), auth_tag_len);
851 
852         /* The authTag (aka the Integrity Check Value) needs to be written
853          * back to the packet. */
854         if (one_entry_in_sg) {
855                 if (unlikely(req->src != req->dst)) {
856                         scatterwalk_unmap(dst);
857                         scatterwalk_done(&dst_sg_walk, 0, 0);
858                 }
859                 scatterwalk_unmap(src);
860                 scatterwalk_unmap(assoc);
861                 scatterwalk_done(&src_sg_walk, 0, 0);
862                 scatterwalk_done(&assoc_sg_walk, 0, 0);
863         } else {
864                 scatterwalk_map_and_copy(dst, req->dst, 0,
865                         req->cryptlen + auth_tag_len, 1);
866                 kfree(src);
867         }
868         return 0;
869 }
870 
871 static int __driver_rfc4106_decrypt(struct aead_request *req)
872 {
873         u8 one_entry_in_sg = 0;
874         u8 *src, *dst, *assoc;
875         unsigned long tempCipherLen = 0;
876         __be32 counter = cpu_to_be32(1);
877         int retval = 0;
878         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
879         struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm);
880         void *aes_ctx = &(ctx->aes_key_expanded);
881         unsigned long auth_tag_len = crypto_aead_authsize(tfm);
882         u8 iv_and_authTag[32+AESNI_ALIGN];
883         u8 *iv = (u8 *) PTR_ALIGN((u8 *)iv_and_authTag, AESNI_ALIGN);
884         u8 *authTag = iv + 16;
885         struct scatter_walk src_sg_walk;
886         struct scatter_walk assoc_sg_walk;
887         struct scatter_walk dst_sg_walk;
888         unsigned int i;
889 
890         if (unlikely((req->cryptlen < auth_tag_len) ||
891                 (req->assoclen != 8 && req->assoclen != 12)))
892                 return -EINVAL;
893         /* Assuming we are supporting rfc4106 64-bit extended */
894         /* sequence numbers We need to have the AAD length */
895         /* equal to 8 or 12 bytes */
896 
897         tempCipherLen = (unsigned long)(req->cryptlen - auth_tag_len);
898         /* IV below built */
899         for (i = 0; i < 4; i++)
900                 *(iv+i) = ctx->nonce[i];
901         for (i = 0; i < 8; i++)
902                 *(iv+4+i) = req->iv[i];
903         *((__be32 *)(iv+12)) = counter;
904 
905         if ((sg_is_last(req->src)) && (sg_is_last(req->assoc))) {
906                 one_entry_in_sg = 1;
907                 scatterwalk_start(&src_sg_walk, req->src);
908                 scatterwalk_start(&assoc_sg_walk, req->assoc);
909                 src = scatterwalk_map(&src_sg_walk);
910                 assoc = scatterwalk_map(&assoc_sg_walk);
911                 dst = src;
912                 if (unlikely(req->src != req->dst)) {
913                         scatterwalk_start(&dst_sg_walk, req->dst);
914                         dst = scatterwalk_map(&dst_sg_walk);
915                 }
916 
917         } else {
918                 /* Allocate memory for src, dst, assoc */
919                 src = kmalloc(req->cryptlen + req->assoclen, GFP_ATOMIC);
920                 if (!src)
921                         return -ENOMEM;
922                 assoc = (src + req->cryptlen + auth_tag_len);
923                 scatterwalk_map_and_copy(src, req->src, 0, req->cryptlen, 0);
924                 scatterwalk_map_and_copy(assoc, req->assoc, 0,
925                         req->assoclen, 0);
926                 dst = src;
927         }
928 
929         aesni_gcm_dec(aes_ctx, dst, src, tempCipherLen, iv,
930                 ctx->hash_subkey, assoc, (unsigned long)req->assoclen,
931                 authTag, auth_tag_len);
932 
933         /* Compare generated tag with passed in tag. */
934         retval = memcmp(src + tempCipherLen, authTag, auth_tag_len) ?
935                 -EBADMSG : 0;
936 
937         if (one_entry_in_sg) {
938                 if (unlikely(req->src != req->dst)) {
939                         scatterwalk_unmap(dst);
940                         scatterwalk_done(&dst_sg_walk, 0, 0);
941                 }
942                 scatterwalk_unmap(src);
943                 scatterwalk_unmap(assoc);
944                 scatterwalk_done(&src_sg_walk, 0, 0);
945                 scatterwalk_done(&assoc_sg_walk, 0, 0);
946         } else {
947                 scatterwalk_map_and_copy(dst, req->dst, 0, req->cryptlen, 1);
948                 kfree(src);
949         }
950         return retval;
951 }
952 #endif
953 
954 static struct crypto_alg aesni_algs[] = { {
955         .cra_name               = "aes",
956         .cra_driver_name        = "aes-aesni",
957         .cra_priority           = 300,
958         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
959         .cra_blocksize          = AES_BLOCK_SIZE,
960         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
961                                   AESNI_ALIGN - 1,
962         .cra_alignmask          = 0,
963         .cra_module             = THIS_MODULE,
964         .cra_u  = {
965                 .cipher = {
966                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
967                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
968                         .cia_setkey             = aes_set_key,
969                         .cia_encrypt            = aes_encrypt,
970                         .cia_decrypt            = aes_decrypt
971                 }
972         }
973 }, {
974         .cra_name               = "__aes-aesni",
975         .cra_driver_name        = "__driver-aes-aesni",
976         .cra_priority           = 0,
977         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
978         .cra_blocksize          = AES_BLOCK_SIZE,
979         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
980                                   AESNI_ALIGN - 1,
981         .cra_alignmask          = 0,
982         .cra_module             = THIS_MODULE,
983         .cra_u  = {
984                 .cipher = {
985                         .cia_min_keysize        = AES_MIN_KEY_SIZE,
986                         .cia_max_keysize        = AES_MAX_KEY_SIZE,
987                         .cia_setkey             = aes_set_key,
988                         .cia_encrypt            = __aes_encrypt,
989                         .cia_decrypt            = __aes_decrypt
990                 }
991         }
992 }, {
993         .cra_name               = "__ecb-aes-aesni",
994         .cra_driver_name        = "__driver-ecb-aes-aesni",
995         .cra_priority           = 0,
996         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
997         .cra_blocksize          = AES_BLOCK_SIZE,
998         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
999                                   AESNI_ALIGN - 1,
1000         .cra_alignmask          = 0,
1001         .cra_type               = &crypto_blkcipher_type,
1002         .cra_module             = THIS_MODULE,
1003         .cra_u = {
1004                 .blkcipher = {
1005                         .min_keysize    = AES_MIN_KEY_SIZE,
1006                         .max_keysize    = AES_MAX_KEY_SIZE,
1007                         .setkey         = aes_set_key,
1008                         .encrypt        = ecb_encrypt,
1009                         .decrypt        = ecb_decrypt,
1010                 },
1011         },
1012 }, {
1013         .cra_name               = "__cbc-aes-aesni",
1014         .cra_driver_name        = "__driver-cbc-aes-aesni",
1015         .cra_priority           = 0,
1016         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1017         .cra_blocksize          = AES_BLOCK_SIZE,
1018         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1019                                   AESNI_ALIGN - 1,
1020         .cra_alignmask          = 0,
1021         .cra_type               = &crypto_blkcipher_type,
1022         .cra_module             = THIS_MODULE,
1023         .cra_u = {
1024                 .blkcipher = {
1025                         .min_keysize    = AES_MIN_KEY_SIZE,
1026                         .max_keysize    = AES_MAX_KEY_SIZE,
1027                         .setkey         = aes_set_key,
1028                         .encrypt        = cbc_encrypt,
1029                         .decrypt        = cbc_decrypt,
1030                 },
1031         },
1032 }, {
1033         .cra_name               = "ecb(aes)",
1034         .cra_driver_name        = "ecb-aes-aesni",
1035         .cra_priority           = 400,
1036         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1037         .cra_blocksize          = AES_BLOCK_SIZE,
1038         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1039         .cra_alignmask          = 0,
1040         .cra_type               = &crypto_ablkcipher_type,
1041         .cra_module             = THIS_MODULE,
1042         .cra_init               = ablk_ecb_init,
1043         .cra_exit               = ablk_exit,
1044         .cra_u = {
1045                 .ablkcipher = {
1046                         .min_keysize    = AES_MIN_KEY_SIZE,
1047                         .max_keysize    = AES_MAX_KEY_SIZE,
1048                         .setkey         = ablk_set_key,
1049                         .encrypt        = ablk_encrypt,
1050                         .decrypt        = ablk_decrypt,
1051                 },
1052         },
1053 }, {
1054         .cra_name               = "cbc(aes)",
1055         .cra_driver_name        = "cbc-aes-aesni",
1056         .cra_priority           = 400,
1057         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1058         .cra_blocksize          = AES_BLOCK_SIZE,
1059         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1060         .cra_alignmask          = 0,
1061         .cra_type               = &crypto_ablkcipher_type,
1062         .cra_module             = THIS_MODULE,
1063         .cra_init               = ablk_cbc_init,
1064         .cra_exit               = ablk_exit,
1065         .cra_u = {
1066                 .ablkcipher = {
1067                         .min_keysize    = AES_MIN_KEY_SIZE,
1068                         .max_keysize    = AES_MAX_KEY_SIZE,
1069                         .ivsize         = AES_BLOCK_SIZE,
1070                         .setkey         = ablk_set_key,
1071                         .encrypt        = ablk_encrypt,
1072                         .decrypt        = ablk_decrypt,
1073                 },
1074         },
1075 #ifdef CONFIG_X86_64
1076 }, {
1077         .cra_name               = "__ctr-aes-aesni",
1078         .cra_driver_name        = "__driver-ctr-aes-aesni",
1079         .cra_priority           = 0,
1080         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1081         .cra_blocksize          = 1,
1082         .cra_ctxsize            = sizeof(struct crypto_aes_ctx) +
1083                                   AESNI_ALIGN - 1,
1084         .cra_alignmask          = 0,
1085         .cra_type               = &crypto_blkcipher_type,
1086         .cra_module             = THIS_MODULE,
1087         .cra_u = {
1088                 .blkcipher = {
1089                         .min_keysize    = AES_MIN_KEY_SIZE,
1090                         .max_keysize    = AES_MAX_KEY_SIZE,
1091                         .ivsize         = AES_BLOCK_SIZE,
1092                         .setkey         = aes_set_key,
1093                         .encrypt        = ctr_crypt,
1094                         .decrypt        = ctr_crypt,
1095                 },
1096         },
1097 }, {
1098         .cra_name               = "ctr(aes)",
1099         .cra_driver_name        = "ctr-aes-aesni",
1100         .cra_priority           = 400,
1101         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1102         .cra_blocksize          = 1,
1103         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1104         .cra_alignmask          = 0,
1105         .cra_type               = &crypto_ablkcipher_type,
1106         .cra_module             = THIS_MODULE,
1107         .cra_init               = ablk_ctr_init,
1108         .cra_exit               = ablk_exit,
1109         .cra_u = {
1110                 .ablkcipher = {
1111                         .min_keysize    = AES_MIN_KEY_SIZE,
1112                         .max_keysize    = AES_MAX_KEY_SIZE,
1113                         .ivsize         = AES_BLOCK_SIZE,
1114                         .setkey         = ablk_set_key,
1115                         .encrypt        = ablk_encrypt,
1116                         .decrypt        = ablk_encrypt,
1117                         .geniv          = "chainiv",
1118                 },
1119         },
1120 }, {
1121         .cra_name               = "__gcm-aes-aesni",
1122         .cra_driver_name        = "__driver-gcm-aes-aesni",
1123         .cra_priority           = 0,
1124         .cra_flags              = CRYPTO_ALG_TYPE_AEAD,
1125         .cra_blocksize          = 1,
1126         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
1127                                   AESNI_ALIGN,
1128         .cra_alignmask          = 0,
1129         .cra_type               = &crypto_aead_type,
1130         .cra_module             = THIS_MODULE,
1131         .cra_u = {
1132                 .aead = {
1133                         .encrypt        = __driver_rfc4106_encrypt,
1134                         .decrypt        = __driver_rfc4106_decrypt,
1135                 },
1136         },
1137 }, {
1138         .cra_name               = "rfc4106(gcm(aes))",
1139         .cra_driver_name        = "rfc4106-gcm-aesni",
1140         .cra_priority           = 400,
1141         .cra_flags              = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1142         .cra_blocksize          = 1,
1143         .cra_ctxsize            = sizeof(struct aesni_rfc4106_gcm_ctx) +
1144                                   AESNI_ALIGN,
1145         .cra_alignmask          = 0,
1146         .cra_type               = &crypto_nivaead_type,
1147         .cra_module             = THIS_MODULE,
1148         .cra_init               = rfc4106_init,
1149         .cra_exit               = rfc4106_exit,
1150         .cra_u = {
1151                 .aead = {
1152                         .setkey         = rfc4106_set_key,
1153                         .setauthsize    = rfc4106_set_authsize,
1154                         .encrypt        = rfc4106_encrypt,
1155                         .decrypt        = rfc4106_decrypt,
1156                         .geniv          = "seqiv",
1157                         .ivsize         = 8,
1158                         .maxauthsize    = 16,
1159                 },
1160         },
1161 #ifdef HAS_CTR
1162 }, {
1163         .cra_name               = "rfc3686(ctr(aes))",
1164         .cra_driver_name        = "rfc3686-ctr-aes-aesni",
1165         .cra_priority           = 400,
1166         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1167         .cra_blocksize          = 1,
1168         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1169         .cra_alignmask          = 0,
1170         .cra_type               = &crypto_ablkcipher_type,
1171         .cra_module             = THIS_MODULE,
1172         .cra_init               = ablk_rfc3686_ctr_init,
1173         .cra_exit               = ablk_exit,
1174         .cra_u = {
1175                 .ablkcipher = {
1176                         .min_keysize = AES_MIN_KEY_SIZE +
1177                                        CTR_RFC3686_NONCE_SIZE,
1178                         .max_keysize = AES_MAX_KEY_SIZE +
1179                                        CTR_RFC3686_NONCE_SIZE,
1180                         .ivsize      = CTR_RFC3686_IV_SIZE,
1181                         .setkey      = ablk_set_key,
1182                         .encrypt     = ablk_encrypt,
1183                         .decrypt     = ablk_decrypt,
1184                         .geniv       = "seqiv",
1185                 },
1186         },
1187 #endif
1188 #endif
1189 #ifdef HAS_PCBC
1190 }, {
1191         .cra_name               = "pcbc(aes)",
1192         .cra_driver_name        = "pcbc-aes-aesni",
1193         .cra_priority           = 400,
1194         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1195         .cra_blocksize          = AES_BLOCK_SIZE,
1196         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1197         .cra_alignmask          = 0,
1198         .cra_type               = &crypto_ablkcipher_type,
1199         .cra_module             = THIS_MODULE,
1200         .cra_init               = ablk_pcbc_init,
1201         .cra_exit               = ablk_exit,
1202         .cra_u = {
1203                 .ablkcipher = {
1204                         .min_keysize    = AES_MIN_KEY_SIZE,
1205                         .max_keysize    = AES_MAX_KEY_SIZE,
1206                         .ivsize         = AES_BLOCK_SIZE,
1207                         .setkey         = ablk_set_key,
1208                         .encrypt        = ablk_encrypt,
1209                         .decrypt        = ablk_decrypt,
1210                 },
1211         },
1212 #endif
1213 }, {
1214         .cra_name               = "__lrw-aes-aesni",
1215         .cra_driver_name        = "__driver-lrw-aes-aesni",
1216         .cra_priority           = 0,
1217         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1218         .cra_blocksize          = AES_BLOCK_SIZE,
1219         .cra_ctxsize            = sizeof(struct aesni_lrw_ctx),
1220         .cra_alignmask          = 0,
1221         .cra_type               = &crypto_blkcipher_type,
1222         .cra_module             = THIS_MODULE,
1223         .cra_exit               = lrw_aesni_exit_tfm,
1224         .cra_u = {
1225                 .blkcipher = {
1226                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1227                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1228                         .ivsize         = AES_BLOCK_SIZE,
1229                         .setkey         = lrw_aesni_setkey,
1230                         .encrypt        = lrw_encrypt,
1231                         .decrypt        = lrw_decrypt,
1232                 },
1233         },
1234 }, {
1235         .cra_name               = "__xts-aes-aesni",
1236         .cra_driver_name        = "__driver-xts-aes-aesni",
1237         .cra_priority           = 0,
1238         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
1239         .cra_blocksize          = AES_BLOCK_SIZE,
1240         .cra_ctxsize            = sizeof(struct aesni_xts_ctx),
1241         .cra_alignmask          = 0,
1242         .cra_type               = &crypto_blkcipher_type,
1243         .cra_module             = THIS_MODULE,
1244         .cra_u = {
1245                 .blkcipher = {
1246                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1247                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1248                         .ivsize         = AES_BLOCK_SIZE,
1249                         .setkey         = xts_aesni_setkey,
1250                         .encrypt        = xts_encrypt,
1251                         .decrypt        = xts_decrypt,
1252                 },
1253         },
1254 }, {
1255         .cra_name               = "lrw(aes)",
1256         .cra_driver_name        = "lrw-aes-aesni",
1257         .cra_priority           = 400,
1258         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1259         .cra_blocksize          = AES_BLOCK_SIZE,
1260         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1261         .cra_alignmask          = 0,
1262         .cra_type               = &crypto_ablkcipher_type,
1263         .cra_module             = THIS_MODULE,
1264         .cra_init               = ablk_init,
1265         .cra_exit               = ablk_exit,
1266         .cra_u = {
1267                 .ablkcipher = {
1268                         .min_keysize    = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1269                         .max_keysize    = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1270                         .ivsize         = AES_BLOCK_SIZE,
1271                         .setkey         = ablk_set_key,
1272                         .encrypt        = ablk_encrypt,
1273                         .decrypt        = ablk_decrypt,
1274                 },
1275         },
1276 }, {
1277         .cra_name               = "xts(aes)",
1278         .cra_driver_name        = "xts-aes-aesni",
1279         .cra_priority           = 400,
1280         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1281         .cra_blocksize          = AES_BLOCK_SIZE,
1282         .cra_ctxsize            = sizeof(struct async_helper_ctx),
1283         .cra_alignmask          = 0,
1284         .cra_type               = &crypto_ablkcipher_type,
1285         .cra_module             = THIS_MODULE,
1286         .cra_init               = ablk_init,
1287         .cra_exit               = ablk_exit,
1288         .cra_u = {
1289                 .ablkcipher = {
1290                         .min_keysize    = 2 * AES_MIN_KEY_SIZE,
1291                         .max_keysize    = 2 * AES_MAX_KEY_SIZE,
1292                         .ivsize         = AES_BLOCK_SIZE,
1293                         .setkey         = ablk_set_key,
1294                         .encrypt        = ablk_encrypt,
1295                         .decrypt        = ablk_decrypt,
1296                 },
1297         },
1298 } };
1299 
1300 
1301 static const struct x86_cpu_id aesni_cpu_id[] = {
1302         X86_FEATURE_MATCH(X86_FEATURE_AES),
1303         {}
1304 };
1305 MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1306 
1307 static int __init aesni_init(void)
1308 {
1309         int err;
1310 
1311         if (!x86_match_cpu(aesni_cpu_id))
1312                 return -ENODEV;
1313 
1314         err = crypto_fpu_init();
1315         if (err)
1316                 return err;
1317 
1318         return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1319 }
1320 
1321 static void __exit aesni_exit(void)
1322 {
1323         crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1324 
1325         crypto_fpu_exit();
1326 }
1327 
1328 module_init(aesni_init);
1329 module_exit(aesni_exit);
1330 
1331 MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized");
1332 MODULE_LICENSE("GPL");
1333 MODULE_ALIAS("aes");
1334 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp