~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/crypto/serpent_avx2_glue.c

Version: ~ [ linux-5.12-rc7 ] ~ [ linux-5.11.13 ] ~ [ linux-5.10.29 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.111 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.186 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.230 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.266 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.266 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.18.140 ] ~ [ linux-3.16.85 ] ~ [ linux-3.14.79 ] ~ [ linux-3.12.74 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
  3  *
  4  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  */
 12 
 13 #include <linux/module.h>
 14 #include <linux/types.h>
 15 #include <linux/crypto.h>
 16 #include <linux/err.h>
 17 #include <crypto/ablk_helper.h>
 18 #include <crypto/algapi.h>
 19 #include <crypto/ctr.h>
 20 #include <crypto/lrw.h>
 21 #include <crypto/xts.h>
 22 #include <crypto/serpent.h>
 23 #include <asm/xcr.h>
 24 #include <asm/xsave.h>
 25 #include <asm/crypto/serpent-avx.h>
 26 #include <asm/crypto/glue_helper.h>
 27 
 28 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
 29 
 30 /* 16-way AVX2 parallel cipher functions */
 31 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
 32                                       const u8 *src);
 33 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
 34                                       const u8 *src);
 35 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
 36 
 37 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
 38                                   le128 *iv);
 39 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
 40                                       const u8 *src, le128 *iv);
 41 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
 42                                       const u8 *src, le128 *iv);
 43 
 44 static const struct common_glue_ctx serpent_enc = {
 45         .num_funcs = 3,
 46         .fpu_blocks_limit = 8,
 47 
 48         .funcs = { {
 49                 .num_blocks = 16,
 50                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
 51         }, {
 52                 .num_blocks = 8,
 53                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
 54         }, {
 55                 .num_blocks = 1,
 56                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
 57         } }
 58 };
 59 
 60 static const struct common_glue_ctx serpent_ctr = {
 61         .num_funcs = 3,
 62         .fpu_blocks_limit = 8,
 63 
 64         .funcs = { {
 65                 .num_blocks = 16,
 66                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
 67         },  {
 68                 .num_blocks = 8,
 69                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
 70         }, {
 71                 .num_blocks = 1,
 72                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
 73         } }
 74 };
 75 
 76 static const struct common_glue_ctx serpent_enc_xts = {
 77         .num_funcs = 3,
 78         .fpu_blocks_limit = 8,
 79 
 80         .funcs = { {
 81                 .num_blocks = 16,
 82                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
 83         }, {
 84                 .num_blocks = 8,
 85                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
 86         }, {
 87                 .num_blocks = 1,
 88                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
 89         } }
 90 };
 91 
 92 static const struct common_glue_ctx serpent_dec = {
 93         .num_funcs = 3,
 94         .fpu_blocks_limit = 8,
 95 
 96         .funcs = { {
 97                 .num_blocks = 16,
 98                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
 99         }, {
100                 .num_blocks = 8,
101                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
102         }, {
103                 .num_blocks = 1,
104                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
105         } }
106 };
107 
108 static const struct common_glue_ctx serpent_dec_cbc = {
109         .num_funcs = 3,
110         .fpu_blocks_limit = 8,
111 
112         .funcs = { {
113                 .num_blocks = 16,
114                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
115         }, {
116                 .num_blocks = 8,
117                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
118         }, {
119                 .num_blocks = 1,
120                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
121         } }
122 };
123 
124 static const struct common_glue_ctx serpent_dec_xts = {
125         .num_funcs = 3,
126         .fpu_blocks_limit = 8,
127 
128         .funcs = { {
129                 .num_blocks = 16,
130                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
131         }, {
132                 .num_blocks = 8,
133                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
134         }, {
135                 .num_blocks = 1,
136                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
137         } }
138 };
139 
140 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
141                        struct scatterlist *src, unsigned int nbytes)
142 {
143         return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
144 }
145 
146 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
147                        struct scatterlist *src, unsigned int nbytes)
148 {
149         return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
150 }
151 
152 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
153                        struct scatterlist *src, unsigned int nbytes)
154 {
155         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
156                                        dst, src, nbytes);
157 }
158 
159 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
160                        struct scatterlist *src, unsigned int nbytes)
161 {
162         return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
163                                        nbytes);
164 }
165 
166 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167                      struct scatterlist *src, unsigned int nbytes)
168 {
169         return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
170 }
171 
172 static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
173 {
174         /* since reusing AVX functions, starts using FPU at 8 parallel blocks */
175         return glue_fpu_begin(SERPENT_BLOCK_SIZE, 8, NULL, fpu_enabled, nbytes);
176 }
177 
178 static inline void serpent_fpu_end(bool fpu_enabled)
179 {
180         glue_fpu_end(fpu_enabled);
181 }
182 
183 struct crypt_priv {
184         struct serpent_ctx *ctx;
185         bool fpu_enabled;
186 };
187 
188 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
189 {
190         const unsigned int bsize = SERPENT_BLOCK_SIZE;
191         struct crypt_priv *ctx = priv;
192         int i;
193 
194         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
195 
196         if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
197                 serpent_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
198                 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
199                 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
200         }
201 
202         while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
203                 serpent_ecb_enc_8way_avx(ctx->ctx, srcdst, srcdst);
204                 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
205                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
206         }
207 
208         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
209                 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
210 }
211 
212 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
213 {
214         const unsigned int bsize = SERPENT_BLOCK_SIZE;
215         struct crypt_priv *ctx = priv;
216         int i;
217 
218         ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
219 
220         if (nbytes >= SERPENT_AVX2_PARALLEL_BLOCKS * bsize) {
221                 serpent_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
222                 srcdst += bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
223                 nbytes -= bsize * SERPENT_AVX2_PARALLEL_BLOCKS;
224         }
225 
226         while (nbytes >= SERPENT_PARALLEL_BLOCKS * bsize) {
227                 serpent_ecb_dec_8way_avx(ctx->ctx, srcdst, srcdst);
228                 srcdst += bsize * SERPENT_PARALLEL_BLOCKS;
229                 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
230         }
231 
232         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
233                 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
234 }
235 
236 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
237                        struct scatterlist *src, unsigned int nbytes)
238 {
239         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
240         be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
241         struct crypt_priv crypt_ctx = {
242                 .ctx = &ctx->serpent_ctx,
243                 .fpu_enabled = false,
244         };
245         struct lrw_crypt_req req = {
246                 .tbuf = buf,
247                 .tbuflen = sizeof(buf),
248 
249                 .table_ctx = &ctx->lrw_table,
250                 .crypt_ctx = &crypt_ctx,
251                 .crypt_fn = encrypt_callback,
252         };
253         int ret;
254 
255         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
256         ret = lrw_crypt(desc, dst, src, nbytes, &req);
257         serpent_fpu_end(crypt_ctx.fpu_enabled);
258 
259         return ret;
260 }
261 
262 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
263                        struct scatterlist *src, unsigned int nbytes)
264 {
265         struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266         be128 buf[SERPENT_AVX2_PARALLEL_BLOCKS];
267         struct crypt_priv crypt_ctx = {
268                 .ctx = &ctx->serpent_ctx,
269                 .fpu_enabled = false,
270         };
271         struct lrw_crypt_req req = {
272                 .tbuf = buf,
273                 .tbuflen = sizeof(buf),
274 
275                 .table_ctx = &ctx->lrw_table,
276                 .crypt_ctx = &crypt_ctx,
277                 .crypt_fn = decrypt_callback,
278         };
279         int ret;
280 
281         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
282         ret = lrw_crypt(desc, dst, src, nbytes, &req);
283         serpent_fpu_end(crypt_ctx.fpu_enabled);
284 
285         return ret;
286 }
287 
288 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
289                        struct scatterlist *src, unsigned int nbytes)
290 {
291         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
292 
293         return glue_xts_crypt_128bit(&serpent_enc_xts, desc, dst, src, nbytes,
294                                      XTS_TWEAK_CAST(__serpent_encrypt),
295                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
296 }
297 
298 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
299                        struct scatterlist *src, unsigned int nbytes)
300 {
301         struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
302 
303         return glue_xts_crypt_128bit(&serpent_dec_xts, desc, dst, src, nbytes,
304                                      XTS_TWEAK_CAST(__serpent_encrypt),
305                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
306 }
307 
308 static struct crypto_alg srp_algs[10] = { {
309         .cra_name               = "__ecb-serpent-avx2",
310         .cra_driver_name        = "__driver-ecb-serpent-avx2",
311         .cra_priority           = 0,
312         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
313         .cra_blocksize          = SERPENT_BLOCK_SIZE,
314         .cra_ctxsize            = sizeof(struct serpent_ctx),
315         .cra_alignmask          = 0,
316         .cra_type               = &crypto_blkcipher_type,
317         .cra_module             = THIS_MODULE,
318         .cra_list               = LIST_HEAD_INIT(srp_algs[0].cra_list),
319         .cra_u = {
320                 .blkcipher = {
321                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
322                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
323                         .setkey         = serpent_setkey,
324                         .encrypt        = ecb_encrypt,
325                         .decrypt        = ecb_decrypt,
326                 },
327         },
328 }, {
329         .cra_name               = "__cbc-serpent-avx2",
330         .cra_driver_name        = "__driver-cbc-serpent-avx2",
331         .cra_priority           = 0,
332         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
333         .cra_blocksize          = SERPENT_BLOCK_SIZE,
334         .cra_ctxsize            = sizeof(struct serpent_ctx),
335         .cra_alignmask          = 0,
336         .cra_type               = &crypto_blkcipher_type,
337         .cra_module             = THIS_MODULE,
338         .cra_list               = LIST_HEAD_INIT(srp_algs[1].cra_list),
339         .cra_u = {
340                 .blkcipher = {
341                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
342                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
343                         .setkey         = serpent_setkey,
344                         .encrypt        = cbc_encrypt,
345                         .decrypt        = cbc_decrypt,
346                 },
347         },
348 }, {
349         .cra_name               = "__ctr-serpent-avx2",
350         .cra_driver_name        = "__driver-ctr-serpent-avx2",
351         .cra_priority           = 0,
352         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
353         .cra_blocksize          = 1,
354         .cra_ctxsize            = sizeof(struct serpent_ctx),
355         .cra_alignmask          = 0,
356         .cra_type               = &crypto_blkcipher_type,
357         .cra_module             = THIS_MODULE,
358         .cra_list               = LIST_HEAD_INIT(srp_algs[2].cra_list),
359         .cra_u = {
360                 .blkcipher = {
361                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
362                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
363                         .ivsize         = SERPENT_BLOCK_SIZE,
364                         .setkey         = serpent_setkey,
365                         .encrypt        = ctr_crypt,
366                         .decrypt        = ctr_crypt,
367                 },
368         },
369 }, {
370         .cra_name               = "__lrw-serpent-avx2",
371         .cra_driver_name        = "__driver-lrw-serpent-avx2",
372         .cra_priority           = 0,
373         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
374         .cra_blocksize          = SERPENT_BLOCK_SIZE,
375         .cra_ctxsize            = sizeof(struct serpent_lrw_ctx),
376         .cra_alignmask          = 0,
377         .cra_type               = &crypto_blkcipher_type,
378         .cra_module             = THIS_MODULE,
379         .cra_list               = LIST_HEAD_INIT(srp_algs[3].cra_list),
380         .cra_exit               = lrw_serpent_exit_tfm,
381         .cra_u = {
382                 .blkcipher = {
383                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
384                                           SERPENT_BLOCK_SIZE,
385                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
386                                           SERPENT_BLOCK_SIZE,
387                         .ivsize         = SERPENT_BLOCK_SIZE,
388                         .setkey         = lrw_serpent_setkey,
389                         .encrypt        = lrw_encrypt,
390                         .decrypt        = lrw_decrypt,
391                 },
392         },
393 }, {
394         .cra_name               = "__xts-serpent-avx2",
395         .cra_driver_name        = "__driver-xts-serpent-avx2",
396         .cra_priority           = 0,
397         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
398         .cra_blocksize          = SERPENT_BLOCK_SIZE,
399         .cra_ctxsize            = sizeof(struct serpent_xts_ctx),
400         .cra_alignmask          = 0,
401         .cra_type               = &crypto_blkcipher_type,
402         .cra_module             = THIS_MODULE,
403         .cra_list               = LIST_HEAD_INIT(srp_algs[4].cra_list),
404         .cra_u = {
405                 .blkcipher = {
406                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
407                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
408                         .ivsize         = SERPENT_BLOCK_SIZE,
409                         .setkey         = xts_serpent_setkey,
410                         .encrypt        = xts_encrypt,
411                         .decrypt        = xts_decrypt,
412                 },
413         },
414 }, {
415         .cra_name               = "ecb(serpent)",
416         .cra_driver_name        = "ecb-serpent-avx2",
417         .cra_priority           = 600,
418         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
419         .cra_blocksize          = SERPENT_BLOCK_SIZE,
420         .cra_ctxsize            = sizeof(struct async_helper_ctx),
421         .cra_alignmask          = 0,
422         .cra_type               = &crypto_ablkcipher_type,
423         .cra_module             = THIS_MODULE,
424         .cra_list               = LIST_HEAD_INIT(srp_algs[5].cra_list),
425         .cra_init               = ablk_init,
426         .cra_exit               = ablk_exit,
427         .cra_u = {
428                 .ablkcipher = {
429                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
430                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
431                         .setkey         = ablk_set_key,
432                         .encrypt        = ablk_encrypt,
433                         .decrypt        = ablk_decrypt,
434                 },
435         },
436 }, {
437         .cra_name               = "cbc(serpent)",
438         .cra_driver_name        = "cbc-serpent-avx2",
439         .cra_priority           = 600,
440         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
441         .cra_blocksize          = SERPENT_BLOCK_SIZE,
442         .cra_ctxsize            = sizeof(struct async_helper_ctx),
443         .cra_alignmask          = 0,
444         .cra_type               = &crypto_ablkcipher_type,
445         .cra_module             = THIS_MODULE,
446         .cra_list               = LIST_HEAD_INIT(srp_algs[6].cra_list),
447         .cra_init               = ablk_init,
448         .cra_exit               = ablk_exit,
449         .cra_u = {
450                 .ablkcipher = {
451                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
452                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
453                         .ivsize         = SERPENT_BLOCK_SIZE,
454                         .setkey         = ablk_set_key,
455                         .encrypt        = __ablk_encrypt,
456                         .decrypt        = ablk_decrypt,
457                 },
458         },
459 }, {
460         .cra_name               = "ctr(serpent)",
461         .cra_driver_name        = "ctr-serpent-avx2",
462         .cra_priority           = 600,
463         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
464         .cra_blocksize          = 1,
465         .cra_ctxsize            = sizeof(struct async_helper_ctx),
466         .cra_alignmask          = 0,
467         .cra_type               = &crypto_ablkcipher_type,
468         .cra_module             = THIS_MODULE,
469         .cra_list               = LIST_HEAD_INIT(srp_algs[7].cra_list),
470         .cra_init               = ablk_init,
471         .cra_exit               = ablk_exit,
472         .cra_u = {
473                 .ablkcipher = {
474                         .min_keysize    = SERPENT_MIN_KEY_SIZE,
475                         .max_keysize    = SERPENT_MAX_KEY_SIZE,
476                         .ivsize         = SERPENT_BLOCK_SIZE,
477                         .setkey         = ablk_set_key,
478                         .encrypt        = ablk_encrypt,
479                         .decrypt        = ablk_encrypt,
480                         .geniv          = "chainiv",
481                 },
482         },
483 }, {
484         .cra_name               = "lrw(serpent)",
485         .cra_driver_name        = "lrw-serpent-avx2",
486         .cra_priority           = 600,
487         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
488         .cra_blocksize          = SERPENT_BLOCK_SIZE,
489         .cra_ctxsize            = sizeof(struct async_helper_ctx),
490         .cra_alignmask          = 0,
491         .cra_type               = &crypto_ablkcipher_type,
492         .cra_module             = THIS_MODULE,
493         .cra_list               = LIST_HEAD_INIT(srp_algs[8].cra_list),
494         .cra_init               = ablk_init,
495         .cra_exit               = ablk_exit,
496         .cra_u = {
497                 .ablkcipher = {
498                         .min_keysize    = SERPENT_MIN_KEY_SIZE +
499                                           SERPENT_BLOCK_SIZE,
500                         .max_keysize    = SERPENT_MAX_KEY_SIZE +
501                                           SERPENT_BLOCK_SIZE,
502                         .ivsize         = SERPENT_BLOCK_SIZE,
503                         .setkey         = ablk_set_key,
504                         .encrypt        = ablk_encrypt,
505                         .decrypt        = ablk_decrypt,
506                 },
507         },
508 }, {
509         .cra_name               = "xts(serpent)",
510         .cra_driver_name        = "xts-serpent-avx2",
511         .cra_priority           = 600,
512         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
513         .cra_blocksize          = SERPENT_BLOCK_SIZE,
514         .cra_ctxsize            = sizeof(struct async_helper_ctx),
515         .cra_alignmask          = 0,
516         .cra_type               = &crypto_ablkcipher_type,
517         .cra_module             = THIS_MODULE,
518         .cra_list               = LIST_HEAD_INIT(srp_algs[9].cra_list),
519         .cra_init               = ablk_init,
520         .cra_exit               = ablk_exit,
521         .cra_u = {
522                 .ablkcipher = {
523                         .min_keysize    = SERPENT_MIN_KEY_SIZE * 2,
524                         .max_keysize    = SERPENT_MAX_KEY_SIZE * 2,
525                         .ivsize         = SERPENT_BLOCK_SIZE,
526                         .setkey         = ablk_set_key,
527                         .encrypt        = ablk_encrypt,
528                         .decrypt        = ablk_decrypt,
529                 },
530         },
531 } };
532 
533 static int __init init(void)
534 {
535         u64 xcr0;
536 
537         if (!cpu_has_avx2 || !cpu_has_osxsave) {
538                 pr_info("AVX2 instructions are not detected.\n");
539                 return -ENODEV;
540         }
541 
542         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
543         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
544                 pr_info("AVX detected but unusable.\n");
545                 return -ENODEV;
546         }
547 
548         return crypto_register_algs(srp_algs, ARRAY_SIZE(srp_algs));
549 }
550 
551 static void __exit fini(void)
552 {
553         crypto_unregister_algs(srp_algs, ARRAY_SIZE(srp_algs));
554 }
555 
556 module_init(init);
557 module_exit(fini);
558 
559 MODULE_LICENSE("GPL");
560 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
561 MODULE_ALIAS_CRYPTO("serpent");
562 MODULE_ALIAS_CRYPTO("serpent-asm");
563 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp