~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/crypto/serpent_avx2_glue.c

Version: ~ [ linux-5.8-rc4 ] ~ [ linux-5.7.7 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.50 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.131 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.187 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.229 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.229 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
  3  *
  4  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  5  *
  6  * This program is free software; you can redistribute it and/or modify
  7  * it under the terms of the GNU General Public License as published by
  8  * the Free Software Foundation; either version 2 of the License, or
  9  * (at your option) any later version.
 10  *
 11  */
 12 
 13 #include <linux/module.h>
 14 #include <linux/types.h>
 15 #include <linux/crypto.h>
 16 #include <linux/err.h>
 17 #include <crypto/algapi.h>
 18 #include <crypto/internal/simd.h>
 19 #include <crypto/serpent.h>
 20 #include <crypto/xts.h>
 21 #include <asm/crypto/glue_helper.h>
 22 #include <asm/crypto/serpent-avx.h>
 23 
 24 #define SERPENT_AVX2_PARALLEL_BLOCKS 16
 25 
 26 /* 16-way AVX2 parallel cipher functions */
 27 asmlinkage void serpent_ecb_enc_16way(struct serpent_ctx *ctx, u8 *dst,
 28                                       const u8 *src);
 29 asmlinkage void serpent_ecb_dec_16way(struct serpent_ctx *ctx, u8 *dst,
 30                                       const u8 *src);
 31 asmlinkage void serpent_cbc_dec_16way(void *ctx, u128 *dst, const u128 *src);
 32 
 33 asmlinkage void serpent_ctr_16way(void *ctx, u128 *dst, const u128 *src,
 34                                   le128 *iv);
 35 asmlinkage void serpent_xts_enc_16way(struct serpent_ctx *ctx, u8 *dst,
 36                                       const u8 *src, le128 *iv);
 37 asmlinkage void serpent_xts_dec_16way(struct serpent_ctx *ctx, u8 *dst,
 38                                       const u8 *src, le128 *iv);
 39 
 40 static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
 41                                    const u8 *key, unsigned int keylen)
 42 {
 43         return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
 44 }
 45 
 46 static const struct common_glue_ctx serpent_enc = {
 47         .num_funcs = 3,
 48         .fpu_blocks_limit = 8,
 49 
 50         .funcs = { {
 51                 .num_blocks = 16,
 52                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_16way) }
 53         }, {
 54                 .num_blocks = 8,
 55                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_enc_8way_avx) }
 56         }, {
 57                 .num_blocks = 1,
 58                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
 59         } }
 60 };
 61 
 62 static const struct common_glue_ctx serpent_ctr = {
 63         .num_funcs = 3,
 64         .fpu_blocks_limit = 8,
 65 
 66         .funcs = { {
 67                 .num_blocks = 16,
 68                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_16way) }
 69         },  {
 70                 .num_blocks = 8,
 71                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
 72         }, {
 73                 .num_blocks = 1,
 74                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
 75         } }
 76 };
 77 
 78 static const struct common_glue_ctx serpent_enc_xts = {
 79         .num_funcs = 3,
 80         .fpu_blocks_limit = 8,
 81 
 82         .funcs = { {
 83                 .num_blocks = 16,
 84                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_16way) }
 85         }, {
 86                 .num_blocks = 8,
 87                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc_8way_avx) }
 88         }, {
 89                 .num_blocks = 1,
 90                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_enc) }
 91         } }
 92 };
 93 
 94 static const struct common_glue_ctx serpent_dec = {
 95         .num_funcs = 3,
 96         .fpu_blocks_limit = 8,
 97 
 98         .funcs = { {
 99                 .num_blocks = 16,
100                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_16way) }
101         }, {
102                 .num_blocks = 8,
103                 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_ecb_dec_8way_avx) }
104         }, {
105                 .num_blocks = 1,
106                 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
107         } }
108 };
109 
110 static const struct common_glue_ctx serpent_dec_cbc = {
111         .num_funcs = 3,
112         .fpu_blocks_limit = 8,
113 
114         .funcs = { {
115                 .num_blocks = 16,
116                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_16way) }
117         }, {
118                 .num_blocks = 8,
119                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_cbc_dec_8way_avx) }
120         }, {
121                 .num_blocks = 1,
122                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
123         } }
124 };
125 
126 static const struct common_glue_ctx serpent_dec_xts = {
127         .num_funcs = 3,
128         .fpu_blocks_limit = 8,
129 
130         .funcs = { {
131                 .num_blocks = 16,
132                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_16way) }
133         }, {
134                 .num_blocks = 8,
135                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec_8way_avx) }
136         }, {
137                 .num_blocks = 1,
138                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(serpent_xts_dec) }
139         } }
140 };
141 
142 static int ecb_encrypt(struct skcipher_request *req)
143 {
144         return glue_ecb_req_128bit(&serpent_enc, req);
145 }
146 
147 static int ecb_decrypt(struct skcipher_request *req)
148 {
149         return glue_ecb_req_128bit(&serpent_dec, req);
150 }
151 
152 static int cbc_encrypt(struct skcipher_request *req)
153 {
154         return glue_cbc_encrypt_req_128bit(GLUE_FUNC_CAST(__serpent_encrypt),
155                                            req);
156 }
157 
158 static int cbc_decrypt(struct skcipher_request *req)
159 {
160         return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
161 }
162 
163 static int ctr_crypt(struct skcipher_request *req)
164 {
165         return glue_ctr_req_128bit(&serpent_ctr, req);
166 }
167 
168 static int xts_encrypt(struct skcipher_request *req)
169 {
170         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
171         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
172 
173         return glue_xts_req_128bit(&serpent_enc_xts, req,
174                                    XTS_TWEAK_CAST(__serpent_encrypt),
175                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
176 }
177 
178 static int xts_decrypt(struct skcipher_request *req)
179 {
180         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
181         struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
182 
183         return glue_xts_req_128bit(&serpent_dec_xts, req,
184                                    XTS_TWEAK_CAST(__serpent_encrypt),
185                                    &ctx->tweak_ctx, &ctx->crypt_ctx);
186 }
187 
188 static struct skcipher_alg serpent_algs[] = {
189         {
190                 .base.cra_name          = "__ecb(serpent)",
191                 .base.cra_driver_name   = "__ecb-serpent-avx2",
192                 .base.cra_priority      = 600,
193                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
194                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
195                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
196                 .base.cra_module        = THIS_MODULE,
197                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
198                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
199                 .setkey                 = serpent_setkey_skcipher,
200                 .encrypt                = ecb_encrypt,
201                 .decrypt                = ecb_decrypt,
202         }, {
203                 .base.cra_name          = "__cbc(serpent)",
204                 .base.cra_driver_name   = "__cbc-serpent-avx2",
205                 .base.cra_priority      = 600,
206                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
207                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
208                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
209                 .base.cra_module        = THIS_MODULE,
210                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
211                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
212                 .ivsize                 = SERPENT_BLOCK_SIZE,
213                 .setkey                 = serpent_setkey_skcipher,
214                 .encrypt                = cbc_encrypt,
215                 .decrypt                = cbc_decrypt,
216         }, {
217                 .base.cra_name          = "__ctr(serpent)",
218                 .base.cra_driver_name   = "__ctr-serpent-avx2",
219                 .base.cra_priority      = 600,
220                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
221                 .base.cra_blocksize     = 1,
222                 .base.cra_ctxsize       = sizeof(struct serpent_ctx),
223                 .base.cra_module        = THIS_MODULE,
224                 .min_keysize            = SERPENT_MIN_KEY_SIZE,
225                 .max_keysize            = SERPENT_MAX_KEY_SIZE,
226                 .ivsize                 = SERPENT_BLOCK_SIZE,
227                 .chunksize              = SERPENT_BLOCK_SIZE,
228                 .setkey                 = serpent_setkey_skcipher,
229                 .encrypt                = ctr_crypt,
230                 .decrypt                = ctr_crypt,
231         }, {
232                 .base.cra_name          = "__xts(serpent)",
233                 .base.cra_driver_name   = "__xts-serpent-avx2",
234                 .base.cra_priority      = 600,
235                 .base.cra_flags         = CRYPTO_ALG_INTERNAL,
236                 .base.cra_blocksize     = SERPENT_BLOCK_SIZE,
237                 .base.cra_ctxsize       = sizeof(struct serpent_xts_ctx),
238                 .base.cra_module        = THIS_MODULE,
239                 .min_keysize            = 2 * SERPENT_MIN_KEY_SIZE,
240                 .max_keysize            = 2 * SERPENT_MAX_KEY_SIZE,
241                 .ivsize                 = SERPENT_BLOCK_SIZE,
242                 .setkey                 = xts_serpent_setkey,
243                 .encrypt                = xts_encrypt,
244                 .decrypt                = xts_decrypt,
245         },
246 };
247 
248 static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
249 
250 static int __init init(void)
251 {
252         const char *feature_name;
253 
254         if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
255                 pr_info("AVX2 instructions are not detected.\n");
256                 return -ENODEV;
257         }
258         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
259                                 &feature_name)) {
260                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
261                 return -ENODEV;
262         }
263 
264         return simd_register_skciphers_compat(serpent_algs,
265                                               ARRAY_SIZE(serpent_algs),
266                                               serpent_simd_algs);
267 }
268 
269 static void __exit fini(void)
270 {
271         simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
272                                   serpent_simd_algs);
273 }
274 
275 module_init(init);
276 module_exit(fini);
277 
278 MODULE_LICENSE("GPL");
279 MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
280 MODULE_ALIAS_CRYPTO("serpent");
281 MODULE_ALIAS_CRYPTO("serpent-asm");
282 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp