~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/crypto/blowfish_glue.c

Version: ~ [ linux-5.10-rc5 ] ~ [ linux-5.9.10 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.79 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.159 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.208 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.245 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.245 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.140 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.85 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Glue Code for assembler optimized version of Blowfish
  3  *
  4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  5  *
  6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  8  * CTR part based on code (crypto/ctr.c) by:
  9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 10  *
 11  * This program is free software; you can redistribute it and/or modify
 12  * it under the terms of the GNU General Public License as published by
 13  * the Free Software Foundation; either version 2 of the License, or
 14  * (at your option) any later version.
 15  *
 16  * This program is distributed in the hope that it will be useful,
 17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 19  * GNU General Public License for more details.
 20  *
 21  * You should have received a copy of the GNU General Public License
 22  * along with this program; if not, write to the Free Software
 23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 24  * USA
 25  *
 26  */
 27 
 28 #include <asm/processor.h>
 29 #include <crypto/blowfish.h>
 30 #include <linux/crypto.h>
 31 #include <linux/init.h>
 32 #include <linux/module.h>
 33 #include <linux/types.h>
 34 #include <crypto/algapi.h>
 35 
 36 /* regular block cipher functions */
 37 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
 38                                    bool xor);
 39 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
 40 
 41 /* 4-way parallel cipher functions */
 42 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
 43                                         const u8 *src, bool xor);
 44 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
 45                                       const u8 *src);
 46 
 47 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
 48 {
 49         __blowfish_enc_blk(ctx, dst, src, false);
 50 }
 51 
 52 static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
 53                                         const u8 *src)
 54 {
 55         __blowfish_enc_blk(ctx, dst, src, true);
 56 }
 57 
 58 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
 59                                          const u8 *src)
 60 {
 61         __blowfish_enc_blk_4way(ctx, dst, src, false);
 62 }
 63 
 64 static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
 65                                       const u8 *src)
 66 {
 67         __blowfish_enc_blk_4way(ctx, dst, src, true);
 68 }
 69 
 70 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 71 {
 72         blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
 73 }
 74 
 75 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 76 {
 77         blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
 78 }
 79 
 80 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
 81                      void (*fn)(struct bf_ctx *, u8 *, const u8 *),
 82                      void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
 83 {
 84         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 85         unsigned int bsize = BF_BLOCK_SIZE;
 86         unsigned int nbytes;
 87         int err;
 88 
 89         err = blkcipher_walk_virt(desc, walk);
 90 
 91         while ((nbytes = walk->nbytes)) {
 92                 u8 *wsrc = walk->src.virt.addr;
 93                 u8 *wdst = walk->dst.virt.addr;
 94 
 95                 /* Process four block batch */
 96                 if (nbytes >= bsize * 4) {
 97                         do {
 98                                 fn_4way(ctx, wdst, wsrc);
 99 
100                                 wsrc += bsize * 4;
101                                 wdst += bsize * 4;
102                                 nbytes -= bsize * 4;
103                         } while (nbytes >= bsize * 4);
104 
105                         if (nbytes < bsize)
106                                 goto done;
107                 }
108 
109                 /* Handle leftovers */
110                 do {
111                         fn(ctx, wdst, wsrc);
112 
113                         wsrc += bsize;
114                         wdst += bsize;
115                         nbytes -= bsize;
116                 } while (nbytes >= bsize);
117 
118 done:
119                 err = blkcipher_walk_done(desc, walk, nbytes);
120         }
121 
122         return err;
123 }
124 
125 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
126                        struct scatterlist *src, unsigned int nbytes)
127 {
128         struct blkcipher_walk walk;
129 
130         blkcipher_walk_init(&walk, dst, src, nbytes);
131         return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
132 }
133 
134 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
135                        struct scatterlist *src, unsigned int nbytes)
136 {
137         struct blkcipher_walk walk;
138 
139         blkcipher_walk_init(&walk, dst, src, nbytes);
140         return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
141 }
142 
143 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
144                                   struct blkcipher_walk *walk)
145 {
146         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
147         unsigned int bsize = BF_BLOCK_SIZE;
148         unsigned int nbytes = walk->nbytes;
149         u64 *src = (u64 *)walk->src.virt.addr;
150         u64 *dst = (u64 *)walk->dst.virt.addr;
151         u64 *iv = (u64 *)walk->iv;
152 
153         do {
154                 *dst = *src ^ *iv;
155                 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
156                 iv = dst;
157 
158                 src += 1;
159                 dst += 1;
160                 nbytes -= bsize;
161         } while (nbytes >= bsize);
162 
163         *(u64 *)walk->iv = *iv;
164         return nbytes;
165 }
166 
167 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
168                        struct scatterlist *src, unsigned int nbytes)
169 {
170         struct blkcipher_walk walk;
171         int err;
172 
173         blkcipher_walk_init(&walk, dst, src, nbytes);
174         err = blkcipher_walk_virt(desc, &walk);
175 
176         while ((nbytes = walk.nbytes)) {
177                 nbytes = __cbc_encrypt(desc, &walk);
178                 err = blkcipher_walk_done(desc, &walk, nbytes);
179         }
180 
181         return err;
182 }
183 
184 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
185                                   struct blkcipher_walk *walk)
186 {
187         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
188         unsigned int bsize = BF_BLOCK_SIZE;
189         unsigned int nbytes = walk->nbytes;
190         u64 *src = (u64 *)walk->src.virt.addr;
191         u64 *dst = (u64 *)walk->dst.virt.addr;
192         u64 ivs[4 - 1];
193         u64 last_iv;
194 
195         /* Start of the last block. */
196         src += nbytes / bsize - 1;
197         dst += nbytes / bsize - 1;
198 
199         last_iv = *src;
200 
201         /* Process four block batch */
202         if (nbytes >= bsize * 4) {
203                 do {
204                         nbytes -= bsize * 4 - bsize;
205                         src -= 4 - 1;
206                         dst -= 4 - 1;
207 
208                         ivs[0] = src[0];
209                         ivs[1] = src[1];
210                         ivs[2] = src[2];
211 
212                         blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
213 
214                         dst[1] ^= ivs[0];
215                         dst[2] ^= ivs[1];
216                         dst[3] ^= ivs[2];
217 
218                         nbytes -= bsize;
219                         if (nbytes < bsize)
220                                 goto done;
221 
222                         *dst ^= *(src - 1);
223                         src -= 1;
224                         dst -= 1;
225                 } while (nbytes >= bsize * 4);
226         }
227 
228         /* Handle leftovers */
229         for (;;) {
230                 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
231 
232                 nbytes -= bsize;
233                 if (nbytes < bsize)
234                         break;
235 
236                 *dst ^= *(src - 1);
237                 src -= 1;
238                 dst -= 1;
239         }
240 
241 done:
242         *dst ^= *(u64 *)walk->iv;
243         *(u64 *)walk->iv = last_iv;
244 
245         return nbytes;
246 }
247 
248 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
249                        struct scatterlist *src, unsigned int nbytes)
250 {
251         struct blkcipher_walk walk;
252         int err;
253 
254         blkcipher_walk_init(&walk, dst, src, nbytes);
255         err = blkcipher_walk_virt(desc, &walk);
256 
257         while ((nbytes = walk.nbytes)) {
258                 nbytes = __cbc_decrypt(desc, &walk);
259                 err = blkcipher_walk_done(desc, &walk, nbytes);
260         }
261 
262         return err;
263 }
264 
265 static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
266 {
267         u8 *ctrblk = walk->iv;
268         u8 keystream[BF_BLOCK_SIZE];
269         u8 *src = walk->src.virt.addr;
270         u8 *dst = walk->dst.virt.addr;
271         unsigned int nbytes = walk->nbytes;
272 
273         blowfish_enc_blk(ctx, keystream, ctrblk);
274         crypto_xor(keystream, src, nbytes);
275         memcpy(dst, keystream, nbytes);
276 
277         crypto_inc(ctrblk, BF_BLOCK_SIZE);
278 }
279 
280 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
281                                 struct blkcipher_walk *walk)
282 {
283         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
284         unsigned int bsize = BF_BLOCK_SIZE;
285         unsigned int nbytes = walk->nbytes;
286         u64 *src = (u64 *)walk->src.virt.addr;
287         u64 *dst = (u64 *)walk->dst.virt.addr;
288         u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
289         __be64 ctrblocks[4];
290 
291         /* Process four block batch */
292         if (nbytes >= bsize * 4) {
293                 do {
294                         if (dst != src) {
295                                 dst[0] = src[0];
296                                 dst[1] = src[1];
297                                 dst[2] = src[2];
298                                 dst[3] = src[3];
299                         }
300 
301                         /* create ctrblks for parallel encrypt */
302                         ctrblocks[0] = cpu_to_be64(ctrblk++);
303                         ctrblocks[1] = cpu_to_be64(ctrblk++);
304                         ctrblocks[2] = cpu_to_be64(ctrblk++);
305                         ctrblocks[3] = cpu_to_be64(ctrblk++);
306 
307                         blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
308                                                   (u8 *)ctrblocks);
309 
310                         src += 4;
311                         dst += 4;
312                 } while ((nbytes -= bsize * 4) >= bsize * 4);
313 
314                 if (nbytes < bsize)
315                         goto done;
316         }
317 
318         /* Handle leftovers */
319         do {
320                 if (dst != src)
321                         *dst = *src;
322 
323                 ctrblocks[0] = cpu_to_be64(ctrblk++);
324 
325                 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
326 
327                 src += 1;
328                 dst += 1;
329         } while ((nbytes -= bsize) >= bsize);
330 
331 done:
332         *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
333         return nbytes;
334 }
335 
336 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
337                      struct scatterlist *src, unsigned int nbytes)
338 {
339         struct blkcipher_walk walk;
340         int err;
341 
342         blkcipher_walk_init(&walk, dst, src, nbytes);
343         err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
344 
345         while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
346                 nbytes = __ctr_crypt(desc, &walk);
347                 err = blkcipher_walk_done(desc, &walk, nbytes);
348         }
349 
350         if (walk.nbytes) {
351                 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
352                 err = blkcipher_walk_done(desc, &walk, 0);
353         }
354 
355         return err;
356 }
357 
358 static struct crypto_alg bf_algs[4] = { {
359         .cra_name               = "blowfish",
360         .cra_driver_name        = "blowfish-asm",
361         .cra_priority           = 200,
362         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
363         .cra_blocksize          = BF_BLOCK_SIZE,
364         .cra_ctxsize            = sizeof(struct bf_ctx),
365         .cra_alignmask          = 0,
366         .cra_module             = THIS_MODULE,
367         .cra_u = {
368                 .cipher = {
369                         .cia_min_keysize        = BF_MIN_KEY_SIZE,
370                         .cia_max_keysize        = BF_MAX_KEY_SIZE,
371                         .cia_setkey             = blowfish_setkey,
372                         .cia_encrypt            = blowfish_encrypt,
373                         .cia_decrypt            = blowfish_decrypt,
374                 }
375         }
376 }, {
377         .cra_name               = "ecb(blowfish)",
378         .cra_driver_name        = "ecb-blowfish-asm",
379         .cra_priority           = 300,
380         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
381         .cra_blocksize          = BF_BLOCK_SIZE,
382         .cra_ctxsize            = sizeof(struct bf_ctx),
383         .cra_alignmask          = 0,
384         .cra_type               = &crypto_blkcipher_type,
385         .cra_module             = THIS_MODULE,
386         .cra_u = {
387                 .blkcipher = {
388                         .min_keysize    = BF_MIN_KEY_SIZE,
389                         .max_keysize    = BF_MAX_KEY_SIZE,
390                         .setkey         = blowfish_setkey,
391                         .encrypt        = ecb_encrypt,
392                         .decrypt        = ecb_decrypt,
393                 },
394         },
395 }, {
396         .cra_name               = "cbc(blowfish)",
397         .cra_driver_name        = "cbc-blowfish-asm",
398         .cra_priority           = 300,
399         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
400         .cra_blocksize          = BF_BLOCK_SIZE,
401         .cra_ctxsize            = sizeof(struct bf_ctx),
402         .cra_alignmask          = 0,
403         .cra_type               = &crypto_blkcipher_type,
404         .cra_module             = THIS_MODULE,
405         .cra_u = {
406                 .blkcipher = {
407                         .min_keysize    = BF_MIN_KEY_SIZE,
408                         .max_keysize    = BF_MAX_KEY_SIZE,
409                         .ivsize         = BF_BLOCK_SIZE,
410                         .setkey         = blowfish_setkey,
411                         .encrypt        = cbc_encrypt,
412                         .decrypt        = cbc_decrypt,
413                 },
414         },
415 }, {
416         .cra_name               = "ctr(blowfish)",
417         .cra_driver_name        = "ctr-blowfish-asm",
418         .cra_priority           = 300,
419         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
420         .cra_blocksize          = 1,
421         .cra_ctxsize            = sizeof(struct bf_ctx),
422         .cra_alignmask          = 0,
423         .cra_type               = &crypto_blkcipher_type,
424         .cra_module             = THIS_MODULE,
425         .cra_u = {
426                 .blkcipher = {
427                         .min_keysize    = BF_MIN_KEY_SIZE,
428                         .max_keysize    = BF_MAX_KEY_SIZE,
429                         .ivsize         = BF_BLOCK_SIZE,
430                         .setkey         = blowfish_setkey,
431                         .encrypt        = ctr_crypt,
432                         .decrypt        = ctr_crypt,
433                 },
434         },
435 } };
436 
437 static bool is_blacklisted_cpu(void)
438 {
439         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
440                 return false;
441 
442         if (boot_cpu_data.x86 == 0x0f) {
443                 /*
444                  * On Pentium 4, blowfish-x86_64 is slower than generic C
445                  * implementation because use of 64bit rotates (which are really
446                  * slow on P4). Therefore blacklist P4s.
447                  */
448                 return true;
449         }
450 
451         return false;
452 }
453 
454 static int force;
455 module_param(force, int, 0);
456 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
457 
458 static int __init init(void)
459 {
460         if (!force && is_blacklisted_cpu()) {
461                 printk(KERN_INFO
462                         "blowfish-x86_64: performance on this CPU "
463                         "would be suboptimal: disabling "
464                         "blowfish-x86_64.\n");
465                 return -ENODEV;
466         }
467 
468         return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs));
469 }
470 
471 static void __exit fini(void)
472 {
473         crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs));
474 }
475 
476 module_init(init);
477 module_exit(fini);
478 
479 MODULE_LICENSE("GPL");
480 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
481 MODULE_ALIAS_CRYPTO("blowfish");
482 MODULE_ALIAS_CRYPTO("blowfish-asm");
483 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp