~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/x86/crypto/blowfish_glue.c

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 /*
  2  * Glue Code for assembler optimized version of Blowfish
  3  *
  4  * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
  5  *
  6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
  7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
  8  * CTR part based on code (crypto/ctr.c) by:
  9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
 10  *
 11  * This program is free software; you can redistribute it and/or modify
 12  * it under the terms of the GNU General Public License as published by
 13  * the Free Software Foundation; either version 2 of the License, or
 14  * (at your option) any later version.
 15  *
 16  * This program is distributed in the hope that it will be useful,
 17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
 18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 19  * GNU General Public License for more details.
 20  *
 21  * You should have received a copy of the GNU General Public License
 22  * along with this program; if not, write to the Free Software
 23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
 24  * USA
 25  *
 26  */
 27 
 28 #include <asm/processor.h>
 29 #include <crypto/blowfish.h>
 30 #include <linux/crypto.h>
 31 #include <linux/init.h>
 32 #include <linux/module.h>
 33 #include <linux/types.h>
 34 #include <crypto/algapi.h>
 35 
 36 /* regular block cipher functions */
 37 asmlinkage void __blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src,
 38                                    bool xor);
 39 asmlinkage void blowfish_dec_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src);
 40 
 41 /* 4-way parallel cipher functions */
 42 asmlinkage void __blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
 43                                         const u8 *src, bool xor);
 44 asmlinkage void blowfish_dec_blk_4way(struct bf_ctx *ctx, u8 *dst,
 45                                       const u8 *src);
 46 
 47 static inline void blowfish_enc_blk(struct bf_ctx *ctx, u8 *dst, const u8 *src)
 48 {
 49         __blowfish_enc_blk(ctx, dst, src, false);
 50 }
 51 
 52 static inline void blowfish_enc_blk_xor(struct bf_ctx *ctx, u8 *dst,
 53                                         const u8 *src)
 54 {
 55         __blowfish_enc_blk(ctx, dst, src, true);
 56 }
 57 
 58 static inline void blowfish_enc_blk_4way(struct bf_ctx *ctx, u8 *dst,
 59                                          const u8 *src)
 60 {
 61         __blowfish_enc_blk_4way(ctx, dst, src, false);
 62 }
 63 
 64 static inline void blowfish_enc_blk_xor_4way(struct bf_ctx *ctx, u8 *dst,
 65                                       const u8 *src)
 66 {
 67         __blowfish_enc_blk_4way(ctx, dst, src, true);
 68 }
 69 
 70 static void blowfish_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 71 {
 72         blowfish_enc_blk(crypto_tfm_ctx(tfm), dst, src);
 73 }
 74 
 75 static void blowfish_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
 76 {
 77         blowfish_dec_blk(crypto_tfm_ctx(tfm), dst, src);
 78 }
 79 
 80 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
 81                      void (*fn)(struct bf_ctx *, u8 *, const u8 *),
 82                      void (*fn_4way)(struct bf_ctx *, u8 *, const u8 *))
 83 {
 84         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 85         unsigned int bsize = BF_BLOCK_SIZE;
 86         unsigned int nbytes;
 87         int err;
 88 
 89         err = blkcipher_walk_virt(desc, walk);
 90 
 91         while ((nbytes = walk->nbytes)) {
 92                 u8 *wsrc = walk->src.virt.addr;
 93                 u8 *wdst = walk->dst.virt.addr;
 94 
 95                 /* Process four block batch */
 96                 if (nbytes >= bsize * 4) {
 97                         do {
 98                                 fn_4way(ctx, wdst, wsrc);
 99 
100                                 wsrc += bsize * 4;
101                                 wdst += bsize * 4;
102                                 nbytes -= bsize * 4;
103                         } while (nbytes >= bsize * 4);
104 
105                         if (nbytes < bsize)
106                                 goto done;
107                 }
108 
109                 /* Handle leftovers */
110                 do {
111                         fn(ctx, wdst, wsrc);
112 
113                         wsrc += bsize;
114                         wdst += bsize;
115                         nbytes -= bsize;
116                 } while (nbytes >= bsize);
117 
118 done:
119                 err = blkcipher_walk_done(desc, walk, nbytes);
120         }
121 
122         return err;
123 }
124 
125 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
126                        struct scatterlist *src, unsigned int nbytes)
127 {
128         struct blkcipher_walk walk;
129 
130         blkcipher_walk_init(&walk, dst, src, nbytes);
131         return ecb_crypt(desc, &walk, blowfish_enc_blk, blowfish_enc_blk_4way);
132 }
133 
134 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
135                        struct scatterlist *src, unsigned int nbytes)
136 {
137         struct blkcipher_walk walk;
138 
139         blkcipher_walk_init(&walk, dst, src, nbytes);
140         return ecb_crypt(desc, &walk, blowfish_dec_blk, blowfish_dec_blk_4way);
141 }
142 
143 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
144                                   struct blkcipher_walk *walk)
145 {
146         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
147         unsigned int bsize = BF_BLOCK_SIZE;
148         unsigned int nbytes = walk->nbytes;
149         u64 *src = (u64 *)walk->src.virt.addr;
150         u64 *dst = (u64 *)walk->dst.virt.addr;
151         u64 *iv = (u64 *)walk->iv;
152 
153         do {
154                 *dst = *src ^ *iv;
155                 blowfish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
156                 iv = dst;
157 
158                 src += 1;
159                 dst += 1;
160                 nbytes -= bsize;
161         } while (nbytes >= bsize);
162 
163         *(u64 *)walk->iv = *iv;
164         return nbytes;
165 }
166 
167 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
168                        struct scatterlist *src, unsigned int nbytes)
169 {
170         struct blkcipher_walk walk;
171         int err;
172 
173         blkcipher_walk_init(&walk, dst, src, nbytes);
174         err = blkcipher_walk_virt(desc, &walk);
175 
176         while ((nbytes = walk.nbytes)) {
177                 nbytes = __cbc_encrypt(desc, &walk);
178                 err = blkcipher_walk_done(desc, &walk, nbytes);
179         }
180 
181         return err;
182 }
183 
184 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
185                                   struct blkcipher_walk *walk)
186 {
187         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
188         unsigned int bsize = BF_BLOCK_SIZE;
189         unsigned int nbytes = walk->nbytes;
190         u64 *src = (u64 *)walk->src.virt.addr;
191         u64 *dst = (u64 *)walk->dst.virt.addr;
192         u64 ivs[4 - 1];
193         u64 last_iv;
194 
195         /* Start of the last block. */
196         src += nbytes / bsize - 1;
197         dst += nbytes / bsize - 1;
198 
199         last_iv = *src;
200 
201         /* Process four block batch */
202         if (nbytes >= bsize * 4) {
203                 do {
204                         nbytes -= bsize * 4 - bsize;
205                         src -= 4 - 1;
206                         dst -= 4 - 1;
207 
208                         ivs[0] = src[0];
209                         ivs[1] = src[1];
210                         ivs[2] = src[2];
211 
212                         blowfish_dec_blk_4way(ctx, (u8 *)dst, (u8 *)src);
213 
214                         dst[1] ^= ivs[0];
215                         dst[2] ^= ivs[1];
216                         dst[3] ^= ivs[2];
217 
218                         nbytes -= bsize;
219                         if (nbytes < bsize)
220                                 goto done;
221 
222                         *dst ^= *(src - 1);
223                         src -= 1;
224                         dst -= 1;
225                 } while (nbytes >= bsize * 4);
226         }
227 
228         /* Handle leftovers */
229         for (;;) {
230                 blowfish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
231 
232                 nbytes -= bsize;
233                 if (nbytes < bsize)
234                         break;
235 
236                 *dst ^= *(src - 1);
237                 src -= 1;
238                 dst -= 1;
239         }
240 
241 done:
242         *dst ^= *(u64 *)walk->iv;
243         *(u64 *)walk->iv = last_iv;
244 
245         return nbytes;
246 }
247 
248 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
249                        struct scatterlist *src, unsigned int nbytes)
250 {
251         struct blkcipher_walk walk;
252         int err;
253 
254         blkcipher_walk_init(&walk, dst, src, nbytes);
255         err = blkcipher_walk_virt(desc, &walk);
256 
257         while ((nbytes = walk.nbytes)) {
258                 nbytes = __cbc_decrypt(desc, &walk);
259                 err = blkcipher_walk_done(desc, &walk, nbytes);
260         }
261 
262         return err;
263 }
264 
265 static void ctr_crypt_final(struct bf_ctx *ctx, struct blkcipher_walk *walk)
266 {
267         u8 *ctrblk = walk->iv;
268         u8 keystream[BF_BLOCK_SIZE];
269         u8 *src = walk->src.virt.addr;
270         u8 *dst = walk->dst.virt.addr;
271         unsigned int nbytes = walk->nbytes;
272 
273         blowfish_enc_blk(ctx, keystream, ctrblk);
274         crypto_xor_cpy(dst, keystream, src, nbytes);
275 
276         crypto_inc(ctrblk, BF_BLOCK_SIZE);
277 }
278 
279 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
280                                 struct blkcipher_walk *walk)
281 {
282         struct bf_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
283         unsigned int bsize = BF_BLOCK_SIZE;
284         unsigned int nbytes = walk->nbytes;
285         u64 *src = (u64 *)walk->src.virt.addr;
286         u64 *dst = (u64 *)walk->dst.virt.addr;
287         u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
288         __be64 ctrblocks[4];
289 
290         /* Process four block batch */
291         if (nbytes >= bsize * 4) {
292                 do {
293                         if (dst != src) {
294                                 dst[0] = src[0];
295                                 dst[1] = src[1];
296                                 dst[2] = src[2];
297                                 dst[3] = src[3];
298                         }
299 
300                         /* create ctrblks for parallel encrypt */
301                         ctrblocks[0] = cpu_to_be64(ctrblk++);
302                         ctrblocks[1] = cpu_to_be64(ctrblk++);
303                         ctrblocks[2] = cpu_to_be64(ctrblk++);
304                         ctrblocks[3] = cpu_to_be64(ctrblk++);
305 
306                         blowfish_enc_blk_xor_4way(ctx, (u8 *)dst,
307                                                   (u8 *)ctrblocks);
308 
309                         src += 4;
310                         dst += 4;
311                 } while ((nbytes -= bsize * 4) >= bsize * 4);
312 
313                 if (nbytes < bsize)
314                         goto done;
315         }
316 
317         /* Handle leftovers */
318         do {
319                 if (dst != src)
320                         *dst = *src;
321 
322                 ctrblocks[0] = cpu_to_be64(ctrblk++);
323 
324                 blowfish_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
325 
326                 src += 1;
327                 dst += 1;
328         } while ((nbytes -= bsize) >= bsize);
329 
330 done:
331         *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
332         return nbytes;
333 }
334 
335 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
336                      struct scatterlist *src, unsigned int nbytes)
337 {
338         struct blkcipher_walk walk;
339         int err;
340 
341         blkcipher_walk_init(&walk, dst, src, nbytes);
342         err = blkcipher_walk_virt_block(desc, &walk, BF_BLOCK_SIZE);
343 
344         while ((nbytes = walk.nbytes) >= BF_BLOCK_SIZE) {
345                 nbytes = __ctr_crypt(desc, &walk);
346                 err = blkcipher_walk_done(desc, &walk, nbytes);
347         }
348 
349         if (walk.nbytes) {
350                 ctr_crypt_final(crypto_blkcipher_ctx(desc->tfm), &walk);
351                 err = blkcipher_walk_done(desc, &walk, 0);
352         }
353 
354         return err;
355 }
356 
357 static struct crypto_alg bf_algs[4] = { {
358         .cra_name               = "blowfish",
359         .cra_driver_name        = "blowfish-asm",
360         .cra_priority           = 200,
361         .cra_flags              = CRYPTO_ALG_TYPE_CIPHER,
362         .cra_blocksize          = BF_BLOCK_SIZE,
363         .cra_ctxsize            = sizeof(struct bf_ctx),
364         .cra_alignmask          = 0,
365         .cra_module             = THIS_MODULE,
366         .cra_u = {
367                 .cipher = {
368                         .cia_min_keysize        = BF_MIN_KEY_SIZE,
369                         .cia_max_keysize        = BF_MAX_KEY_SIZE,
370                         .cia_setkey             = blowfish_setkey,
371                         .cia_encrypt            = blowfish_encrypt,
372                         .cia_decrypt            = blowfish_decrypt,
373                 }
374         }
375 }, {
376         .cra_name               = "ecb(blowfish)",
377         .cra_driver_name        = "ecb-blowfish-asm",
378         .cra_priority           = 300,
379         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
380         .cra_blocksize          = BF_BLOCK_SIZE,
381         .cra_ctxsize            = sizeof(struct bf_ctx),
382         .cra_alignmask          = 0,
383         .cra_type               = &crypto_blkcipher_type,
384         .cra_module             = THIS_MODULE,
385         .cra_u = {
386                 .blkcipher = {
387                         .min_keysize    = BF_MIN_KEY_SIZE,
388                         .max_keysize    = BF_MAX_KEY_SIZE,
389                         .setkey         = blowfish_setkey,
390                         .encrypt        = ecb_encrypt,
391                         .decrypt        = ecb_decrypt,
392                 },
393         },
394 }, {
395         .cra_name               = "cbc(blowfish)",
396         .cra_driver_name        = "cbc-blowfish-asm",
397         .cra_priority           = 300,
398         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
399         .cra_blocksize          = BF_BLOCK_SIZE,
400         .cra_ctxsize            = sizeof(struct bf_ctx),
401         .cra_alignmask          = 0,
402         .cra_type               = &crypto_blkcipher_type,
403         .cra_module             = THIS_MODULE,
404         .cra_u = {
405                 .blkcipher = {
406                         .min_keysize    = BF_MIN_KEY_SIZE,
407                         .max_keysize    = BF_MAX_KEY_SIZE,
408                         .ivsize         = BF_BLOCK_SIZE,
409                         .setkey         = blowfish_setkey,
410                         .encrypt        = cbc_encrypt,
411                         .decrypt        = cbc_decrypt,
412                 },
413         },
414 }, {
415         .cra_name               = "ctr(blowfish)",
416         .cra_driver_name        = "ctr-blowfish-asm",
417         .cra_priority           = 300,
418         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER,
419         .cra_blocksize          = 1,
420         .cra_ctxsize            = sizeof(struct bf_ctx),
421         .cra_alignmask          = 0,
422         .cra_type               = &crypto_blkcipher_type,
423         .cra_module             = THIS_MODULE,
424         .cra_u = {
425                 .blkcipher = {
426                         .min_keysize    = BF_MIN_KEY_SIZE,
427                         .max_keysize    = BF_MAX_KEY_SIZE,
428                         .ivsize         = BF_BLOCK_SIZE,
429                         .setkey         = blowfish_setkey,
430                         .encrypt        = ctr_crypt,
431                         .decrypt        = ctr_crypt,
432                 },
433         },
434 } };
435 
436 static bool is_blacklisted_cpu(void)
437 {
438         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
439                 return false;
440 
441         if (boot_cpu_data.x86 == 0x0f) {
442                 /*
443                  * On Pentium 4, blowfish-x86_64 is slower than generic C
444                  * implementation because use of 64bit rotates (which are really
445                  * slow on P4). Therefore blacklist P4s.
446                  */
447                 return true;
448         }
449 
450         return false;
451 }
452 
453 static int force;
454 module_param(force, int, 0);
455 MODULE_PARM_DESC(force, "Force module load, ignore CPU blacklist");
456 
457 static int __init init(void)
458 {
459         if (!force && is_blacklisted_cpu()) {
460                 printk(KERN_INFO
461                         "blowfish-x86_64: performance on this CPU "
462                         "would be suboptimal: disabling "
463                         "blowfish-x86_64.\n");
464                 return -ENODEV;
465         }
466 
467         return crypto_register_algs(bf_algs, ARRAY_SIZE(bf_algs));
468 }
469 
470 static void __exit fini(void)
471 {
472         crypto_unregister_algs(bf_algs, ARRAY_SIZE(bf_algs));
473 }
474 
475 module_init(init);
476 module_exit(fini);
477 
478 MODULE_LICENSE("GPL");
479 MODULE_DESCRIPTION("Blowfish Cipher Algorithm, asm optimized");
480 MODULE_ALIAS_CRYPTO("blowfish");
481 MODULE_ALIAS_CRYPTO("blowfish-asm");
482 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp