~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/arm64/crypto/crct10dif-ce-glue.c

Version: ~ [ linux-6.2-rc3 ] ~ [ linux-6.1.5 ] ~ [ linux-6.0.19 ] ~ [ linux-5.19.17 ] ~ [ linux-5.18.19 ] ~ [ linux-5.17.15 ] ~ [ linux-5.16.20 ] ~ [ linux-5.15.87 ] ~ [ linux-5.14.21 ] ~ [ linux-5.13.19 ] ~ [ linux-5.12.19 ] ~ [ linux-5.11.22 ] ~ [ linux-5.10.162 ] ~ [ linux-5.9.16 ] ~ [ linux-5.8.18 ] ~ [ linux-5.7.19 ] ~ [ linux-5.6.19 ] ~ [ linux-5.5.19 ] ~ [ linux-5.4.228 ] ~ [ linux-5.3.18 ] ~ [ linux-5.2.21 ] ~ [ linux-5.1.21 ] ~ [ linux-5.0.21 ] ~ [ linux-4.20.17 ] ~ [ linux-4.19.269 ] ~ [ linux-4.18.20 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.302 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.337 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.302 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.10.108 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.9 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0-only
  2 /*
  3  * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions
  4  *
  5  * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
  6  */
  7 
  8 #include <linux/cpufeature.h>
  9 #include <linux/crc-t10dif.h>
 10 #include <linux/init.h>
 11 #include <linux/kernel.h>
 12 #include <linux/module.h>
 13 #include <linux/string.h>
 14 
 15 #include <crypto/internal/hash.h>
 16 #include <crypto/internal/simd.h>
 17 
 18 #include <asm/neon.h>
 19 #include <asm/simd.h>
 20 
 21 #define CRC_T10DIF_PMULL_CHUNK_SIZE     16U
 22 
 23 asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len);
 24 asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len);
 25 
 26 static int crct10dif_init(struct shash_desc *desc)
 27 {
 28         u16 *crc = shash_desc_ctx(desc);
 29 
 30         *crc = 0;
 31         return 0;
 32 }
 33 
 34 static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data,
 35                             unsigned int length)
 36 {
 37         u16 *crc = shash_desc_ctx(desc);
 38 
 39         if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
 40                 do {
 41                         unsigned int chunk = length;
 42 
 43                         if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
 44                                 chunk = SZ_4K;
 45 
 46                         kernel_neon_begin();
 47                         *crc = crc_t10dif_pmull_p8(*crc, data, chunk);
 48                         kernel_neon_end();
 49                         data += chunk;
 50                         length -= chunk;
 51                 } while (length);
 52         } else {
 53                 *crc = crc_t10dif_generic(*crc, data, length);
 54         }
 55 
 56         return 0;
 57 }
 58 
 59 static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data,
 60                             unsigned int length)
 61 {
 62         u16 *crc = shash_desc_ctx(desc);
 63 
 64         if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) {
 65                 do {
 66                         unsigned int chunk = length;
 67 
 68                         if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE)
 69                                 chunk = SZ_4K;
 70 
 71                         kernel_neon_begin();
 72                         *crc = crc_t10dif_pmull_p64(*crc, data, chunk);
 73                         kernel_neon_end();
 74                         data += chunk;
 75                         length -= chunk;
 76                 } while (length);
 77         } else {
 78                 *crc = crc_t10dif_generic(*crc, data, length);
 79         }
 80 
 81         return 0;
 82 }
 83 
 84 static int crct10dif_final(struct shash_desc *desc, u8 *out)
 85 {
 86         u16 *crc = shash_desc_ctx(desc);
 87 
 88         *(u16 *)out = *crc;
 89         return 0;
 90 }
 91 
 92 static struct shash_alg crc_t10dif_alg[] = {{
 93         .digestsize             = CRC_T10DIF_DIGEST_SIZE,
 94         .init                   = crct10dif_init,
 95         .update                 = crct10dif_update_pmull_p8,
 96         .final                  = crct10dif_final,
 97         .descsize               = CRC_T10DIF_DIGEST_SIZE,
 98 
 99         .base.cra_name          = "crct10dif",
100         .base.cra_driver_name   = "crct10dif-arm64-neon",
101         .base.cra_priority      = 100,
102         .base.cra_blocksize     = CRC_T10DIF_BLOCK_SIZE,
103         .base.cra_module        = THIS_MODULE,
104 }, {
105         .digestsize             = CRC_T10DIF_DIGEST_SIZE,
106         .init                   = crct10dif_init,
107         .update                 = crct10dif_update_pmull_p64,
108         .final                  = crct10dif_final,
109         .descsize               = CRC_T10DIF_DIGEST_SIZE,
110 
111         .base.cra_name          = "crct10dif",
112         .base.cra_driver_name   = "crct10dif-arm64-ce",
113         .base.cra_priority      = 200,
114         .base.cra_blocksize     = CRC_T10DIF_BLOCK_SIZE,
115         .base.cra_module        = THIS_MODULE,
116 }};
117 
118 static int __init crc_t10dif_mod_init(void)
119 {
120         if (cpu_have_named_feature(PMULL))
121                 return crypto_register_shashes(crc_t10dif_alg,
122                                                ARRAY_SIZE(crc_t10dif_alg));
123         else
124                 /* only register the first array element */
125                 return crypto_register_shash(crc_t10dif_alg);
126 }
127 
128 static void __exit crc_t10dif_mod_exit(void)
129 {
130         if (cpu_have_named_feature(PMULL))
131                 crypto_unregister_shashes(crc_t10dif_alg,
132                                           ARRAY_SIZE(crc_t10dif_alg));
133         else
134                 crypto_unregister_shash(crc_t10dif_alg);
135 }
136 
137 module_cpu_feature_match(ASIMD, crc_t10dif_mod_init);
138 module_exit(crc_t10dif_mod_exit);
139 
140 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
141 MODULE_LICENSE("GPL v2");
142 MODULE_ALIAS_CRYPTO("crct10dif");
143 MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce");
144 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp