~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

TOMOYO Linux Cross Reference
Linux/arch/s390/crypto/crc32-vx.c

Version: ~ [ linux-4.19-rc7 ] ~ [ linux-4.18.12 ] ~ [ linux-4.17.19 ] ~ [ linux-4.16.18 ] ~ [ linux-4.15.18 ] ~ [ linux-4.14.74 ] ~ [ linux-4.13.16 ] ~ [ linux-4.12.14 ] ~ [ linux-4.11.12 ] ~ [ linux-4.10.17 ] ~ [ linux-4.9.131 ] ~ [ linux-4.8.17 ] ~ [ linux-4.7.10 ] ~ [ linux-4.6.7 ] ~ [ linux-4.5.7 ] ~ [ linux-4.4.159 ] ~ [ linux-4.3.6 ] ~ [ linux-4.2.8 ] ~ [ linux-4.1.52 ] ~ [ linux-4.0.9 ] ~ [ linux-3.19.8 ] ~ [ linux-3.18.123 ] ~ [ linux-3.17.8 ] ~ [ linux-3.16.59 ] ~ [ linux-3.15.10 ] ~ [ linux-3.14.79 ] ~ [ linux-3.13.11 ] ~ [ linux-3.12.74 ] ~ [ linux-3.11.10 ] ~ [ linux-3.10.108 ] ~ [ linux-3.9.11 ] ~ [ linux-3.8.13 ] ~ [ linux-3.7.10 ] ~ [ linux-3.6.11 ] ~ [ linux-3.5.7 ] ~ [ linux-3.4.113 ] ~ [ linux-3.3.8 ] ~ [ linux-3.2.102 ] ~ [ linux-3.1.10 ] ~ [ linux-3.0.101 ] ~ [ linux-2.6.39.4 ] ~ [ linux-2.6.38.8 ] ~ [ linux-2.6.37.6 ] ~ [ linux-2.6.36.4 ] ~ [ linux-2.6.35.14 ] ~ [ linux-2.6.34.15 ] ~ [ linux-2.6.33.20 ] ~ [ linux-2.6.32.71 ] ~ [ linux-2.6.31.14 ] ~ [ linux-2.6.30.10 ] ~ [ linux-2.6.29.6 ] ~ [ linux-2.6.28.10 ] ~ [ linux-2.6.27.62 ] ~ [ linux-2.6.0 ] ~ [ linux-2.4.37.11 ] ~ [ unix-v6-master ] ~ [ ccs-tools-1.8.5 ] ~ [ policy-sample ] ~
Architecture: ~ [ i386 ] ~ [ alpha ] ~ [ m68k ] ~ [ mips ] ~ [ ppc ] ~ [ sparc ] ~ [ sparc64 ] ~

  1 // SPDX-License-Identifier: GPL-2.0
  2 /*
  3  * Crypto-API module for CRC-32 algorithms implemented with the
  4  * z/Architecture Vector Extension Facility.
  5  *
  6  * Copyright IBM Corp. 2015
  7  * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
  8  */
  9 #define KMSG_COMPONENT  "crc32-vx"
 10 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
 11 
 12 #include <linux/module.h>
 13 #include <linux/cpufeature.h>
 14 #include <linux/crc32.h>
 15 #include <crypto/internal/hash.h>
 16 #include <asm/fpu/api.h>
 17 
 18 
 19 #define CRC32_BLOCK_SIZE        1
 20 #define CRC32_DIGEST_SIZE       4
 21 
 22 #define VX_MIN_LEN              64
 23 #define VX_ALIGNMENT            16L
 24 #define VX_ALIGN_MASK           (VX_ALIGNMENT - 1)
 25 
 26 struct crc_ctx {
 27         u32 key;
 28 };
 29 
 30 struct crc_desc_ctx {
 31         u32 crc;
 32 };
 33 
 34 /* Prototypes for functions in assembly files */
 35 u32 crc32_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
 36 u32 crc32_be_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
 37 u32 crc32c_le_vgfm_16(u32 crc, unsigned char const *buf, size_t size);
 38 
 39 /*
 40  * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension
 41  *
 42  * Creates a function to perform a particular CRC-32 computation. Depending
 43  * on the message buffer, the hardware-accelerated or software implementation
 44  * is used.   Note that the message buffer is aligned to improve fetch
 45  * operations of VECTOR LOAD MULTIPLE instructions.
 46  *
 47  */
 48 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw)                 \
 49         static u32 __pure ___fname(u32 crc,                                 \
 50                                 unsigned char const *data, size_t datalen)  \
 51         {                                                                   \
 52                 struct kernel_fpu vxstate;                                  \
 53                 unsigned long prealign, aligned, remaining;                 \
 54                                                                             \
 55                 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK)                   \
 56                         return ___crc32_sw(crc, data, datalen);             \
 57                                                                             \
 58                 if ((unsigned long)data & VX_ALIGN_MASK) {                  \
 59                         prealign = VX_ALIGNMENT -                           \
 60                                   ((unsigned long)data & VX_ALIGN_MASK);    \
 61                         datalen -= prealign;                                \
 62                         crc = ___crc32_sw(crc, data, prealign);             \
 63                         data = (void *)((unsigned long)data + prealign);    \
 64                 }                                                           \
 65                                                                             \
 66                 aligned = datalen & ~VX_ALIGN_MASK;                         \
 67                 remaining = datalen & VX_ALIGN_MASK;                        \
 68                                                                             \
 69                 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW);                 \
 70                 crc = ___crc32_vx(crc, data, aligned);                      \
 71                 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW);                   \
 72                                                                             \
 73                 if (remaining)                                              \
 74                         crc = ___crc32_sw(crc, data + aligned, remaining);  \
 75                                                                             \
 76                 return crc;                                                 \
 77         }
 78 
 79 DEFINE_CRC32_VX(crc32_le_vx, crc32_le_vgfm_16, crc32_le)
 80 DEFINE_CRC32_VX(crc32_be_vx, crc32_be_vgfm_16, crc32_be)
 81 DEFINE_CRC32_VX(crc32c_le_vx, crc32c_le_vgfm_16, __crc32c_le)
 82 
 83 
 84 static int crc32_vx_cra_init_zero(struct crypto_tfm *tfm)
 85 {
 86         struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
 87 
 88         mctx->key = 0;
 89         return 0;
 90 }
 91 
 92 static int crc32_vx_cra_init_invert(struct crypto_tfm *tfm)
 93 {
 94         struct crc_ctx *mctx = crypto_tfm_ctx(tfm);
 95 
 96         mctx->key = ~0;
 97         return 0;
 98 }
 99 
100 static int crc32_vx_init(struct shash_desc *desc)
101 {
102         struct crc_ctx *mctx = crypto_shash_ctx(desc->tfm);
103         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
104 
105         ctx->crc = mctx->key;
106         return 0;
107 }
108 
109 static int crc32_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
110                            unsigned int newkeylen)
111 {
112         struct crc_ctx *mctx = crypto_shash_ctx(tfm);
113 
114         if (newkeylen != sizeof(mctx->key)) {
115                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
116                 return -EINVAL;
117         }
118         mctx->key = le32_to_cpu(*(__le32 *)newkey);
119         return 0;
120 }
121 
122 static int crc32be_vx_setkey(struct crypto_shash *tfm, const u8 *newkey,
123                              unsigned int newkeylen)
124 {
125         struct crc_ctx *mctx = crypto_shash_ctx(tfm);
126 
127         if (newkeylen != sizeof(mctx->key)) {
128                 crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
129                 return -EINVAL;
130         }
131         mctx->key = be32_to_cpu(*(__be32 *)newkey);
132         return 0;
133 }
134 
135 static int crc32le_vx_final(struct shash_desc *desc, u8 *out)
136 {
137         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
138 
139         *(__le32 *)out = cpu_to_le32p(&ctx->crc);
140         return 0;
141 }
142 
143 static int crc32be_vx_final(struct shash_desc *desc, u8 *out)
144 {
145         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
146 
147         *(__be32 *)out = cpu_to_be32p(&ctx->crc);
148         return 0;
149 }
150 
151 static int crc32c_vx_final(struct shash_desc *desc, u8 *out)
152 {
153         struct crc_desc_ctx *ctx = shash_desc_ctx(desc);
154 
155         /*
156          * Perform a final XOR with 0xFFFFFFFF to be in sync
157          * with the generic crc32c shash implementation.
158          */
159         *(__le32 *)out = ~cpu_to_le32p(&ctx->crc);
160         return 0;
161 }
162 
163 static int __crc32le_vx_finup(u32 *crc, const u8 *data, unsigned int len,
164                               u8 *out)
165 {
166         *(__le32 *)out = cpu_to_le32(crc32_le_vx(*crc, data, len));
167         return 0;
168 }
169 
170 static int __crc32be_vx_finup(u32 *crc, const u8 *data, unsigned int len,
171                               u8 *out)
172 {
173         *(__be32 *)out = cpu_to_be32(crc32_be_vx(*crc, data, len));
174         return 0;
175 }
176 
177 static int __crc32c_vx_finup(u32 *crc, const u8 *data, unsigned int len,
178                              u8 *out)
179 {
180         /*
181          * Perform a final XOR with 0xFFFFFFFF to be in sync
182          * with the generic crc32c shash implementation.
183          */
184         *(__le32 *)out = ~cpu_to_le32(crc32c_le_vx(*crc, data, len));
185         return 0;
186 }
187 
188 
189 #define CRC32_VX_FINUP(alg, func)                                             \
190         static int alg ## _vx_finup(struct shash_desc *desc, const u8 *data,  \
191                                    unsigned int datalen, u8 *out)             \
192         {                                                                     \
193                 return __ ## alg ## _vx_finup(shash_desc_ctx(desc),           \
194                                               data, datalen, out);            \
195         }
196 
197 CRC32_VX_FINUP(crc32le, crc32_le_vx)
198 CRC32_VX_FINUP(crc32be, crc32_be_vx)
199 CRC32_VX_FINUP(crc32c, crc32c_le_vx)
200 
201 #define CRC32_VX_DIGEST(alg, func)                                            \
202         static int alg ## _vx_digest(struct shash_desc *desc, const u8 *data, \
203                                      unsigned int len, u8 *out)               \
204         {                                                                     \
205                 return __ ## alg ## _vx_finup(crypto_shash_ctx(desc->tfm),    \
206                                               data, len, out);                \
207         }
208 
209 CRC32_VX_DIGEST(crc32le, crc32_le_vx)
210 CRC32_VX_DIGEST(crc32be, crc32_be_vx)
211 CRC32_VX_DIGEST(crc32c, crc32c_le_vx)
212 
213 #define CRC32_VX_UPDATE(alg, func)                                            \
214         static int alg ## _vx_update(struct shash_desc *desc, const u8 *data, \
215                                      unsigned int datalen)                    \
216         {                                                                     \
217                 struct crc_desc_ctx *ctx = shash_desc_ctx(desc);              \
218                 ctx->crc = func(ctx->crc, data, datalen);                     \
219                 return 0;                                                     \
220         }
221 
222 CRC32_VX_UPDATE(crc32le, crc32_le_vx)
223 CRC32_VX_UPDATE(crc32be, crc32_be_vx)
224 CRC32_VX_UPDATE(crc32c, crc32c_le_vx)
225 
226 
227 static struct shash_alg crc32_vx_algs[] = {
228         /* CRC-32 LE */
229         {
230                 .init           =       crc32_vx_init,
231                 .setkey         =       crc32_vx_setkey,
232                 .update         =       crc32le_vx_update,
233                 .final          =       crc32le_vx_final,
234                 .finup          =       crc32le_vx_finup,
235                 .digest         =       crc32le_vx_digest,
236                 .descsize       =       sizeof(struct crc_desc_ctx),
237                 .digestsize     =       CRC32_DIGEST_SIZE,
238                 .base           =       {
239                         .cra_name        = "crc32",
240                         .cra_driver_name = "crc32-vx",
241                         .cra_priority    = 200,
242                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
243                         .cra_blocksize   = CRC32_BLOCK_SIZE,
244                         .cra_ctxsize     = sizeof(struct crc_ctx),
245                         .cra_module      = THIS_MODULE,
246                         .cra_init        = crc32_vx_cra_init_zero,
247                 },
248         },
249         /* CRC-32 BE */
250         {
251                 .init           =       crc32_vx_init,
252                 .setkey         =       crc32be_vx_setkey,
253                 .update         =       crc32be_vx_update,
254                 .final          =       crc32be_vx_final,
255                 .finup          =       crc32be_vx_finup,
256                 .digest         =       crc32be_vx_digest,
257                 .descsize       =       sizeof(struct crc_desc_ctx),
258                 .digestsize     =       CRC32_DIGEST_SIZE,
259                 .base           =       {
260                         .cra_name        = "crc32be",
261                         .cra_driver_name = "crc32be-vx",
262                         .cra_priority    = 200,
263                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
264                         .cra_blocksize   = CRC32_BLOCK_SIZE,
265                         .cra_ctxsize     = sizeof(struct crc_ctx),
266                         .cra_module      = THIS_MODULE,
267                         .cra_init        = crc32_vx_cra_init_zero,
268                 },
269         },
270         /* CRC-32C LE */
271         {
272                 .init           =       crc32_vx_init,
273                 .setkey         =       crc32_vx_setkey,
274                 .update         =       crc32c_vx_update,
275                 .final          =       crc32c_vx_final,
276                 .finup          =       crc32c_vx_finup,
277                 .digest         =       crc32c_vx_digest,
278                 .descsize       =       sizeof(struct crc_desc_ctx),
279                 .digestsize     =       CRC32_DIGEST_SIZE,
280                 .base           =       {
281                         .cra_name        = "crc32c",
282                         .cra_driver_name = "crc32c-vx",
283                         .cra_priority    = 200,
284                         .cra_flags       = CRYPTO_ALG_OPTIONAL_KEY,
285                         .cra_blocksize   = CRC32_BLOCK_SIZE,
286                         .cra_ctxsize     = sizeof(struct crc_ctx),
287                         .cra_module      = THIS_MODULE,
288                         .cra_init        = crc32_vx_cra_init_invert,
289                 },
290         },
291 };
292 
293 
294 static int __init crc_vx_mod_init(void)
295 {
296         return crypto_register_shashes(crc32_vx_algs,
297                                        ARRAY_SIZE(crc32_vx_algs));
298 }
299 
300 static void __exit crc_vx_mod_exit(void)
301 {
302         crypto_unregister_shashes(crc32_vx_algs, ARRAY_SIZE(crc32_vx_algs));
303 }
304 
305 module_cpu_feature_match(VXRS, crc_vx_mod_init);
306 module_exit(crc_vx_mod_exit);
307 
308 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>");
309 MODULE_LICENSE("GPL");
310 
311 MODULE_ALIAS_CRYPTO("crc32");
312 MODULE_ALIAS_CRYPTO("crc32-vx");
313 MODULE_ALIAS_CRYPTO("crc32c");
314 MODULE_ALIAS_CRYPTO("crc32c-vx");
315 

~ [ source navigation ] ~ [ diff markup ] ~ [ identifier search ] ~

kernel.org | git.kernel.org | LWN.net | Project Home | Wiki (Japanese) | Wiki (English) | SVN repository | Mail admin

Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.

osdn.jp