2 * Glue Code for AVX assembler version of Twofish Cipher
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
7 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
26 #include <linux/module.h>
27 #include <linux/hardirq.h>
28 #include <linux/types.h>
29 #include <linux/crypto.h>
30 #include <linux/err.h>
31 #include <crypto/ablk_helper.h>
32 #include <crypto/algapi.h>
33 #include <crypto/twofish.h>
34 #include <crypto/cryptd.h>
35 #include <crypto/b128ops.h>
36 #include <crypto/ctr.h>
37 #include <crypto/lrw.h>
38 #include <crypto/xts.h>
41 #include <asm/xsave.h>
42 #include <asm/crypto/twofish.h>
43 #include <asm/crypto/glue_helper.h>
44 #include <crypto/scatterwalk.h>
45 #include <linux/workqueue.h>
46 #include <linux/spinlock.h>
48 #define TWOFISH_PARALLEL_BLOCKS 8
50 /* 8-way parallel cipher functions */
51 asmlinkage void twofish_ecb_enc_8way(struct twofish_ctx *ctx, u8 *dst,
53 asmlinkage void twofish_ecb_dec_8way(struct twofish_ctx *ctx, u8 *dst,
56 asmlinkage void twofish_cbc_dec_8way(struct twofish_ctx *ctx, u8 *dst,
58 asmlinkage void twofish_ctr_8way(struct twofish_ctx *ctx, u8 *dst,
59 const u8 *src, le128 *iv);
61 asmlinkage void twofish_xts_enc_8way(struct twofish_ctx *ctx, u8 *dst,
62 const u8 *src, le128 *iv);
63 asmlinkage void twofish_xts_dec_8way(struct twofish_ctx *ctx, u8 *dst,
64 const u8 *src, le128 *iv);
66 static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
69 __twofish_enc_blk_3way(ctx, dst, src, false);
72 static void twofish_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
74 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
75 GLUE_FUNC_CAST(twofish_enc_blk));
78 static void twofish_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
80 glue_xts_crypt_128bit_one(ctx, dst, src, iv,
81 GLUE_FUNC_CAST(twofish_dec_blk));
85 static const struct common_glue_ctx twofish_enc = {
87 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
90 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
91 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_enc_8way) }
94 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
97 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
101 static const struct common_glue_ctx twofish_ctr = {
103 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
106 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
107 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_ctr_8way) }
110 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
113 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
117 static const struct common_glue_ctx twofish_enc_xts = {
119 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
122 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
123 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc_8way) }
126 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_enc) }
130 static const struct common_glue_ctx twofish_dec = {
132 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
135 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
136 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_ecb_dec_8way) }
139 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
142 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
146 static const struct common_glue_ctx twofish_dec_cbc = {
148 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
151 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
152 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_cbc_dec_8way) }
155 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
158 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
162 static const struct common_glue_ctx twofish_dec_xts = {
164 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
167 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
168 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec_8way) }
171 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(twofish_xts_dec) }
175 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
176 struct scatterlist *src, unsigned int nbytes)
178 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
181 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
182 struct scatterlist *src, unsigned int nbytes)
184 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
187 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
188 struct scatterlist *src, unsigned int nbytes)
190 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
194 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
195 struct scatterlist *src, unsigned int nbytes)
197 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
201 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
202 struct scatterlist *src, unsigned int nbytes)
204 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
207 static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
209 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
210 fpu_enabled, nbytes);
213 static inline void twofish_fpu_end(bool fpu_enabled)
215 glue_fpu_end(fpu_enabled);
219 struct twofish_ctx *ctx;
223 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
225 const unsigned int bsize = TF_BLOCK_SIZE;
226 struct crypt_priv *ctx = priv;
229 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
231 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
232 twofish_ecb_enc_8way(ctx->ctx, srcdst, srcdst);
236 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
237 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
241 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
242 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
245 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
247 const unsigned int bsize = TF_BLOCK_SIZE;
248 struct crypt_priv *ctx = priv;
251 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
253 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
254 twofish_ecb_dec_8way(ctx->ctx, srcdst, srcdst);
258 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
259 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
263 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
264 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
267 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268 struct scatterlist *src, unsigned int nbytes)
270 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
271 be128 buf[TWOFISH_PARALLEL_BLOCKS];
272 struct crypt_priv crypt_ctx = {
273 .ctx = &ctx->twofish_ctx,
274 .fpu_enabled = false,
276 struct lrw_crypt_req req = {
278 .tbuflen = sizeof(buf),
280 .table_ctx = &ctx->lrw_table,
281 .crypt_ctx = &crypt_ctx,
282 .crypt_fn = encrypt_callback,
286 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
287 ret = lrw_crypt(desc, dst, src, nbytes, &req);
288 twofish_fpu_end(crypt_ctx.fpu_enabled);
293 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
294 struct scatterlist *src, unsigned int nbytes)
296 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
297 be128 buf[TWOFISH_PARALLEL_BLOCKS];
298 struct crypt_priv crypt_ctx = {
299 .ctx = &ctx->twofish_ctx,
300 .fpu_enabled = false,
302 struct lrw_crypt_req req = {
304 .tbuflen = sizeof(buf),
306 .table_ctx = &ctx->lrw_table,
307 .crypt_ctx = &crypt_ctx,
308 .crypt_fn = decrypt_callback,
312 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
313 ret = lrw_crypt(desc, dst, src, nbytes, &req);
314 twofish_fpu_end(crypt_ctx.fpu_enabled);
319 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
320 struct scatterlist *src, unsigned int nbytes)
322 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
324 return glue_xts_crypt_128bit(&twofish_enc_xts, desc, dst, src, nbytes,
325 XTS_TWEAK_CAST(twofish_enc_blk),
326 &ctx->tweak_ctx, &ctx->crypt_ctx);
329 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330 struct scatterlist *src, unsigned int nbytes)
332 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
334 return glue_xts_crypt_128bit(&twofish_dec_xts, desc, dst, src, nbytes,
335 XTS_TWEAK_CAST(twofish_enc_blk),
336 &ctx->tweak_ctx, &ctx->crypt_ctx);
339 static struct crypto_alg twofish_algs[10] = { {
340 .cra_name = "__ecb-twofish-avx",
341 .cra_driver_name = "__driver-ecb-twofish-avx",
343 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
345 .cra_blocksize = TF_BLOCK_SIZE,
346 .cra_ctxsize = sizeof(struct twofish_ctx),
348 .cra_type = &crypto_blkcipher_type,
349 .cra_module = THIS_MODULE,
352 .min_keysize = TF_MIN_KEY_SIZE,
353 .max_keysize = TF_MAX_KEY_SIZE,
354 .setkey = twofish_setkey,
355 .encrypt = ecb_encrypt,
356 .decrypt = ecb_decrypt,
360 .cra_name = "__cbc-twofish-avx",
361 .cra_driver_name = "__driver-cbc-twofish-avx",
363 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
365 .cra_blocksize = TF_BLOCK_SIZE,
366 .cra_ctxsize = sizeof(struct twofish_ctx),
368 .cra_type = &crypto_blkcipher_type,
369 .cra_module = THIS_MODULE,
372 .min_keysize = TF_MIN_KEY_SIZE,
373 .max_keysize = TF_MAX_KEY_SIZE,
374 .setkey = twofish_setkey,
375 .encrypt = cbc_encrypt,
376 .decrypt = cbc_decrypt,
380 .cra_name = "__ctr-twofish-avx",
381 .cra_driver_name = "__driver-ctr-twofish-avx",
383 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
386 .cra_ctxsize = sizeof(struct twofish_ctx),
388 .cra_type = &crypto_blkcipher_type,
389 .cra_module = THIS_MODULE,
392 .min_keysize = TF_MIN_KEY_SIZE,
393 .max_keysize = TF_MAX_KEY_SIZE,
394 .ivsize = TF_BLOCK_SIZE,
395 .setkey = twofish_setkey,
396 .encrypt = ctr_crypt,
397 .decrypt = ctr_crypt,
401 .cra_name = "__lrw-twofish-avx",
402 .cra_driver_name = "__driver-lrw-twofish-avx",
404 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
406 .cra_blocksize = TF_BLOCK_SIZE,
407 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
409 .cra_type = &crypto_blkcipher_type,
410 .cra_module = THIS_MODULE,
411 .cra_exit = lrw_twofish_exit_tfm,
414 .min_keysize = TF_MIN_KEY_SIZE +
416 .max_keysize = TF_MAX_KEY_SIZE +
418 .ivsize = TF_BLOCK_SIZE,
419 .setkey = lrw_twofish_setkey,
420 .encrypt = lrw_encrypt,
421 .decrypt = lrw_decrypt,
425 .cra_name = "__xts-twofish-avx",
426 .cra_driver_name = "__driver-xts-twofish-avx",
428 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
430 .cra_blocksize = TF_BLOCK_SIZE,
431 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
433 .cra_type = &crypto_blkcipher_type,
434 .cra_module = THIS_MODULE,
437 .min_keysize = TF_MIN_KEY_SIZE * 2,
438 .max_keysize = TF_MAX_KEY_SIZE * 2,
439 .ivsize = TF_BLOCK_SIZE,
440 .setkey = xts_twofish_setkey,
441 .encrypt = xts_encrypt,
442 .decrypt = xts_decrypt,
446 .cra_name = "ecb(twofish)",
447 .cra_driver_name = "ecb-twofish-avx",
449 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
450 .cra_blocksize = TF_BLOCK_SIZE,
451 .cra_ctxsize = sizeof(struct async_helper_ctx),
453 .cra_type = &crypto_ablkcipher_type,
454 .cra_module = THIS_MODULE,
455 .cra_init = ablk_init,
456 .cra_exit = ablk_exit,
459 .min_keysize = TF_MIN_KEY_SIZE,
460 .max_keysize = TF_MAX_KEY_SIZE,
461 .setkey = ablk_set_key,
462 .encrypt = ablk_encrypt,
463 .decrypt = ablk_decrypt,
467 .cra_name = "cbc(twofish)",
468 .cra_driver_name = "cbc-twofish-avx",
470 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
471 .cra_blocksize = TF_BLOCK_SIZE,
472 .cra_ctxsize = sizeof(struct async_helper_ctx),
474 .cra_type = &crypto_ablkcipher_type,
475 .cra_module = THIS_MODULE,
476 .cra_init = ablk_init,
477 .cra_exit = ablk_exit,
480 .min_keysize = TF_MIN_KEY_SIZE,
481 .max_keysize = TF_MAX_KEY_SIZE,
482 .ivsize = TF_BLOCK_SIZE,
483 .setkey = ablk_set_key,
484 .encrypt = __ablk_encrypt,
485 .decrypt = ablk_decrypt,
489 .cra_name = "ctr(twofish)",
490 .cra_driver_name = "ctr-twofish-avx",
492 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
494 .cra_ctxsize = sizeof(struct async_helper_ctx),
496 .cra_type = &crypto_ablkcipher_type,
497 .cra_module = THIS_MODULE,
498 .cra_init = ablk_init,
499 .cra_exit = ablk_exit,
502 .min_keysize = TF_MIN_KEY_SIZE,
503 .max_keysize = TF_MAX_KEY_SIZE,
504 .ivsize = TF_BLOCK_SIZE,
505 .setkey = ablk_set_key,
506 .encrypt = ablk_encrypt,
507 .decrypt = ablk_encrypt,
512 .cra_name = "lrw(twofish)",
513 .cra_driver_name = "lrw-twofish-avx",
515 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
516 .cra_blocksize = TF_BLOCK_SIZE,
517 .cra_ctxsize = sizeof(struct async_helper_ctx),
519 .cra_type = &crypto_ablkcipher_type,
520 .cra_module = THIS_MODULE,
521 .cra_init = ablk_init,
522 .cra_exit = ablk_exit,
525 .min_keysize = TF_MIN_KEY_SIZE +
527 .max_keysize = TF_MAX_KEY_SIZE +
529 .ivsize = TF_BLOCK_SIZE,
530 .setkey = ablk_set_key,
531 .encrypt = ablk_encrypt,
532 .decrypt = ablk_decrypt,
536 .cra_name = "xts(twofish)",
537 .cra_driver_name = "xts-twofish-avx",
539 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
540 .cra_blocksize = TF_BLOCK_SIZE,
541 .cra_ctxsize = sizeof(struct async_helper_ctx),
543 .cra_type = &crypto_ablkcipher_type,
544 .cra_module = THIS_MODULE,
545 .cra_init = ablk_init,
546 .cra_exit = ablk_exit,
549 .min_keysize = TF_MIN_KEY_SIZE * 2,
550 .max_keysize = TF_MAX_KEY_SIZE * 2,
551 .ivsize = TF_BLOCK_SIZE,
552 .setkey = ablk_set_key,
553 .encrypt = ablk_encrypt,
554 .decrypt = ablk_decrypt,
559 static int __init twofish_init(void)
563 if (!cpu_has_avx || !cpu_has_osxsave) {
564 printk(KERN_INFO "AVX instructions are not detected.\n");
568 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
569 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
570 printk(KERN_INFO "AVX detected but unusable.\n");
574 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
577 static void __exit twofish_exit(void)
579 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
582 module_init(twofish_init);
583 module_exit(twofish_exit);
585 MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
586 MODULE_LICENSE("GPL");
587 MODULE_ALIAS_CRYPTO("twofish");