kernel/arch/x86/crypto/cast5_avx_glue.c

   1 /*
   2  * Glue Code for the AVX assembler implemention of the Cast5 Cipher
   3  *
   4  * Copyright (C) 2012 Johannes Goetzfried
   5  *     <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  20  * USA
  21  *
  22  */
  23
  24 #include <linux/module.h>
  25 #include <linux/hardirq.h>
  26 #include <linux/types.h>
  27 #include <linux/crypto.h>
  28 #include <linux/err.h>
  29 #include <crypto/ablk_helper.h>
  30 #include <crypto/algapi.h>
  31 #include <crypto/cast5.h>
  32 #include <crypto/cryptd.h>
  33 #include <crypto/ctr.h>
  34 #include <asm/fpu/api.h>
  35 #include <asm/crypto/glue_helper.h>
  36
  37 #define CAST5_PARALLEL_BLOCKS 16
  38
  39 asmlinkage void cast5_ecb_enc_16way(struct cast5_ctx *ctx, u8 *dst,
  40                                     const u8 *src);
  41 asmlinkage void cast5_ecb_dec_16way(struct cast5_ctx *ctx, u8 *dst,
  42                                     const u8 *src);
  43 asmlinkage void cast5_cbc_dec_16way(struct cast5_ctx *ctx, u8 *dst,
  44                                     const u8 *src);
  45 asmlinkage void cast5_ctr_16way(struct cast5_ctx *ctx, u8 *dst, const u8 *src,
  46                                 __be64 *iv);
  47
  48 static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes)
  49 {
  50         return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
  51                               NULL, fpu_enabled, nbytes);
  52 }
  53
  54 static inline void cast5_fpu_end(bool fpu_enabled)
  55 {
  56         return glue_fpu_end(fpu_enabled);
  57 }
  58
  59 static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
  60                      bool enc)
  61 {
  62         bool fpu_enabled;
  63         struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
  64         const unsigned int bsize = CAST5_BLOCK_SIZE;
  65         unsigned int nbytes;
  66         void (*fn)(struct cast5_ctx *ctx, u8 *dst, const u8 *src);
  67         int err;
  68
  69         fn = (enc) ? cast5_ecb_enc_16way : cast5_ecb_dec_16way;
  70
  71         err = blkcipher_walk_virt(desc, walk);
  72         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  73
  74         while ((nbytes = walk->nbytes)) {
  75                 u8 *wsrc = walk->src.virt.addr;
  76                 u8 *wdst = walk->dst.virt.addr;
  77
  78                 fpu_enabled = cast5_fpu_begin(false, nbytes);
  79
  80                 /* Process multi-block batch */
  81                 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
  82                         do {
  83                                 fn(ctx, wdst, wsrc);
  84
  85                                 wsrc += bsize * CAST5_PARALLEL_BLOCKS;
  86                                 wdst += bsize * CAST5_PARALLEL_BLOCKS;
  87                                 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
  88                         } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
  89
  90                         if (nbytes < bsize)
  91                                 goto done;
  92                 }
  93
  94                 fn = (enc) ? __cast5_encrypt : __cast5_decrypt;
  95
  96                 /* Handle leftovers */
  97                 do {
  98                         fn(ctx, wdst, wsrc);
  99
 100                         wsrc += bsize;
 101                         wdst += bsize;
 102                         nbytes -= bsize;
 103                 } while (nbytes >= bsize);
 104
 105 done:
 106                 cast5_fpu_end(fpu_enabled);
 107                 err = blkcipher_walk_done(desc, walk, nbytes);
 108         }
 109         return err;
 110 }
 111
 112 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 113                        struct scatterlist *src, unsigned int nbytes)
 114 {
 115         struct blkcipher_walk walk;
 116
 117         blkcipher_walk_init(&walk, dst, src, nbytes);
 118         return ecb_crypt(desc, &walk, true);
 119 }
 120
 121 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 122                        struct scatterlist *src, unsigned int nbytes)
 123 {
 124         struct blkcipher_walk walk;
 125
 126         blkcipher_walk_init(&walk, dst, src, nbytes);
 127         return ecb_crypt(desc, &walk, false);
 128 }
 129
 130 static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
 131                                   struct blkcipher_walk *walk)
 132 {
 133         struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 134         const unsigned int bsize = CAST5_BLOCK_SIZE;
 135         unsigned int nbytes = walk->nbytes;
 136         u64 *src = (u64 *)walk->src.virt.addr;
 137         u64 *dst = (u64 *)walk->dst.virt.addr;
 138         u64 *iv = (u64 *)walk->iv;
 139
 140         do {
 141                 *dst = *src ^ *iv;
 142                 __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
 143                 iv = dst;
 144
 145                 src += 1;
 146                 dst += 1;
 147                 nbytes -= bsize;
 148         } while (nbytes >= bsize);
 149
 150         *(u64 *)walk->iv = *iv;
 151         return nbytes;
 152 }
 153
 154 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 155                        struct scatterlist *src, unsigned int nbytes)
 156 {
 157         struct blkcipher_walk walk;
 158         int err;
 159
 160         blkcipher_walk_init(&walk, dst, src, nbytes);
 161         err = blkcipher_walk_virt(desc, &walk);
 162
 163         while ((nbytes = walk.nbytes)) {
 164                 nbytes = __cbc_encrypt(desc, &walk);
 165                 err = blkcipher_walk_done(desc, &walk, nbytes);
 166         }
 167
 168         return err;
 169 }
 170
 171 static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
 172                                   struct blkcipher_walk *walk)
 173 {
 174         struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 175         const unsigned int bsize = CAST5_BLOCK_SIZE;
 176         unsigned int nbytes = walk->nbytes;
 177         u64 *src = (u64 *)walk->src.virt.addr;
 178         u64 *dst = (u64 *)walk->dst.virt.addr;
 179         u64 last_iv;
 180
 181         /* Start of the last block. */
 182         src += nbytes / bsize - 1;
 183         dst += nbytes / bsize - 1;
 184
 185         last_iv = *src;
 186
 187         /* Process multi-block batch */
 188         if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
 189                 do {
 190                         nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
 191                         src -= CAST5_PARALLEL_BLOCKS - 1;
 192                         dst -= CAST5_PARALLEL_BLOCKS - 1;
 193
 194                         cast5_cbc_dec_16way(ctx, (u8 *)dst, (u8 *)src);
 195
 196                         nbytes -= bsize;
 197                         if (nbytes < bsize)
 198                                 goto done;
 199
 200                         *dst ^= *(src - 1);
 201                         src -= 1;
 202                         dst -= 1;
 203                 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
 204         }
 205
 206         /* Handle leftovers */
 207         for (;;) {
 208                 __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
 209
 210                 nbytes -= bsize;
 211                 if (nbytes < bsize)
 212                         break;
 213
 214                 *dst ^= *(src - 1);
 215                 src -= 1;
 216                 dst -= 1;
 217         }
 218
 219 done:
 220         *dst ^= *(u64 *)walk->iv;
 221         *(u64 *)walk->iv = last_iv;
 222
 223         return nbytes;
 224 }
 225
 226 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 227                        struct scatterlist *src, unsigned int nbytes)
 228 {
 229         bool fpu_enabled;
 230         struct blkcipher_walk walk;
 231         int err;
 232
 233         blkcipher_walk_init(&walk, dst, src, nbytes);
 234         err = blkcipher_walk_virt(desc, &walk);
 235         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 236
 237         while ((nbytes = walk.nbytes)) {
 238                 fpu_enabled = cast5_fpu_begin(false, nbytes);
 239                 nbytes = __cbc_decrypt(desc, &walk);
 240                 cast5_fpu_end(fpu_enabled);
 241                 err = blkcipher_walk_done(desc, &walk, nbytes);
 242         }
 243         return err;
 244 }
 245
 246 static void ctr_crypt_final(struct blkcipher_desc *desc,
 247                             struct blkcipher_walk *walk)
 248 {
 249         struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 250         u8 *ctrblk = walk->iv;
 251         u8 keystream[CAST5_BLOCK_SIZE];
 252         u8 *src = walk->src.virt.addr;
 253         u8 *dst = walk->dst.virt.addr;
 254         unsigned int nbytes = walk->nbytes;
 255
 256         __cast5_encrypt(ctx, keystream, ctrblk);
 257         crypto_xor(keystream, src, nbytes);
 258         memcpy(dst, keystream, nbytes);
 259
 260         crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
 261 }
 262
 263 static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
 264                                 struct blkcipher_walk *walk)
 265 {
 266         struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 267         const unsigned int bsize = CAST5_BLOCK_SIZE;
 268         unsigned int nbytes = walk->nbytes;
 269         u64 *src = (u64 *)walk->src.virt.addr;
 270         u64 *dst = (u64 *)walk->dst.virt.addr;
 271
 272         /* Process multi-block batch */
 273         if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
 274                 do {
 275                         cast5_ctr_16way(ctx, (u8 *)dst, (u8 *)src,
 276                                         (__be64 *)walk->iv);
 277
 278                         src += CAST5_PARALLEL_BLOCKS;
 279                         dst += CAST5_PARALLEL_BLOCKS;
 280                         nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
 281                 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
 282
 283                 if (nbytes < bsize)
 284                         goto done;
 285         }
 286
 287         /* Handle leftovers */
 288         do {
 289                 u64 ctrblk;
 290
 291                 if (dst != src)
 292                         *dst = *src;
 293
 294                 ctrblk = *(u64 *)walk->iv;
 295                 be64_add_cpu((__be64 *)walk->iv, 1);
 296
 297                 __cast5_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
 298                 *dst ^= ctrblk;
 299
 300                 src += 1;
 301                 dst += 1;
 302                 nbytes -= bsize;
 303         } while (nbytes >= bsize);
 304
 305 done:
 306         return nbytes;
 307 }
 308
 309 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 310                      struct scatterlist *src, unsigned int nbytes)
 311 {
 312         bool fpu_enabled;
 313         struct blkcipher_walk walk;
 314         int err;
 315
 316         blkcipher_walk_init(&walk, dst, src, nbytes);
 317         err = blkcipher_walk_virt_block(desc, &walk, CAST5_BLOCK_SIZE);
 318         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 319
 320         while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
 321                 fpu_enabled = cast5_fpu_begin(false, nbytes);
 322                 nbytes = __ctr_crypt(desc, &walk);
 323                 cast5_fpu_end(fpu_enabled);
 324                 err = blkcipher_walk_done(desc, &walk, nbytes);
 325         }
 326
 327         if (walk.nbytes) {
 328                 ctr_crypt_final(desc, &walk);
 329                 err = blkcipher_walk_done(desc, &walk, 0);
 330         }
 331
 332         return err;
 333 }
 334
 335
 336 static struct crypto_alg cast5_algs[6] = { {
 337         .cra_name               = "__ecb-cast5-avx",
 338         .cra_driver_name        = "__driver-ecb-cast5-avx",
 339         .cra_priority           = 0,
 340         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 341                                   CRYPTO_ALG_INTERNAL,
 342         .cra_blocksize          = CAST5_BLOCK_SIZE,
 343         .cra_ctxsize            = sizeof(struct cast5_ctx),
 344         .cra_alignmask          = 0,
 345         .cra_type               = &crypto_blkcipher_type,
 346         .cra_module             = THIS_MODULE,
 347         .cra_u = {
 348                 .blkcipher = {
 349                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 350                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 351                         .setkey         = cast5_setkey,
 352                         .encrypt        = ecb_encrypt,
 353                         .decrypt        = ecb_decrypt,
 354                 },
 355         },
 356 }, {
 357         .cra_name               = "__cbc-cast5-avx",
 358         .cra_driver_name        = "__driver-cbc-cast5-avx",
 359         .cra_priority           = 0,
 360         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 361                                   CRYPTO_ALG_INTERNAL,
 362         .cra_blocksize          = CAST5_BLOCK_SIZE,
 363         .cra_ctxsize            = sizeof(struct cast5_ctx),
 364         .cra_alignmask          = 0,
 365         .cra_type               = &crypto_blkcipher_type,
 366         .cra_module             = THIS_MODULE,
 367         .cra_u = {
 368                 .blkcipher = {
 369                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 370                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 371                         .setkey         = cast5_setkey,
 372                         .encrypt        = cbc_encrypt,
 373                         .decrypt        = cbc_decrypt,
 374                 },
 375         },
 376 }, {
 377         .cra_name               = "__ctr-cast5-avx",
 378         .cra_driver_name        = "__driver-ctr-cast5-avx",
 379         .cra_priority           = 0,
 380         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 381                                   CRYPTO_ALG_INTERNAL,
 382         .cra_blocksize          = 1,
 383         .cra_ctxsize            = sizeof(struct cast5_ctx),
 384         .cra_alignmask          = 0,
 385         .cra_type               = &crypto_blkcipher_type,
 386         .cra_module             = THIS_MODULE,
 387         .cra_u = {
 388                 .blkcipher = {
 389                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 390                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 391                         .ivsize         = CAST5_BLOCK_SIZE,
 392                         .setkey         = cast5_setkey,
 393                         .encrypt        = ctr_crypt,
 394                         .decrypt        = ctr_crypt,
 395                 },
 396         },
 397 }, {
 398         .cra_name               = "ecb(cast5)",
 399         .cra_driver_name        = "ecb-cast5-avx",
 400         .cra_priority           = 200,
 401         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 402         .cra_blocksize          = CAST5_BLOCK_SIZE,
 403         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 404         .cra_alignmask          = 0,
 405         .cra_type               = &crypto_ablkcipher_type,
 406         .cra_module             = THIS_MODULE,
 407         .cra_init               = ablk_init,
 408         .cra_exit               = ablk_exit,
 409         .cra_u = {
 410                 .ablkcipher = {
 411                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 412                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 413                         .setkey         = ablk_set_key,
 414                         .encrypt        = ablk_encrypt,
 415                         .decrypt        = ablk_decrypt,
 416                 },
 417         },
 418 }, {
 419         .cra_name               = "cbc(cast5)",
 420         .cra_driver_name        = "cbc-cast5-avx",
 421         .cra_priority           = 200,
 422         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 423         .cra_blocksize          = CAST5_BLOCK_SIZE,
 424         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 425         .cra_alignmask          = 0,
 426         .cra_type               = &crypto_ablkcipher_type,
 427         .cra_module             = THIS_MODULE,
 428         .cra_init               = ablk_init,
 429         .cra_exit               = ablk_exit,
 430         .cra_u = {
 431                 .ablkcipher = {
 432                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 433                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 434                         .ivsize         = CAST5_BLOCK_SIZE,
 435                         .setkey         = ablk_set_key,
 436                         .encrypt        = __ablk_encrypt,
 437                         .decrypt        = ablk_decrypt,
 438                 },
 439         },
 440 }, {
 441         .cra_name               = "ctr(cast5)",
 442         .cra_driver_name        = "ctr-cast5-avx",
 443         .cra_priority           = 200,
 444         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 445         .cra_blocksize          = 1,
 446         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 447         .cra_alignmask          = 0,
 448         .cra_type               = &crypto_ablkcipher_type,
 449         .cra_module             = THIS_MODULE,
 450         .cra_init               = ablk_init,
 451         .cra_exit               = ablk_exit,
 452         .cra_u = {
 453                 .ablkcipher = {
 454                         .min_keysize    = CAST5_MIN_KEY_SIZE,
 455                         .max_keysize    = CAST5_MAX_KEY_SIZE,
 456                         .ivsize         = CAST5_BLOCK_SIZE,
 457                         .setkey         = ablk_set_key,
 458                         .encrypt        = ablk_encrypt,
 459                         .decrypt        = ablk_encrypt,
 460                         .geniv          = "chainiv",
 461                 },
 462         },
 463 } };
 464
 465 static int __init cast5_init(void)
 466 {
 467         const char *feature_name;
 468
 469         if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
 470                                 &feature_name)) {
 471                 pr_info("CPU feature '%s' is not supported.\n", feature_name);
 472                 return -ENODEV;
 473         }
 474
 475         return crypto_register_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
 476 }
 477
 478 static void __exit cast5_exit(void)
 479 {
 480         crypto_unregister_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
 481 }
 482
 483 module_init(cast5_init);
 484 module_exit(cast5_exit);
 485
 486 MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
 487 MODULE_LICENSE("GPL");
 488 MODULE_ALIAS_CRYPTO("cast5");