kernel/arch/x86/crypto/glue_helper.c

   1 /*
   2  * Shared glue code for 128bit block ciphers
   3  *
   4  * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   5  *
   6  * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
   7  *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   8  * CTR part based on code (crypto/ctr.c) by:
   9  *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10  *
  11  * This program is free software; you can redistribute it and/or modify
  12  * it under the terms of the GNU General Public License as published by
  13  * the Free Software Foundation; either version 2 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU General Public License
  22  * along with this program; if not, write to the Free Software
  23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  24  * USA
  25  *
  26  */
  27
  28 #include <linux/module.h>
  29 #include <crypto/b128ops.h>
  30 #include <crypto/lrw.h>
  31 #include <crypto/xts.h>
  32 #include <asm/crypto/glue_helper.h>
  33 #include <crypto/scatterwalk.h>
  34
  35 static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  36                                    struct blkcipher_desc *desc,
  37                                    struct blkcipher_walk *walk)
  38 {
  39         void *ctx = crypto_blkcipher_ctx(desc->tfm);
  40         const unsigned int bsize = 128 / 8;
  41         unsigned int nbytes, i, func_bytes;
  42         bool fpu_enabled;
  43         int err;
  44
  45         err = blkcipher_walk_virt(desc, walk);
  46
  47         while ((nbytes = walk->nbytes)) {
  48                 u8 *wsrc = walk->src.virt.addr;
  49                 u8 *wdst = walk->dst.virt.addr;
  50
  51                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  52                                              desc, false, nbytes);
  53
  54                 for (i = 0; i < gctx->num_funcs; i++) {
  55                         func_bytes = bsize * gctx->funcs[i].num_blocks;
  56
  57                         /* Process multi-block batch */
  58                         if (nbytes >= func_bytes) {
  59                                 do {
  60                                         gctx->funcs[i].fn_u.ecb(ctx, wdst,
  61                                                                 wsrc);
  62
  63                                         wsrc += func_bytes;
  64                                         wdst += func_bytes;
  65                                         nbytes -= func_bytes;
  66                                 } while (nbytes >= func_bytes);
  67
  68                                 if (nbytes < bsize)
  69                                         goto done;
  70                         }
  71                 }
  72
  73 done:
  74                 glue_fpu_end(fpu_enabled);
  75                 err = blkcipher_walk_done(desc, walk, nbytes);
  76         }
  77
  78         return err;
  79 }
  80
  81 int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  82                           struct blkcipher_desc *desc, struct scatterlist *dst,
  83                           struct scatterlist *src, unsigned int nbytes)
  84 {
  85         struct blkcipher_walk walk;
  86
  87         blkcipher_walk_init(&walk, dst, src, nbytes);
  88         return __glue_ecb_crypt_128bit(gctx, desc, &walk);
  89 }
  90 EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
  91
  92 static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
  93                                               struct blkcipher_desc *desc,
  94                                               struct blkcipher_walk *walk)
  95 {
  96         void *ctx = crypto_blkcipher_ctx(desc->tfm);
  97         const unsigned int bsize = 128 / 8;
  98         unsigned int nbytes = walk->nbytes;
  99         u128 *src = (u128 *)walk->src.virt.addr;
 100         u128 *dst = (u128 *)walk->dst.virt.addr;
 101         u128 *iv = (u128 *)walk->iv;
 102
 103         do {
 104                 u128_xor(dst, src, iv);
 105                 fn(ctx, (u8 *)dst, (u8 *)dst);
 106                 iv = dst;
 107
 108                 src += 1;
 109                 dst += 1;
 110                 nbytes -= bsize;
 111         } while (nbytes >= bsize);
 112
 113         *(u128 *)walk->iv = *iv;
 114         return nbytes;
 115 }
 116
 117 int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
 118                             struct blkcipher_desc *desc,
 119                             struct scatterlist *dst,
 120                             struct scatterlist *src, unsigned int nbytes)
 121 {
 122         struct blkcipher_walk walk;
 123         int err;
 124
 125         blkcipher_walk_init(&walk, dst, src, nbytes);
 126         err = blkcipher_walk_virt(desc, &walk);
 127
 128         while ((nbytes = walk.nbytes)) {
 129                 nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
 130                 err = blkcipher_walk_done(desc, &walk, nbytes);
 131         }
 132
 133         return err;
 134 }
 135 EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
 136
 137 static unsigned int
 138 __glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 139                           struct blkcipher_desc *desc,
 140                           struct blkcipher_walk *walk)
 141 {
 142         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 143         const unsigned int bsize = 128 / 8;
 144         unsigned int nbytes = walk->nbytes;
 145         u128 *src = (u128 *)walk->src.virt.addr;
 146         u128 *dst = (u128 *)walk->dst.virt.addr;
 147         u128 last_iv;
 148         unsigned int num_blocks, func_bytes;
 149         unsigned int i;
 150
 151         /* Start of the last block. */
 152         src += nbytes / bsize - 1;
 153         dst += nbytes / bsize - 1;
 154
 155         last_iv = *src;
 156
 157         for (i = 0; i < gctx->num_funcs; i++) {
 158                 num_blocks = gctx->funcs[i].num_blocks;
 159                 func_bytes = bsize * num_blocks;
 160
 161                 /* Process multi-block batch */
 162                 if (nbytes >= func_bytes) {
 163                         do {
 164                                 nbytes -= func_bytes - bsize;
 165                                 src -= num_blocks - 1;
 166                                 dst -= num_blocks - 1;
 167
 168                                 gctx->funcs[i].fn_u.cbc(ctx, dst, src);
 169
 170                                 nbytes -= bsize;
 171                                 if (nbytes < bsize)
 172                                         goto done;
 173
 174                                 u128_xor(dst, dst, src - 1);
 175                                 src -= 1;
 176                                 dst -= 1;
 177                         } while (nbytes >= func_bytes);
 178
 179                         if (nbytes < bsize)
 180                                 goto done;
 181                 }
 182         }
 183
 184 done:
 185         u128_xor(dst, dst, (u128 *)walk->iv);
 186         *(u128 *)walk->iv = last_iv;
 187
 188         return nbytes;
 189 }
 190
 191 int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 192                             struct blkcipher_desc *desc,
 193                             struct scatterlist *dst,
 194                             struct scatterlist *src, unsigned int nbytes)
 195 {
 196         const unsigned int bsize = 128 / 8;
 197         bool fpu_enabled;
 198         struct blkcipher_walk walk;
 199         int err;
 200
 201         blkcipher_walk_init(&walk, dst, src, nbytes);
 202         err = blkcipher_walk_virt(desc, &walk);
 203
 204         while ((nbytes = walk.nbytes)) {
 205                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 206                                              desc, false, nbytes);
 207                 nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
 208                 glue_fpu_end(fpu_enabled);
 209                 err = blkcipher_walk_done(desc, &walk, nbytes);
 210         }
 211
 212         return err;
 213 }
 214 EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
 215
 216 static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
 217                                         struct blkcipher_desc *desc,
 218                                         struct blkcipher_walk *walk)
 219 {
 220         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 221         u8 *src = (u8 *)walk->src.virt.addr;
 222         u8 *dst = (u8 *)walk->dst.virt.addr;
 223         unsigned int nbytes = walk->nbytes;
 224         le128 ctrblk;
 225         u128 tmp;
 226
 227         be128_to_le128(&ctrblk, (be128 *)walk->iv);
 228
 229         memcpy(&tmp, src, nbytes);
 230         fn_ctr(ctx, &tmp, &tmp, &ctrblk);
 231         memcpy(dst, &tmp, nbytes);
 232
 233         le128_to_be128((be128 *)walk->iv, &ctrblk);
 234 }
 235
 236 static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 237                                             struct blkcipher_desc *desc,
 238                                             struct blkcipher_walk *walk)
 239 {
 240         const unsigned int bsize = 128 / 8;
 241         void *ctx = crypto_blkcipher_ctx(desc->tfm);
 242         unsigned int nbytes = walk->nbytes;
 243         u128 *src = (u128 *)walk->src.virt.addr;
 244         u128 *dst = (u128 *)walk->dst.virt.addr;
 245         le128 ctrblk;
 246         unsigned int num_blocks, func_bytes;
 247         unsigned int i;
 248
 249         be128_to_le128(&ctrblk, (be128 *)walk->iv);
 250
 251         /* Process multi-block batch */
 252         for (i = 0; i < gctx->num_funcs; i++) {
 253                 num_blocks = gctx->funcs[i].num_blocks;
 254                 func_bytes = bsize * num_blocks;
 255
 256                 if (nbytes >= func_bytes) {
 257                         do {
 258                                 gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
 259
 260                                 src += num_blocks;
 261                                 dst += num_blocks;
 262                                 nbytes -= func_bytes;
 263                         } while (nbytes >= func_bytes);
 264
 265                         if (nbytes < bsize)
 266                                 goto done;
 267                 }
 268         }
 269
 270 done:
 271         le128_to_be128((be128 *)walk->iv, &ctrblk);
 272         return nbytes;
 273 }
 274
 275 int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 276                           struct blkcipher_desc *desc, struct scatterlist *dst,
 277                           struct scatterlist *src, unsigned int nbytes)
 278 {
 279         const unsigned int bsize = 128 / 8;
 280         bool fpu_enabled;
 281         struct blkcipher_walk walk;
 282         int err;
 283
 284         blkcipher_walk_init(&walk, dst, src, nbytes);
 285         err = blkcipher_walk_virt_block(desc, &walk, bsize);
 286
 287         while ((nbytes = walk.nbytes) >= bsize) {
 288                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 289                                              desc, false, nbytes);
 290                 nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
 291                 glue_fpu_end(fpu_enabled);
 292                 err = blkcipher_walk_done(desc, &walk, nbytes);
 293         }
 294
 295         if (walk.nbytes) {
 296                 glue_ctr_crypt_final_128bit(
 297                         gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
 298                 err = blkcipher_walk_done(desc, &walk, 0);
 299         }
 300
 301         return err;
 302 }
 303 EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
 304
 305 static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 306                                             void *ctx,
 307                                             struct blkcipher_desc *desc,
 308                                             struct blkcipher_walk *walk)
 309 {
 310         const unsigned int bsize = 128 / 8;
 311         unsigned int nbytes = walk->nbytes;
 312         u128 *src = (u128 *)walk->src.virt.addr;
 313         u128 *dst = (u128 *)walk->dst.virt.addr;
 314         unsigned int num_blocks, func_bytes;
 315         unsigned int i;
 316
 317         /* Process multi-block batch */
 318         for (i = 0; i < gctx->num_funcs; i++) {
 319                 num_blocks = gctx->funcs[i].num_blocks;
 320                 func_bytes = bsize * num_blocks;
 321
 322                 if (nbytes >= func_bytes) {
 323                         do {
 324                                 gctx->funcs[i].fn_u.xts(ctx, dst, src,
 325                                                         (le128 *)walk->iv);
 326
 327                                 src += num_blocks;
 328                                 dst += num_blocks;
 329                                 nbytes -= func_bytes;
 330                         } while (nbytes >= func_bytes);
 331
 332                         if (nbytes < bsize)
 333                                 goto done;
 334                 }
 335         }
 336
 337 done:
 338         return nbytes;
 339 }
 340
 341 /* for implementations implementing faster XTS IV generator */
 342 int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 343                           struct blkcipher_desc *desc, struct scatterlist *dst,
 344                           struct scatterlist *src, unsigned int nbytes,
 345                           void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src),
 346                           void *tweak_ctx, void *crypt_ctx)
 347 {
 348         const unsigned int bsize = 128 / 8;
 349         bool fpu_enabled;
 350         struct blkcipher_walk walk;
 351         int err;
 352
 353         blkcipher_walk_init(&walk, dst, src, nbytes);
 354
 355         err = blkcipher_walk_virt(desc, &walk);
 356         nbytes = walk.nbytes;
 357         if (!nbytes)
 358                 return err;
 359
 360         /* set minimum length to bsize, for tweak_fn */
 361         fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 362                                      desc, false,
 363                                      nbytes < bsize ? bsize : nbytes);
 364         /* calculate first value of T */
 365         tweak_fn(tweak_ctx, walk.iv, walk.iv);
 366         glue_fpu_end(fpu_enabled);
 367
 368         while (nbytes) {
 369                 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 370                                 desc, false, nbytes);
 371                 nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
 372
 373                 glue_fpu_end(fpu_enabled);
 374                 err = blkcipher_walk_done(desc, &walk, nbytes);
 375                 nbytes = walk.nbytes;
 376         }
 377         return err;
 378 }
 379 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
 380
 381 void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
 382                                common_glue_func_t fn)
 383 {
 384         le128 ivblk = *iv;
 385
 386         /* generate next IV */
 387         le128_gf128mul_x_ble(iv, &ivblk);
 388
 389         /* CC <- T xor C */
 390         u128_xor(dst, src, (u128 *)&ivblk);
 391
 392         /* PP <- D(Key2,CC) */
 393         fn(ctx, (u8 *)dst, (u8 *)dst);
 394
 395         /* P <- T xor PP */
 396         u128_xor(dst, dst, (u128 *)&ivblk);
 397 }
 398 EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
 399
 400 MODULE_LICENSE("GPL");