kernel/arch/x86/crypto/camellia_aesni_avx2_glue.c

   1 /*
   2  * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
   3  *
   4  * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/crypto.h>
  16 #include <linux/err.h>
  17 #include <crypto/ablk_helper.h>
  18 #include <crypto/algapi.h>
  19 #include <crypto/ctr.h>
  20 #include <crypto/lrw.h>
  21 #include <crypto/xts.h>
  22 #include <asm/xcr.h>
  23 #include <asm/xsave.h>
  24 #include <asm/crypto/camellia.h>
  25 #include <asm/crypto/glue_helper.h>
  26
  27 #define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
  28 #define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
  29
  30 /* 32-way AVX2/AES-NI parallel cipher functions */
  31 asmlinkage void camellia_ecb_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  32                                        const u8 *src);
  33 asmlinkage void camellia_ecb_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  34                                        const u8 *src);
  35
  36 asmlinkage void camellia_cbc_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  37                                        const u8 *src);
  38 asmlinkage void camellia_ctr_32way(struct camellia_ctx *ctx, u8 *dst,
  39                                    const u8 *src, le128 *iv);
  40
  41 asmlinkage void camellia_xts_enc_32way(struct camellia_ctx *ctx, u8 *dst,
  42                                        const u8 *src, le128 *iv);
  43 asmlinkage void camellia_xts_dec_32way(struct camellia_ctx *ctx, u8 *dst,
  44                                        const u8 *src, le128 *iv);
  45
  46 static const struct common_glue_ctx camellia_enc = {
  47         .num_funcs = 4,
  48         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  49
  50         .funcs = { {
  51                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  52                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_32way) }
  53         }, {
  54                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  55                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_enc_16way) }
  56         }, {
  57                 .num_blocks = 2,
  58                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
  59         }, {
  60                 .num_blocks = 1,
  61                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
  62         } }
  63 };
  64
  65 static const struct common_glue_ctx camellia_ctr = {
  66         .num_funcs = 4,
  67         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  68
  69         .funcs = { {
  70                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  71                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_32way) }
  72         }, {
  73                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  74                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_ctr_16way) }
  75         }, {
  76                 .num_blocks = 2,
  77                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
  78         }, {
  79                 .num_blocks = 1,
  80                 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
  81         } }
  82 };
  83
  84 static const struct common_glue_ctx camellia_enc_xts = {
  85         .num_funcs = 3,
  86         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  87
  88         .funcs = { {
  89                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
  90                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_32way) }
  91         }, {
  92                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
  93                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc_16way) }
  94         }, {
  95                 .num_blocks = 1,
  96                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_enc) }
  97         } }
  98 };
  99
 100 static const struct common_glue_ctx camellia_dec = {
 101         .num_funcs = 4,
 102         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 103
 104         .funcs = { {
 105                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 106                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_32way) }
 107         }, {
 108                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 109                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_ecb_dec_16way) }
 110         }, {
 111                 .num_blocks = 2,
 112                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
 113         }, {
 114                 .num_blocks = 1,
 115                 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
 116         } }
 117 };
 118
 119 static const struct common_glue_ctx camellia_dec_cbc = {
 120         .num_funcs = 4,
 121         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 122
 123         .funcs = { {
 124                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 125                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_32way) }
 126         }, {
 127                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 128                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_cbc_dec_16way) }
 129         }, {
 130                 .num_blocks = 2,
 131                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
 132         }, {
 133                 .num_blocks = 1,
 134                 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
 135         } }
 136 };
 137
 138 static const struct common_glue_ctx camellia_dec_xts = {
 139         .num_funcs = 3,
 140         .fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 141
 142         .funcs = { {
 143                 .num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
 144                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_32way) }
 145         }, {
 146                 .num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
 147                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec_16way) }
 148         }, {
 149                 .num_blocks = 1,
 150                 .fn_u = { .xts = GLUE_XTS_FUNC_CAST(camellia_xts_dec) }
 151         } }
 152 };
 153
 154 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 155                        struct scatterlist *src, unsigned int nbytes)
 156 {
 157         return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
 158 }
 159
 160 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 161                        struct scatterlist *src, unsigned int nbytes)
 162 {
 163         return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
 164 }
 165
 166 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 167                        struct scatterlist *src, unsigned int nbytes)
 168 {
 169         return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
 170                                        dst, src, nbytes);
 171 }
 172
 173 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 174                        struct scatterlist *src, unsigned int nbytes)
 175 {
 176         return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
 177                                        nbytes);
 178 }
 179
 180 static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 181                      struct scatterlist *src, unsigned int nbytes)
 182 {
 183         return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
 184 }
 185
 186 static inline bool camellia_fpu_begin(bool fpu_enabled, unsigned int nbytes)
 187 {
 188         return glue_fpu_begin(CAMELLIA_BLOCK_SIZE,
 189                               CAMELLIA_AESNI_PARALLEL_BLOCKS, NULL, fpu_enabled,
 190                               nbytes);
 191 }
 192
 193 static inline void camellia_fpu_end(bool fpu_enabled)
 194 {
 195         glue_fpu_end(fpu_enabled);
 196 }
 197
 198 static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 199                            unsigned int key_len)
 200 {
 201         return __camellia_setkey(crypto_tfm_ctx(tfm), in_key, key_len,
 202                                  &tfm->crt_flags);
 203 }
 204
 205 struct crypt_priv {
 206         struct camellia_ctx *ctx;
 207         bool fpu_enabled;
 208 };
 209
 210 static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 211 {
 212         const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
 213         struct crypt_priv *ctx = priv;
 214         int i;
 215
 216         ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
 217
 218         if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
 219                 camellia_ecb_enc_32way(ctx->ctx, srcdst, srcdst);
 220                 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 221                 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 222         }
 223
 224         if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
 225                 camellia_ecb_enc_16way(ctx->ctx, srcdst, srcdst);
 226                 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 227                 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 228         }
 229
 230         while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
 231                 camellia_enc_blk_2way(ctx->ctx, srcdst, srcdst);
 232                 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
 233                 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
 234         }
 235
 236         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 237                 camellia_enc_blk(ctx->ctx, srcdst, srcdst);
 238 }
 239
 240 static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
 241 {
 242         const unsigned int bsize = CAMELLIA_BLOCK_SIZE;
 243         struct crypt_priv *ctx = priv;
 244         int i;
 245
 246         ctx->fpu_enabled = camellia_fpu_begin(ctx->fpu_enabled, nbytes);
 247
 248         if (nbytes >= CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS * bsize) {
 249                 camellia_ecb_dec_32way(ctx->ctx, srcdst, srcdst);
 250                 srcdst += bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 251                 nbytes -= bsize * CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS;
 252         }
 253
 254         if (nbytes >= CAMELLIA_AESNI_PARALLEL_BLOCKS * bsize) {
 255                 camellia_ecb_dec_16way(ctx->ctx, srcdst, srcdst);
 256                 srcdst += bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 257                 nbytes -= bsize * CAMELLIA_AESNI_PARALLEL_BLOCKS;
 258         }
 259
 260         while (nbytes >= CAMELLIA_PARALLEL_BLOCKS * bsize) {
 261                 camellia_dec_blk_2way(ctx->ctx, srcdst, srcdst);
 262                 srcdst += bsize * CAMELLIA_PARALLEL_BLOCKS;
 263                 nbytes -= bsize * CAMELLIA_PARALLEL_BLOCKS;
 264         }
 265
 266         for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
 267                 camellia_dec_blk(ctx->ctx, srcdst, srcdst);
 268 }
 269
 270 static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 271                        struct scatterlist *src, unsigned int nbytes)
 272 {
 273         struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 274         be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
 275         struct crypt_priv crypt_ctx = {
 276                 .ctx = &ctx->camellia_ctx,
 277                 .fpu_enabled = false,
 278         };
 279         struct lrw_crypt_req req = {
 280                 .tbuf = buf,
 281                 .tbuflen = sizeof(buf),
 282
 283                 .table_ctx = &ctx->lrw_table,
 284                 .crypt_ctx = &crypt_ctx,
 285                 .crypt_fn = encrypt_callback,
 286         };
 287         int ret;
 288
 289         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 290         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 291         camellia_fpu_end(crypt_ctx.fpu_enabled);
 292
 293         return ret;
 294 }
 295
 296 static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 297                        struct scatterlist *src, unsigned int nbytes)
 298 {
 299         struct camellia_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 300         be128 buf[CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS];
 301         struct crypt_priv crypt_ctx = {
 302                 .ctx = &ctx->camellia_ctx,
 303                 .fpu_enabled = false,
 304         };
 305         struct lrw_crypt_req req = {
 306                 .tbuf = buf,
 307                 .tbuflen = sizeof(buf),
 308
 309                 .table_ctx = &ctx->lrw_table,
 310                 .crypt_ctx = &crypt_ctx,
 311                 .crypt_fn = decrypt_callback,
 312         };
 313         int ret;
 314
 315         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 316         ret = lrw_crypt(desc, dst, src, nbytes, &req);
 317         camellia_fpu_end(crypt_ctx.fpu_enabled);
 318
 319         return ret;
 320 }
 321
 322 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 323                        struct scatterlist *src, unsigned int nbytes)
 324 {
 325         struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 326
 327         return glue_xts_crypt_128bit(&camellia_enc_xts, desc, dst, src, nbytes,
 328                                      XTS_TWEAK_CAST(camellia_enc_blk),
 329                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 330 }
 331
 332 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 333                        struct scatterlist *src, unsigned int nbytes)
 334 {
 335         struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 336
 337         return glue_xts_crypt_128bit(&camellia_dec_xts, desc, dst, src, nbytes,
 338                                      XTS_TWEAK_CAST(camellia_enc_blk),
 339                                      &ctx->tweak_ctx, &ctx->crypt_ctx);
 340 }
 341
 342 static struct crypto_alg cmll_algs[10] = { {
 343         .cra_name               = "__ecb-camellia-aesni-avx2",
 344         .cra_driver_name        = "__driver-ecb-camellia-aesni-avx2",
 345         .cra_priority           = 0,
 346         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 347                                   CRYPTO_ALG_INTERNAL,
 348         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 349         .cra_ctxsize            = sizeof(struct camellia_ctx),
 350         .cra_alignmask          = 0,
 351         .cra_type               = &crypto_blkcipher_type,
 352         .cra_module             = THIS_MODULE,
 353         .cra_u = {
 354                 .blkcipher = {
 355                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 356                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 357                         .setkey         = camellia_setkey,
 358                         .encrypt        = ecb_encrypt,
 359                         .decrypt        = ecb_decrypt,
 360                 },
 361         },
 362 }, {
 363         .cra_name               = "__cbc-camellia-aesni-avx2",
 364         .cra_driver_name        = "__driver-cbc-camellia-aesni-avx2",
 365         .cra_priority           = 0,
 366         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 367                                   CRYPTO_ALG_INTERNAL,
 368         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 369         .cra_ctxsize            = sizeof(struct camellia_ctx),
 370         .cra_alignmask          = 0,
 371         .cra_type               = &crypto_blkcipher_type,
 372         .cra_module             = THIS_MODULE,
 373         .cra_u = {
 374                 .blkcipher = {
 375                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 376                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 377                         .setkey         = camellia_setkey,
 378                         .encrypt        = cbc_encrypt,
 379                         .decrypt        = cbc_decrypt,
 380                 },
 381         },
 382 }, {
 383         .cra_name               = "__ctr-camellia-aesni-avx2",
 384         .cra_driver_name        = "__driver-ctr-camellia-aesni-avx2",
 385         .cra_priority           = 0,
 386         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 387                                   CRYPTO_ALG_INTERNAL,
 388         .cra_blocksize          = 1,
 389         .cra_ctxsize            = sizeof(struct camellia_ctx),
 390         .cra_alignmask          = 0,
 391         .cra_type               = &crypto_blkcipher_type,
 392         .cra_module             = THIS_MODULE,
 393         .cra_u = {
 394                 .blkcipher = {
 395                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 396                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 397                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 398                         .setkey         = camellia_setkey,
 399                         .encrypt        = ctr_crypt,
 400                         .decrypt        = ctr_crypt,
 401                 },
 402         },
 403 }, {
 404         .cra_name               = "__lrw-camellia-aesni-avx2",
 405         .cra_driver_name        = "__driver-lrw-camellia-aesni-avx2",
 406         .cra_priority           = 0,
 407         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 408                                   CRYPTO_ALG_INTERNAL,
 409         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 410         .cra_ctxsize            = sizeof(struct camellia_lrw_ctx),
 411         .cra_alignmask          = 0,
 412         .cra_type               = &crypto_blkcipher_type,
 413         .cra_module             = THIS_MODULE,
 414         .cra_exit               = lrw_camellia_exit_tfm,
 415         .cra_u = {
 416                 .blkcipher = {
 417                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
 418                                           CAMELLIA_BLOCK_SIZE,
 419                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
 420                                           CAMELLIA_BLOCK_SIZE,
 421                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 422                         .setkey         = lrw_camellia_setkey,
 423                         .encrypt        = lrw_encrypt,
 424                         .decrypt        = lrw_decrypt,
 425                 },
 426         },
 427 }, {
 428         .cra_name               = "__xts-camellia-aesni-avx2",
 429         .cra_driver_name        = "__driver-xts-camellia-aesni-avx2",
 430         .cra_priority           = 0,
 431         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 432                                   CRYPTO_ALG_INTERNAL,
 433         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 434         .cra_ctxsize            = sizeof(struct camellia_xts_ctx),
 435         .cra_alignmask          = 0,
 436         .cra_type               = &crypto_blkcipher_type,
 437         .cra_module             = THIS_MODULE,
 438         .cra_u = {
 439                 .blkcipher = {
 440                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
 441                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
 442                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 443                         .setkey         = xts_camellia_setkey,
 444                         .encrypt        = xts_encrypt,
 445                         .decrypt        = xts_decrypt,
 446                 },
 447         },
 448 }, {
 449         .cra_name               = "ecb(camellia)",
 450         .cra_driver_name        = "ecb-camellia-aesni-avx2",
 451         .cra_priority           = 500,
 452         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 453         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 454         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 455         .cra_alignmask          = 0,
 456         .cra_type               = &crypto_ablkcipher_type,
 457         .cra_module             = THIS_MODULE,
 458         .cra_init               = ablk_init,
 459         .cra_exit               = ablk_exit,
 460         .cra_u = {
 461                 .ablkcipher = {
 462                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 463                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 464                         .setkey         = ablk_set_key,
 465                         .encrypt        = ablk_encrypt,
 466                         .decrypt        = ablk_decrypt,
 467                 },
 468         },
 469 }, {
 470         .cra_name               = "cbc(camellia)",
 471         .cra_driver_name        = "cbc-camellia-aesni-avx2",
 472         .cra_priority           = 500,
 473         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 474         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 475         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 476         .cra_alignmask          = 0,
 477         .cra_type               = &crypto_ablkcipher_type,
 478         .cra_module             = THIS_MODULE,
 479         .cra_init               = ablk_init,
 480         .cra_exit               = ablk_exit,
 481         .cra_u = {
 482                 .ablkcipher = {
 483                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 484                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 485                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 486                         .setkey         = ablk_set_key,
 487                         .encrypt        = __ablk_encrypt,
 488                         .decrypt        = ablk_decrypt,
 489                 },
 490         },
 491 }, {
 492         .cra_name               = "ctr(camellia)",
 493         .cra_driver_name        = "ctr-camellia-aesni-avx2",
 494         .cra_priority           = 500,
 495         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 496         .cra_blocksize          = 1,
 497         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 498         .cra_alignmask          = 0,
 499         .cra_type               = &crypto_ablkcipher_type,
 500         .cra_module             = THIS_MODULE,
 501         .cra_init               = ablk_init,
 502         .cra_exit               = ablk_exit,
 503         .cra_u = {
 504                 .ablkcipher = {
 505                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE,
 506                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE,
 507                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 508                         .setkey         = ablk_set_key,
 509                         .encrypt        = ablk_encrypt,
 510                         .decrypt        = ablk_encrypt,
 511                         .geniv          = "chainiv",
 512                 },
 513         },
 514 }, {
 515         .cra_name               = "lrw(camellia)",
 516         .cra_driver_name        = "lrw-camellia-aesni-avx2",
 517         .cra_priority           = 500,
 518         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 519         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 520         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 521         .cra_alignmask          = 0,
 522         .cra_type               = &crypto_ablkcipher_type,
 523         .cra_module             = THIS_MODULE,
 524         .cra_init               = ablk_init,
 525         .cra_exit               = ablk_exit,
 526         .cra_u = {
 527                 .ablkcipher = {
 528                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE +
 529                                           CAMELLIA_BLOCK_SIZE,
 530                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE +
 531                                           CAMELLIA_BLOCK_SIZE,
 532                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 533                         .setkey         = ablk_set_key,
 534                         .encrypt        = ablk_encrypt,
 535                         .decrypt        = ablk_decrypt,
 536                 },
 537         },
 538 }, {
 539         .cra_name               = "xts(camellia)",
 540         .cra_driver_name        = "xts-camellia-aesni-avx2",
 541         .cra_priority           = 500,
 542         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
 543         .cra_blocksize          = CAMELLIA_BLOCK_SIZE,
 544         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 545         .cra_alignmask          = 0,
 546         .cra_type               = &crypto_ablkcipher_type,
 547         .cra_module             = THIS_MODULE,
 548         .cra_init               = ablk_init,
 549         .cra_exit               = ablk_exit,
 550         .cra_u = {
 551                 .ablkcipher = {
 552                         .min_keysize    = CAMELLIA_MIN_KEY_SIZE * 2,
 553                         .max_keysize    = CAMELLIA_MAX_KEY_SIZE * 2,
 554                         .ivsize         = CAMELLIA_BLOCK_SIZE,
 555                         .setkey         = ablk_set_key,
 556                         .encrypt        = ablk_encrypt,
 557                         .decrypt        = ablk_decrypt,
 558                 },
 559         },
 560 } };
 561
 562 static int __init camellia_aesni_init(void)
 563 {
 564         u64 xcr0;
 565
 566         if (!cpu_has_avx2 || !cpu_has_avx || !cpu_has_aes || !cpu_has_osxsave) {
 567                 pr_info("AVX2 or AES-NI instructions are not detected.\n");
 568                 return -ENODEV;
 569         }
 570
 571         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 572         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
 573                 pr_info("AVX2 detected but unusable.\n");
 574                 return -ENODEV;
 575         }
 576
 577         return crypto_register_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
 578 }
 579
 580 static void __exit camellia_aesni_fini(void)
 581 {
 582         crypto_unregister_algs(cmll_algs, ARRAY_SIZE(cmll_algs));
 583 }
 584
 585 module_init(camellia_aesni_init);
 586 module_exit(camellia_aesni_fini);
 587
 588 MODULE_LICENSE("GPL");
 589 MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
 590 MODULE_ALIAS_CRYPTO("camellia");
 591 MODULE_ALIAS_CRYPTO("camellia-asm");