kernel/arch/arm/crypto/aes-ce-glue.c

   1 /*
   2  * aes-ce-glue.c - wrapper code for ARMv8 AES
   3  *
   4  * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/hwcap.h>
  12 #include <asm/neon.h>
  13 #include <asm/hwcap.h>
  14 #include <crypto/aes.h>
  15 #include <crypto/ablk_helper.h>
  16 #include <crypto/algapi.h>
  17 #include <linux/module.h>
  18
  19 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
  20 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  21 MODULE_LICENSE("GPL v2");
  22
  23 /* defined in aes-ce-core.S */
  24 asmlinkage u32 ce_aes_sub(u32 input);
  25 asmlinkage void ce_aes_invert(void *dst, void *src);
  26
  27 asmlinkage void ce_aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
  28                                    int rounds, int blocks);
  29 asmlinkage void ce_aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
  30                                    int rounds, int blocks);
  31
  32 asmlinkage void ce_aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
  33                                    int rounds, int blocks, u8 iv[]);
  34 asmlinkage void ce_aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
  35                                    int rounds, int blocks, u8 iv[]);
  36
  37 asmlinkage void ce_aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
  38                                    int rounds, int blocks, u8 ctr[]);
  39
  40 asmlinkage void ce_aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
  41                                    int rounds, int blocks, u8 iv[],
  42                                    u8 const rk2[], int first);
  43 asmlinkage void ce_aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
  44                                    int rounds, int blocks, u8 iv[],
  45                                    u8 const rk2[], int first);
  46
  47 struct aes_block {
  48         u8 b[AES_BLOCK_SIZE];
  49 };
  50
  51 static int num_rounds(struct crypto_aes_ctx *ctx)
  52 {
  53         /*
  54          * # of rounds specified by AES:
  55          * 128 bit key          10 rounds
  56          * 192 bit key          12 rounds
  57          * 256 bit key          14 rounds
  58          * => n byte key        => 6 + (n/4) rounds
  59          */
  60         return 6 + ctx->key_length / 4;
  61 }
  62
  63 static int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
  64                             unsigned int key_len)
  65 {
  66         /*
  67          * The AES key schedule round constants
  68          */
  69         static u8 const rcon[] = {
  70                 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
  71         };
  72
  73         u32 kwords = key_len / sizeof(u32);
  74         struct aes_block *key_enc, *key_dec;
  75         int i, j;
  76
  77         if (key_len != AES_KEYSIZE_128 &&
  78             key_len != AES_KEYSIZE_192 &&
  79             key_len != AES_KEYSIZE_256)
  80                 return -EINVAL;
  81
  82         memcpy(ctx->key_enc, in_key, key_len);
  83         ctx->key_length = key_len;
  84
  85         kernel_neon_begin();
  86         for (i = 0; i < sizeof(rcon); i++) {
  87                 u32 *rki = ctx->key_enc + (i * kwords);
  88                 u32 *rko = rki + kwords;
  89
  90                 rko[0] = ror32(ce_aes_sub(rki[kwords - 1]), 8);
  91                 rko[0] = rko[0] ^ rki[0] ^ rcon[i];
  92                 rko[1] = rko[0] ^ rki[1];
  93                 rko[2] = rko[1] ^ rki[2];
  94                 rko[3] = rko[2] ^ rki[3];
  95
  96                 if (key_len == AES_KEYSIZE_192) {
  97                         if (i >= 7)
  98                                 break;
  99                         rko[4] = rko[3] ^ rki[4];
 100                         rko[5] = rko[4] ^ rki[5];
 101                 } else if (key_len == AES_KEYSIZE_256) {
 102                         if (i >= 6)
 103                                 break;
 104                         rko[4] = ce_aes_sub(rko[3]) ^ rki[4];
 105                         rko[5] = rko[4] ^ rki[5];
 106                         rko[6] = rko[5] ^ rki[6];
 107                         rko[7] = rko[6] ^ rki[7];
 108                 }
 109         }
 110
 111         /*
 112          * Generate the decryption keys for the Equivalent Inverse Cipher.
 113          * This involves reversing the order of the round keys, and applying
 114          * the Inverse Mix Columns transformation on all but the first and
 115          * the last one.
 116          */
 117         key_enc = (struct aes_block *)ctx->key_enc;
 118         key_dec = (struct aes_block *)ctx->key_dec;
 119         j = num_rounds(ctx);
 120
 121         key_dec[0] = key_enc[j];
 122         for (i = 1, j--; j > 0; i++, j--)
 123                 ce_aes_invert(key_dec + i, key_enc + j);
 124         key_dec[i] = key_enc[0];
 125
 126         kernel_neon_end();
 127         return 0;
 128 }
 129
 130 static int ce_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
 131                          unsigned int key_len)
 132 {
 133         struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
 134         int ret;
 135
 136         ret = ce_aes_expandkey(ctx, in_key, key_len);
 137         if (!ret)
 138                 return 0;
 139
 140         tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 141         return -EINVAL;
 142 }
 143
 144 struct crypto_aes_xts_ctx {
 145         struct crypto_aes_ctx key1;
 146         struct crypto_aes_ctx __aligned(8) key2;
 147 };
 148
 149 static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
 150                        unsigned int key_len)
 151 {
 152         struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 153         int ret;
 154
 155         ret = ce_aes_expandkey(&ctx->key1, in_key, key_len / 2);
 156         if (!ret)
 157                 ret = ce_aes_expandkey(&ctx->key2, &in_key[key_len / 2],
 158                                        key_len / 2);
 159         if (!ret)
 160                 return 0;
 161
 162         tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 163         return -EINVAL;
 164 }
 165
 166 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 167                        struct scatterlist *src, unsigned int nbytes)
 168 {
 169         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 170         struct blkcipher_walk walk;
 171         unsigned int blocks;
 172         int err;
 173
 174         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 175         blkcipher_walk_init(&walk, dst, src, nbytes);
 176         err = blkcipher_walk_virt(desc, &walk);
 177
 178         kernel_neon_begin();
 179         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 180                 ce_aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 181                                    (u8 *)ctx->key_enc, num_rounds(ctx), blocks);
 182                 err = blkcipher_walk_done(desc, &walk,
 183                                           walk.nbytes % AES_BLOCK_SIZE);
 184         }
 185         kernel_neon_end();
 186         return err;
 187 }
 188
 189 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 190                        struct scatterlist *src, unsigned int nbytes)
 191 {
 192         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 193         struct blkcipher_walk walk;
 194         unsigned int blocks;
 195         int err;
 196
 197         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 198         blkcipher_walk_init(&walk, dst, src, nbytes);
 199         err = blkcipher_walk_virt(desc, &walk);
 200
 201         kernel_neon_begin();
 202         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 203                 ce_aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 204                                    (u8 *)ctx->key_dec, num_rounds(ctx), blocks);
 205                 err = blkcipher_walk_done(desc, &walk,
 206                                           walk.nbytes % AES_BLOCK_SIZE);
 207         }
 208         kernel_neon_end();
 209         return err;
 210 }
 211
 212 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 213                        struct scatterlist *src, unsigned int nbytes)
 214 {
 215         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 216         struct blkcipher_walk walk;
 217         unsigned int blocks;
 218         int err;
 219
 220         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 221         blkcipher_walk_init(&walk, dst, src, nbytes);
 222         err = blkcipher_walk_virt(desc, &walk);
 223
 224         kernel_neon_begin();
 225         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 226                 ce_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 227                                    (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
 228                                    walk.iv);
 229                 err = blkcipher_walk_done(desc, &walk,
 230                                           walk.nbytes % AES_BLOCK_SIZE);
 231         }
 232         kernel_neon_end();
 233         return err;
 234 }
 235
 236 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 237                        struct scatterlist *src, unsigned int nbytes)
 238 {
 239         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 240         struct blkcipher_walk walk;
 241         unsigned int blocks;
 242         int err;
 243
 244         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 245         blkcipher_walk_init(&walk, dst, src, nbytes);
 246         err = blkcipher_walk_virt(desc, &walk);
 247
 248         kernel_neon_begin();
 249         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 250                 ce_aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 251                                    (u8 *)ctx->key_dec, num_rounds(ctx), blocks,
 252                                    walk.iv);
 253                 err = blkcipher_walk_done(desc, &walk,
 254                                           walk.nbytes % AES_BLOCK_SIZE);
 255         }
 256         kernel_neon_end();
 257         return err;
 258 }
 259
 260 static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 261                        struct scatterlist *src, unsigned int nbytes)
 262 {
 263         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 264         struct blkcipher_walk walk;
 265         int err, blocks;
 266
 267         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 268         blkcipher_walk_init(&walk, dst, src, nbytes);
 269         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 270
 271         kernel_neon_begin();
 272         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 273                 ce_aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 274                                    (u8 *)ctx->key_enc, num_rounds(ctx), blocks,
 275                                    walk.iv);
 276                 nbytes -= blocks * AES_BLOCK_SIZE;
 277                 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
 278                         break;
 279                 err = blkcipher_walk_done(desc, &walk,
 280                                           walk.nbytes % AES_BLOCK_SIZE);
 281         }
 282         if (nbytes) {
 283                 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
 284                 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 285                 u8 __aligned(8) tail[AES_BLOCK_SIZE];
 286
 287                 /*
 288                  * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
 289                  * to tell aes_ctr_encrypt() to only read half a block.
 290                  */
 291                 blocks = (nbytes <= 8) ? -1 : 1;
 292
 293                 ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc,
 294                                    num_rounds(ctx), blocks, walk.iv);
 295                 memcpy(tdst, tail, nbytes);
 296                 err = blkcipher_walk_done(desc, &walk, 0);
 297         }
 298         kernel_neon_end();
 299
 300         return err;
 301 }
 302
 303 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 304                        struct scatterlist *src, unsigned int nbytes)
 305 {
 306         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 307         int err, first, rounds = num_rounds(&ctx->key1);
 308         struct blkcipher_walk walk;
 309         unsigned int blocks;
 310
 311         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 312         blkcipher_walk_init(&walk, dst, src, nbytes);
 313         err = blkcipher_walk_virt(desc, &walk);
 314
 315         kernel_neon_begin();
 316         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 317                 ce_aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 318                                    (u8 *)ctx->key1.key_enc, rounds, blocks,
 319                                    walk.iv, (u8 *)ctx->key2.key_enc, first);
 320                 err = blkcipher_walk_done(desc, &walk,
 321                                           walk.nbytes % AES_BLOCK_SIZE);
 322         }
 323         kernel_neon_end();
 324
 325         return err;
 326 }
 327
 328 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 329                        struct scatterlist *src, unsigned int nbytes)
 330 {
 331         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 332         int err, first, rounds = num_rounds(&ctx->key1);
 333         struct blkcipher_walk walk;
 334         unsigned int blocks;
 335
 336         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 337         blkcipher_walk_init(&walk, dst, src, nbytes);
 338         err = blkcipher_walk_virt(desc, &walk);
 339
 340         kernel_neon_begin();
 341         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 342                 ce_aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 343                                    (u8 *)ctx->key1.key_dec, rounds, blocks,
 344                                    walk.iv, (u8 *)ctx->key2.key_enc, first);
 345                 err = blkcipher_walk_done(desc, &walk,
 346                                           walk.nbytes % AES_BLOCK_SIZE);
 347         }
 348         kernel_neon_end();
 349
 350         return err;
 351 }
 352
 353 static struct crypto_alg aes_algs[] = { {
 354         .cra_name               = "__ecb-aes-ce",
 355         .cra_driver_name        = "__driver-ecb-aes-ce",
 356         .cra_priority           = 0,
 357         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 358                                   CRYPTO_ALG_INTERNAL,
 359         .cra_blocksize          = AES_BLOCK_SIZE,
 360         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 361         .cra_alignmask          = 7,
 362         .cra_type               = &crypto_blkcipher_type,
 363         .cra_module             = THIS_MODULE,
 364         .cra_blkcipher = {
 365                 .min_keysize    = AES_MIN_KEY_SIZE,
 366                 .max_keysize    = AES_MAX_KEY_SIZE,
 367                 .ivsize         = AES_BLOCK_SIZE,
 368                 .setkey         = ce_aes_setkey,
 369                 .encrypt        = ecb_encrypt,
 370                 .decrypt        = ecb_decrypt,
 371         },
 372 }, {
 373         .cra_name               = "__cbc-aes-ce",
 374         .cra_driver_name        = "__driver-cbc-aes-ce",
 375         .cra_priority           = 0,
 376         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 377                                   CRYPTO_ALG_INTERNAL,
 378         .cra_blocksize          = AES_BLOCK_SIZE,
 379         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 380         .cra_alignmask          = 7,
 381         .cra_type               = &crypto_blkcipher_type,
 382         .cra_module             = THIS_MODULE,
 383         .cra_blkcipher = {
 384                 .min_keysize    = AES_MIN_KEY_SIZE,
 385                 .max_keysize    = AES_MAX_KEY_SIZE,
 386                 .ivsize         = AES_BLOCK_SIZE,
 387                 .setkey         = ce_aes_setkey,
 388                 .encrypt        = cbc_encrypt,
 389                 .decrypt        = cbc_decrypt,
 390         },
 391 }, {
 392         .cra_name               = "__ctr-aes-ce",
 393         .cra_driver_name        = "__driver-ctr-aes-ce",
 394         .cra_priority           = 0,
 395         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 396                                   CRYPTO_ALG_INTERNAL,
 397         .cra_blocksize          = 1,
 398         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 399         .cra_alignmask          = 7,
 400         .cra_type               = &crypto_blkcipher_type,
 401         .cra_module             = THIS_MODULE,
 402         .cra_blkcipher = {
 403                 .min_keysize    = AES_MIN_KEY_SIZE,
 404                 .max_keysize    = AES_MAX_KEY_SIZE,
 405                 .ivsize         = AES_BLOCK_SIZE,
 406                 .setkey         = ce_aes_setkey,
 407                 .encrypt        = ctr_encrypt,
 408                 .decrypt        = ctr_encrypt,
 409         },
 410 }, {
 411         .cra_name               = "__xts-aes-ce",
 412         .cra_driver_name        = "__driver-xts-aes-ce",
 413         .cra_priority           = 0,
 414         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 415                                   CRYPTO_ALG_INTERNAL,
 416         .cra_blocksize          = AES_BLOCK_SIZE,
 417         .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 418         .cra_alignmask          = 7,
 419         .cra_type               = &crypto_blkcipher_type,
 420         .cra_module             = THIS_MODULE,
 421         .cra_blkcipher = {
 422                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 423                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 424                 .ivsize         = AES_BLOCK_SIZE,
 425                 .setkey         = xts_set_key,
 426                 .encrypt        = xts_encrypt,
 427                 .decrypt        = xts_decrypt,
 428         },
 429 }, {
 430         .cra_name               = "ecb(aes)",
 431         .cra_driver_name        = "ecb-aes-ce",
 432         .cra_priority           = 300,
 433         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 434         .cra_blocksize          = AES_BLOCK_SIZE,
 435         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 436         .cra_alignmask          = 7,
 437         .cra_type               = &crypto_ablkcipher_type,
 438         .cra_module             = THIS_MODULE,
 439         .cra_init               = ablk_init,
 440         .cra_exit               = ablk_exit,
 441         .cra_ablkcipher = {
 442                 .min_keysize    = AES_MIN_KEY_SIZE,
 443                 .max_keysize    = AES_MAX_KEY_SIZE,
 444                 .ivsize         = AES_BLOCK_SIZE,
 445                 .setkey         = ablk_set_key,
 446                 .encrypt        = ablk_encrypt,
 447                 .decrypt        = ablk_decrypt,
 448         }
 449 }, {
 450         .cra_name               = "cbc(aes)",
 451         .cra_driver_name        = "cbc-aes-ce",
 452         .cra_priority           = 300,
 453         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 454         .cra_blocksize          = AES_BLOCK_SIZE,
 455         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 456         .cra_alignmask          = 7,
 457         .cra_type               = &crypto_ablkcipher_type,
 458         .cra_module             = THIS_MODULE,
 459         .cra_init               = ablk_init,
 460         .cra_exit               = ablk_exit,
 461         .cra_ablkcipher = {
 462                 .min_keysize    = AES_MIN_KEY_SIZE,
 463                 .max_keysize    = AES_MAX_KEY_SIZE,
 464                 .ivsize         = AES_BLOCK_SIZE,
 465                 .setkey         = ablk_set_key,
 466                 .encrypt        = ablk_encrypt,
 467                 .decrypt        = ablk_decrypt,
 468         }
 469 }, {
 470         .cra_name               = "ctr(aes)",
 471         .cra_driver_name        = "ctr-aes-ce",
 472         .cra_priority           = 300,
 473         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 474         .cra_blocksize          = 1,
 475         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 476         .cra_alignmask          = 7,
 477         .cra_type               = &crypto_ablkcipher_type,
 478         .cra_module             = THIS_MODULE,
 479         .cra_init               = ablk_init,
 480         .cra_exit               = ablk_exit,
 481         .cra_ablkcipher = {
 482                 .min_keysize    = AES_MIN_KEY_SIZE,
 483                 .max_keysize    = AES_MAX_KEY_SIZE,
 484                 .ivsize         = AES_BLOCK_SIZE,
 485                 .setkey         = ablk_set_key,
 486                 .encrypt        = ablk_encrypt,
 487                 .decrypt        = ablk_decrypt,
 488         }
 489 }, {
 490         .cra_name               = "xts(aes)",
 491         .cra_driver_name        = "xts-aes-ce",
 492         .cra_priority           = 300,
 493         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 494         .cra_blocksize          = AES_BLOCK_SIZE,
 495         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 496         .cra_alignmask          = 7,
 497         .cra_type               = &crypto_ablkcipher_type,
 498         .cra_module             = THIS_MODULE,
 499         .cra_init               = ablk_init,
 500         .cra_exit               = ablk_exit,
 501         .cra_ablkcipher = {
 502                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 503                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 504                 .ivsize         = AES_BLOCK_SIZE,
 505                 .setkey         = ablk_set_key,
 506                 .encrypt        = ablk_encrypt,
 507                 .decrypt        = ablk_decrypt,
 508         }
 509 } };
 510
 511 static int __init aes_init(void)
 512 {
 513         if (!(elf_hwcap2 & HWCAP2_AES))
 514                 return -ENODEV;
 515         return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
 516 }
 517
 518 static void __exit aes_exit(void)
 519 {
 520         crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
 521 }
 522
 523 module_init(aes_init);
 524 module_exit(aes_exit);