Add the rt linux 4.1.3-rt3 as base

[kvmfornfv.git] / kernel / arch / arm64 / crypto / ghash-ce-glue.c

diff --git a/kernel/arch/arm64/crypto/ghash-ce-glue.c b/kernel/arch/arm64/crypto/ghash-ce-glue.c
new file mode 100644 (file)
index 0000000..833ec1e
--- /dev/null
+++ b/kernel/arch/arm64/crypto/ghash-ce-glue.c
@@ -0,0 +1,156 @@
+/*
+ * Accelerated GHASH implementation with ARMv8 PMULL instructions.
+ *
+ * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <asm/unaligned.h>
+#include <crypto/internal/hash.h>
+#include <linux/cpufeature.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+#define GHASH_BLOCK_SIZE       16
+#define GHASH_DIGEST_SIZE      16
+
+struct ghash_key {
+       u64 a;
+       u64 b;
+};
+
+struct ghash_desc_ctx {
+       u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
+       u8 buf[GHASH_BLOCK_SIZE];
+       u32 count;
+};
+
+asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
+                                  struct ghash_key const *k, const char *head);
+
+static int ghash_init(struct shash_desc *desc)
+{
+       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+
+       *ctx = (struct ghash_desc_ctx){};
+       return 0;
+}
+
+static int ghash_update(struct shash_desc *desc, const u8 *src,
+                       unsigned int len)
+{
+       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
+
+       ctx->count += len;
+
+       if ((partial + len) >= GHASH_BLOCK_SIZE) {
+               struct ghash_key *key = crypto_shash_ctx(desc->tfm);
+               int blocks;
+
+               if (partial) {
+                       int p = GHASH_BLOCK_SIZE - partial;
+
+                       memcpy(ctx->buf + partial, src, p);
+                       src += p;
+                       len -= p;
+               }
+
+               blocks = len / GHASH_BLOCK_SIZE;
+               len %= GHASH_BLOCK_SIZE;
+
+               kernel_neon_begin_partial(8);
+               pmull_ghash_update(blocks, ctx->digest, src, key,
+                                  partial ? ctx->buf : NULL);
+               kernel_neon_end();
+               src += blocks * GHASH_BLOCK_SIZE;
+               partial = 0;
+       }
+       if (len)
+               memcpy(ctx->buf + partial, src, len);
+       return 0;
+}
+
+static int ghash_final(struct shash_desc *desc, u8 *dst)
+{
+       struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
+       unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
+
+       if (partial) {
+               struct ghash_key *key = crypto_shash_ctx(desc->tfm);
+
+               memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
+
+               kernel_neon_begin_partial(8);
+               pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
+               kernel_neon_end();
+       }
+       put_unaligned_be64(ctx->digest[1], dst);
+       put_unaligned_be64(ctx->digest[0], dst + 8);
+
+       *ctx = (struct ghash_desc_ctx){};
+       return 0;
+}
+
+static int ghash_setkey(struct crypto_shash *tfm,
+                       const u8 *inkey, unsigned int keylen)
+{
+       struct ghash_key *key = crypto_shash_ctx(tfm);
+       u64 a, b;
+
+       if (keylen != GHASH_BLOCK_SIZE) {
+               crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+               return -EINVAL;
+       }
+
+       /* perform multiplication by 'x' in GF(2^128) */
+       b = get_unaligned_be64(inkey);
+       a = get_unaligned_be64(inkey + 8);
+
+       key->a = (a << 1) | (b >> 63);
+       key->b = (b << 1) | (a >> 63);
+
+       if (b >> 63)
+               key->b ^= 0xc200000000000000UL;
+
+       return 0;
+}
+
+static struct shash_alg ghash_alg = {
+       .digestsize     = GHASH_DIGEST_SIZE,
+       .init           = ghash_init,
+       .update         = ghash_update,
+       .final          = ghash_final,
+       .setkey         = ghash_setkey,
+       .descsize       = sizeof(struct ghash_desc_ctx),
+       .base           = {
+               .cra_name               = "ghash",
+               .cra_driver_name        = "ghash-ce",
+               .cra_priority           = 200,
+               .cra_flags              = CRYPTO_ALG_TYPE_SHASH,
+               .cra_blocksize          = GHASH_BLOCK_SIZE,
+               .cra_ctxsize            = sizeof(struct ghash_key),
+               .cra_module             = THIS_MODULE,
+       },
+};
+
+static int __init ghash_ce_mod_init(void)
+{
+       return crypto_register_shash(&ghash_alg);
+}
+
+static void __exit ghash_ce_mod_exit(void)
+{
+       crypto_unregister_shash(&ghash_alg);
+}
+
+module_cpu_feature_match(PMULL, ghash_ce_mod_init);
+module_exit(ghash_ce_mod_exit);