kernel/arch/x86/crypto/sha256_ssse3_glue.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Glue code for the SHA256 Secure Hash Algorithm assembler
   5  * implementation using supplemental SSE3 / AVX / AVX2 instructions.
   6  *
   7  * This file is based on sha256_generic.c
   8  *
   9  * Copyright (C) 2013 Intel Corporation.
  10  *
  11  * Author:
  12  *     Tim Chen <tim.c.chen@linux.intel.com>
  13  *
  14  * This program is free software; you can redistribute it and/or modify it
  15  * under the terms of the GNU General Public License as published by the Free
  16  * Software Foundation; either version 2 of the License, or (at your option)
  17  * any later version.
  18  *
  19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  22  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  23  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  24  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  25  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  26  * SOFTWARE.
  27  */
  28
  29
  30 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  31
  32 #include <crypto/internal/hash.h>
  33 #include <linux/init.h>
  34 #include <linux/module.h>
  35 #include <linux/mm.h>
  36 #include <linux/cryptohash.h>
  37 #include <linux/types.h>
  38 #include <crypto/sha.h>
  39 #include <crypto/sha256_base.h>
  40 #include <asm/i387.h>
  41 #include <asm/xcr.h>
  42 #include <asm/xsave.h>
  43 #include <linux/string.h>
  44
  45 asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data,
  46                                        u64 rounds);
  47 #ifdef CONFIG_AS_AVX
  48 asmlinkage void sha256_transform_avx(u32 *digest, const char *data,
  49                                      u64 rounds);
  50 #endif
  51 #ifdef CONFIG_AS_AVX2
  52 asmlinkage void sha256_transform_rorx(u32 *digest, const char *data,
  53                                       u64 rounds);
  54 #endif
  55
  56 static void (*sha256_transform_asm)(u32 *, const char *, u64);
  57
  58 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
  59                              unsigned int len)
  60 {
  61         struct sha256_state *sctx = shash_desc_ctx(desc);
  62
  63         if (!irq_fpu_usable() ||
  64             (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
  65                 return crypto_sha256_update(desc, data, len);
  66
  67         /* make sure casting to sha256_block_fn() is safe */
  68         BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
  69
  70         kernel_fpu_begin();
  71         sha256_base_do_update(desc, data, len,
  72                               (sha256_block_fn *)sha256_transform_asm);
  73         kernel_fpu_end();
  74
  75         return 0;
  76 }
  77
  78 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
  79                               unsigned int len, u8 *out)
  80 {
  81         if (!irq_fpu_usable())
  82                 return crypto_sha256_finup(desc, data, len, out);
  83
  84         kernel_fpu_begin();
  85         if (len)
  86                 sha256_base_do_update(desc, data, len,
  87                                       (sha256_block_fn *)sha256_transform_asm);
  88         sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm);
  89         kernel_fpu_end();
  90
  91         return sha256_base_finish(desc, out);
  92 }
  93
  94 /* Add padding and return the message digest. */
  95 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
  96 {
  97         return sha256_ssse3_finup(desc, NULL, 0, out);
  98 }
  99
 100 static struct shash_alg algs[] = { {
 101         .digestsize     =       SHA256_DIGEST_SIZE,
 102         .init           =       sha256_base_init,
 103         .update         =       sha256_ssse3_update,
 104         .final          =       sha256_ssse3_final,
 105         .finup          =       sha256_ssse3_finup,
 106         .descsize       =       sizeof(struct sha256_state),
 107         .base           =       {
 108                 .cra_name       =       "sha256",
 109                 .cra_driver_name =      "sha256-ssse3",
 110                 .cra_priority   =       150,
 111                 .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
 112                 .cra_blocksize  =       SHA256_BLOCK_SIZE,
 113                 .cra_module     =       THIS_MODULE,
 114         }
 115 }, {
 116         .digestsize     =       SHA224_DIGEST_SIZE,
 117         .init           =       sha224_base_init,
 118         .update         =       sha256_ssse3_update,
 119         .final          =       sha256_ssse3_final,
 120         .finup          =       sha256_ssse3_finup,
 121         .descsize       =       sizeof(struct sha256_state),
 122         .base           =       {
 123                 .cra_name       =       "sha224",
 124                 .cra_driver_name =      "sha224-ssse3",
 125                 .cra_priority   =       150,
 126                 .cra_flags      =       CRYPTO_ALG_TYPE_SHASH,
 127                 .cra_blocksize  =       SHA224_BLOCK_SIZE,
 128                 .cra_module     =       THIS_MODULE,
 129         }
 130 } };
 131
 132 #ifdef CONFIG_AS_AVX
 133 static bool __init avx_usable(void)
 134 {
 135         u64 xcr0;
 136
 137         if (!cpu_has_avx || !cpu_has_osxsave)
 138                 return false;
 139
 140         xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
 141         if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
 142                 pr_info("AVX detected but unusable.\n");
 143
 144                 return false;
 145         }
 146
 147         return true;
 148 }
 149 #endif
 150
 151 static int __init sha256_ssse3_mod_init(void)
 152 {
 153         /* test for SSSE3 first */
 154         if (cpu_has_ssse3)
 155                 sha256_transform_asm = sha256_transform_ssse3;
 156
 157 #ifdef CONFIG_AS_AVX
 158         /* allow AVX to override SSSE3, it's a little faster */
 159         if (avx_usable()) {
 160 #ifdef CONFIG_AS_AVX2
 161                 if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2))
 162                         sha256_transform_asm = sha256_transform_rorx;
 163                 else
 164 #endif
 165                         sha256_transform_asm = sha256_transform_avx;
 166         }
 167 #endif
 168
 169         if (sha256_transform_asm) {
 170 #ifdef CONFIG_AS_AVX
 171                 if (sha256_transform_asm == sha256_transform_avx)
 172                         pr_info("Using AVX optimized SHA-256 implementation\n");
 173 #ifdef CONFIG_AS_AVX2
 174                 else if (sha256_transform_asm == sha256_transform_rorx)
 175                         pr_info("Using AVX2 optimized SHA-256 implementation\n");
 176 #endif
 177                 else
 178 #endif
 179                         pr_info("Using SSSE3 optimized SHA-256 implementation\n");
 180                 return crypto_register_shashes(algs, ARRAY_SIZE(algs));
 181         }
 182         pr_info("Neither AVX nor SSSE3 is available/usable.\n");
 183
 184         return -ENODEV;
 185 }
 186
 187 static void __exit sha256_ssse3_mod_fini(void)
 188 {
 189         crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
 190 }
 191
 192 module_init(sha256_ssse3_mod_init);
 193 module_exit(sha256_ssse3_mod_fini);
 194
 195 MODULE_LICENSE("GPL");
 196 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
 197
 198 MODULE_ALIAS_CRYPTO("sha256");
 199 MODULE_ALIAS_CRYPTO("sha224");