kernel/drivers/crypto/qat/qat_common/qat_algs.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/module.h>
  48 #include <linux/slab.h>
  49 #include <linux/crypto.h>
  50 #include <crypto/aead.h>
  51 #include <crypto/aes.h>
  52 #include <crypto/sha.h>
  53 #include <crypto/hash.h>
  54 #include <crypto/algapi.h>
  55 #include <crypto/authenc.h>
  56 #include <crypto/rng.h>
  57 #include <linux/dma-mapping.h>
  58 #include "adf_accel_devices.h"
  59 #include "adf_transport.h"
  60 #include "adf_common_drv.h"
  61 #include "qat_crypto.h"
  62 #include "icp_qat_hw.h"
  63 #include "icp_qat_fw.h"
  64 #include "icp_qat_fw_la.h"
  65
  66 #define QAT_AES_HW_CONFIG_CBC_ENC(alg) \
  67         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
  68                                        ICP_QAT_HW_CIPHER_NO_CONVERT, \
  69                                        ICP_QAT_HW_CIPHER_ENCRYPT)
  70
  71 #define QAT_AES_HW_CONFIG_CBC_DEC(alg) \
  72         ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \
  73                                        ICP_QAT_HW_CIPHER_KEY_CONVERT, \
  74                                        ICP_QAT_HW_CIPHER_DECRYPT)
  75
  76 static DEFINE_MUTEX(algs_lock);
  77 static unsigned int active_devs;
  78
  79 struct qat_alg_buf {
  80         uint32_t len;
  81         uint32_t resrvd;
  82         uint64_t addr;
  83 } __packed;
  84
  85 struct qat_alg_buf_list {
  86         uint64_t resrvd;
  87         uint32_t num_bufs;
  88         uint32_t num_mapped_bufs;
  89         struct qat_alg_buf bufers[];
  90 } __packed __aligned(64);
  91
  92 /* Common content descriptor */
  93 struct qat_alg_cd {
  94         union {
  95                 struct qat_enc { /* Encrypt content desc */
  96                         struct icp_qat_hw_cipher_algo_blk cipher;
  97                         struct icp_qat_hw_auth_algo_blk hash;
  98                 } qat_enc_cd;
  99                 struct qat_dec { /* Decrytp content desc */
 100                         struct icp_qat_hw_auth_algo_blk hash;
 101                         struct icp_qat_hw_cipher_algo_blk cipher;
 102                 } qat_dec_cd;
 103         };
 104 } __aligned(64);
 105
 106 struct qat_alg_aead_ctx {
 107         struct qat_alg_cd *enc_cd;
 108         struct qat_alg_cd *dec_cd;
 109         dma_addr_t enc_cd_paddr;
 110         dma_addr_t dec_cd_paddr;
 111         struct icp_qat_fw_la_bulk_req enc_fw_req;
 112         struct icp_qat_fw_la_bulk_req dec_fw_req;
 113         struct crypto_shash *hash_tfm;
 114         enum icp_qat_hw_auth_algo qat_hash_alg;
 115         struct qat_crypto_instance *inst;
 116         struct crypto_tfm *tfm;
 117         uint8_t salt[AES_BLOCK_SIZE];
 118         spinlock_t lock;        /* protects qat_alg_aead_ctx struct */
 119 };
 120
 121 struct qat_alg_ablkcipher_ctx {
 122         struct icp_qat_hw_cipher_algo_blk *enc_cd;
 123         struct icp_qat_hw_cipher_algo_blk *dec_cd;
 124         dma_addr_t enc_cd_paddr;
 125         dma_addr_t dec_cd_paddr;
 126         struct icp_qat_fw_la_bulk_req enc_fw_req;
 127         struct icp_qat_fw_la_bulk_req dec_fw_req;
 128         struct qat_crypto_instance *inst;
 129         struct crypto_tfm *tfm;
 130         spinlock_t lock;        /* protects qat_alg_ablkcipher_ctx struct */
 131 };
 132
 133 static int get_current_node(void)
 134 {
 135         return cpu_data(current_thread_info()->cpu).phys_proc_id;
 136 }
 137
 138 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
 139 {
 140         switch (qat_hash_alg) {
 141         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 142                 return ICP_QAT_HW_SHA1_STATE1_SZ;
 143         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 144                 return ICP_QAT_HW_SHA256_STATE1_SZ;
 145         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 146                 return ICP_QAT_HW_SHA512_STATE1_SZ;
 147         default:
 148                 return -EFAULT;
 149         };
 150         return -EFAULT;
 151 }
 152
 153 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
 154                                   struct qat_alg_aead_ctx *ctx,
 155                                   const uint8_t *auth_key,
 156                                   unsigned int auth_keylen)
 157 {
 158         SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
 159         struct sha1_state sha1;
 160         struct sha256_state sha256;
 161         struct sha512_state sha512;
 162         int block_size = crypto_shash_blocksize(ctx->hash_tfm);
 163         int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
 164         char ipad[block_size];
 165         char opad[block_size];
 166         __be32 *hash_state_out;
 167         __be64 *hash512_state_out;
 168         int i, offset;
 169
 170         memset(ipad, 0, block_size);
 171         memset(opad, 0, block_size);
 172         shash->tfm = ctx->hash_tfm;
 173         shash->flags = 0x0;
 174
 175         if (auth_keylen > block_size) {
 176                 int ret = crypto_shash_digest(shash, auth_key,
 177                                               auth_keylen, ipad);
 178                 if (ret)
 179                         return ret;
 180
 181                 memcpy(opad, ipad, digest_size);
 182         } else {
 183                 memcpy(ipad, auth_key, auth_keylen);
 184                 memcpy(opad, auth_key, auth_keylen);
 185         }
 186
 187         for (i = 0; i < block_size; i++) {
 188                 char *ipad_ptr = ipad + i;
 189                 char *opad_ptr = opad + i;
 190                 *ipad_ptr ^= 0x36;
 191                 *opad_ptr ^= 0x5C;
 192         }
 193
 194         if (crypto_shash_init(shash))
 195                 return -EFAULT;
 196
 197         if (crypto_shash_update(shash, ipad, block_size))
 198                 return -EFAULT;
 199
 200         hash_state_out = (__be32 *)hash->sha.state1;
 201         hash512_state_out = (__be64 *)hash_state_out;
 202
 203         switch (ctx->qat_hash_alg) {
 204         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 205                 if (crypto_shash_export(shash, &sha1))
 206                         return -EFAULT;
 207                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 208                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 209                 break;
 210         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 211                 if (crypto_shash_export(shash, &sha256))
 212                         return -EFAULT;
 213                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 214                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 215                 break;
 216         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 217                 if (crypto_shash_export(shash, &sha512))
 218                         return -EFAULT;
 219                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 220                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 221                 break;
 222         default:
 223                 return -EFAULT;
 224         }
 225
 226         if (crypto_shash_init(shash))
 227                 return -EFAULT;
 228
 229         if (crypto_shash_update(shash, opad, block_size))
 230                 return -EFAULT;
 231
 232         offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
 233         hash_state_out = (__be32 *)(hash->sha.state1 + offset);
 234         hash512_state_out = (__be64 *)hash_state_out;
 235
 236         switch (ctx->qat_hash_alg) {
 237         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 238                 if (crypto_shash_export(shash, &sha1))
 239                         return -EFAULT;
 240                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 241                         *hash_state_out = cpu_to_be32(*(sha1.state + i));
 242                 break;
 243         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 244                 if (crypto_shash_export(shash, &sha256))
 245                         return -EFAULT;
 246                 for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
 247                         *hash_state_out = cpu_to_be32(*(sha256.state + i));
 248                 break;
 249         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 250                 if (crypto_shash_export(shash, &sha512))
 251                         return -EFAULT;
 252                 for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
 253                         *hash512_state_out = cpu_to_be64(*(sha512.state + i));
 254                 break;
 255         default:
 256                 return -EFAULT;
 257         }
 258         memzero_explicit(ipad, block_size);
 259         memzero_explicit(opad, block_size);
 260         return 0;
 261 }
 262
 263 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
 264 {
 265         header->hdr_flags =
 266                 ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
 267         header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
 268         header->comn_req_flags =
 269                 ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
 270                                             QAT_COMN_PTR_TYPE_SGL);
 271         ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
 272                                   ICP_QAT_FW_LA_PARTIAL_NONE);
 273         ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
 274                                            ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
 275         ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
 276                                 ICP_QAT_FW_LA_NO_PROTO);
 277         ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
 278                                        ICP_QAT_FW_LA_NO_UPDATE_STATE);
 279 }
 280
 281 static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx,
 282                                          int alg,
 283                                          struct crypto_authenc_keys *keys)
 284 {
 285         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
 286         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
 287         struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
 288         struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
 289         struct icp_qat_hw_auth_algo_blk *hash =
 290                 (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
 291                 sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
 292         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
 293         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 294         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 295         void *ptr = &req_tmpl->cd_ctrl;
 296         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 297         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 298
 299         /* CD setup */
 300         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
 301         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 302         hash->sha.inner_setup.auth_config.config =
 303                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 304                                              ctx->qat_hash_alg, digestsize);
 305         hash->sha.inner_setup.auth_counter.counter =
 306                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 307
 308         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 309                 return -EFAULT;
 310
 311         /* Request setup */
 312         qat_alg_init_common_hdr(header);
 313         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 314         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 315                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 316         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 317                                    ICP_QAT_FW_LA_RET_AUTH_RES);
 318         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 319                                    ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
 320         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 321         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 322
 323         /* Cipher CD config setup */
 324         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 325         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 326         cipher_cd_ctrl->cipher_cfg_offset = 0;
 327         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 328         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 329         /* Auth CD config setup */
 330         hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
 331         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 332         hash_cd_ctrl->inner_res_sz = digestsize;
 333         hash_cd_ctrl->final_sz = digestsize;
 334
 335         switch (ctx->qat_hash_alg) {
 336         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 337                 hash_cd_ctrl->inner_state1_sz =
 338                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 339                 hash_cd_ctrl->inner_state2_sz =
 340                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 341                 break;
 342         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 343                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 344                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 345                 break;
 346         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 347                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 348                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 349                 break;
 350         default:
 351                 break;
 352         }
 353         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 354                         ((sizeof(struct icp_qat_hw_auth_setup) +
 355                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 356         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 357         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 358         return 0;
 359 }
 360
 361 static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx,
 362                                          int alg,
 363                                          struct crypto_authenc_keys *keys)
 364 {
 365         struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm);
 366         unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize;
 367         struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
 368         struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
 369         struct icp_qat_hw_cipher_algo_blk *cipher =
 370                 (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
 371                 sizeof(struct icp_qat_hw_auth_setup) +
 372                 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
 373         struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
 374         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
 375         struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
 376         void *ptr = &req_tmpl->cd_ctrl;
 377         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
 378         struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
 379         struct icp_qat_fw_la_auth_req_params *auth_param =
 380                 (struct icp_qat_fw_la_auth_req_params *)
 381                 ((char *)&req_tmpl->serv_specif_rqpars +
 382                 sizeof(struct icp_qat_fw_la_cipher_req_params));
 383
 384         /* CD setup */
 385         cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
 386         memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
 387         hash->sha.inner_setup.auth_config.config =
 388                 ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
 389                                              ctx->qat_hash_alg,
 390                                              digestsize);
 391         hash->sha.inner_setup.auth_counter.counter =
 392                 cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
 393
 394         if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
 395                 return -EFAULT;
 396
 397         /* Request setup */
 398         qat_alg_init_common_hdr(header);
 399         header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 400         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 401                                            ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 402         ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
 403                                    ICP_QAT_FW_LA_NO_RET_AUTH_RES);
 404         ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
 405                                    ICP_QAT_FW_LA_CMP_AUTH_RES);
 406         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 407         cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
 408
 409         /* Cipher CD config setup */
 410         cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
 411         cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 412         cipher_cd_ctrl->cipher_cfg_offset =
 413                 (sizeof(struct icp_qat_hw_auth_setup) +
 414                  roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
 415         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 416         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 417
 418         /* Auth CD config setup */
 419         hash_cd_ctrl->hash_cfg_offset = 0;
 420         hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
 421         hash_cd_ctrl->inner_res_sz = digestsize;
 422         hash_cd_ctrl->final_sz = digestsize;
 423
 424         switch (ctx->qat_hash_alg) {
 425         case ICP_QAT_HW_AUTH_ALGO_SHA1:
 426                 hash_cd_ctrl->inner_state1_sz =
 427                         round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
 428                 hash_cd_ctrl->inner_state2_sz =
 429                         round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
 430                 break;
 431         case ICP_QAT_HW_AUTH_ALGO_SHA256:
 432                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
 433                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
 434                 break;
 435         case ICP_QAT_HW_AUTH_ALGO_SHA512:
 436                 hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
 437                 hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
 438                 break;
 439         default:
 440                 break;
 441         }
 442
 443         hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
 444                         ((sizeof(struct icp_qat_hw_auth_setup) +
 445                          round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
 446         auth_param->auth_res_sz = digestsize;
 447         ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
 448         ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 449         return 0;
 450 }
 451
 452 static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx,
 453                                         struct icp_qat_fw_la_bulk_req *req,
 454                                         struct icp_qat_hw_cipher_algo_blk *cd,
 455                                         const uint8_t *key, unsigned int keylen)
 456 {
 457         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 458         struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
 459         struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
 460
 461         memcpy(cd->aes.key, key, keylen);
 462         qat_alg_init_common_hdr(header);
 463         header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
 464         cd_pars->u.s.content_desc_params_sz =
 465                                 sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
 466         /* Cipher CD config setup */
 467         cd_ctrl->cipher_key_sz = keylen >> 3;
 468         cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
 469         cd_ctrl->cipher_cfg_offset = 0;
 470         ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 471         ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 472 }
 473
 474 static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx,
 475                                         int alg, const uint8_t *key,
 476                                         unsigned int keylen)
 477 {
 478         struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
 479         struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
 480         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 481
 482         qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen);
 483         cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
 484         enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg);
 485 }
 486
 487 static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx,
 488                                         int alg, const uint8_t *key,
 489                                         unsigned int keylen)
 490 {
 491         struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
 492         struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
 493         struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
 494
 495         qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen);
 496         cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
 497         dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg);
 498 }
 499
 500 static int qat_alg_validate_key(int key_len, int *alg)
 501 {
 502         switch (key_len) {
 503         case AES_KEYSIZE_128:
 504                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
 505                 break;
 506         case AES_KEYSIZE_192:
 507                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
 508                 break;
 509         case AES_KEYSIZE_256:
 510                 *alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
 511                 break;
 512         default:
 513                 return -EINVAL;
 514         }
 515         return 0;
 516 }
 517
 518 static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx,
 519                                       const uint8_t *key, unsigned int keylen)
 520 {
 521         struct crypto_authenc_keys keys;
 522         int alg;
 523
 524         if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE))
 525                 return -EFAULT;
 526
 527         if (crypto_authenc_extractkeys(&keys, key, keylen))
 528                 goto bad_key;
 529
 530         if (qat_alg_validate_key(keys.enckeylen, &alg))
 531                 goto bad_key;
 532
 533         if (qat_alg_aead_init_enc_session(ctx, alg, &keys))
 534                 goto error;
 535
 536         if (qat_alg_aead_init_dec_session(ctx, alg, &keys))
 537                 goto error;
 538
 539         return 0;
 540 bad_key:
 541         crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 542         return -EINVAL;
 543 error:
 544         return -EFAULT;
 545 }
 546
 547 static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx,
 548                                             const uint8_t *key,
 549                                             unsigned int keylen)
 550 {
 551         int alg;
 552
 553         if (qat_alg_validate_key(keylen, &alg))
 554                 goto bad_key;
 555
 556         qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen);
 557         qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen);
 558         return 0;
 559 bad_key:
 560         crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
 561         return -EINVAL;
 562 }
 563
 564 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key,
 565                                unsigned int keylen)
 566 {
 567         struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
 568         struct device *dev;
 569
 570         spin_lock(&ctx->lock);
 571         if (ctx->enc_cd) {
 572                 /* rekeying */
 573                 dev = &GET_DEV(ctx->inst->accel_dev);
 574                 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 575                 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 576                 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 577                 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 578         } else {
 579                 /* new key */
 580                 int node = get_current_node();
 581                 struct qat_crypto_instance *inst =
 582                                 qat_crypto_get_instance_node(node);
 583                 if (!inst) {
 584                         spin_unlock(&ctx->lock);
 585                         return -EINVAL;
 586                 }
 587
 588                 dev = &GET_DEV(inst->accel_dev);
 589                 ctx->inst = inst;
 590                 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
 591                                                   &ctx->enc_cd_paddr,
 592                                                   GFP_ATOMIC);
 593                 if (!ctx->enc_cd) {
 594                         spin_unlock(&ctx->lock);
 595                         return -ENOMEM;
 596                 }
 597                 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
 598                                                   &ctx->dec_cd_paddr,
 599                                                   GFP_ATOMIC);
 600                 if (!ctx->dec_cd) {
 601                         spin_unlock(&ctx->lock);
 602                         goto out_free_enc;
 603                 }
 604         }
 605         spin_unlock(&ctx->lock);
 606         if (qat_alg_aead_init_sessions(ctx, key, keylen))
 607                 goto out_free_all;
 608
 609         return 0;
 610
 611 out_free_all:
 612         memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
 613         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 614                           ctx->dec_cd, ctx->dec_cd_paddr);
 615         ctx->dec_cd = NULL;
 616 out_free_enc:
 617         memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
 618         dma_free_coherent(dev, sizeof(struct qat_alg_cd),
 619                           ctx->enc_cd, ctx->enc_cd_paddr);
 620         ctx->enc_cd = NULL;
 621         return -ENOMEM;
 622 }
 623
 624 static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
 625                               struct qat_crypto_request *qat_req)
 626 {
 627         struct device *dev = &GET_DEV(inst->accel_dev);
 628         struct qat_alg_buf_list *bl = qat_req->buf.bl;
 629         struct qat_alg_buf_list *blout = qat_req->buf.blout;
 630         dma_addr_t blp = qat_req->buf.blp;
 631         dma_addr_t blpout = qat_req->buf.bloutp;
 632         size_t sz = qat_req->buf.sz;
 633         size_t sz_out = qat_req->buf.sz_out;
 634         int i;
 635
 636         for (i = 0; i < bl->num_bufs; i++)
 637                 dma_unmap_single(dev, bl->bufers[i].addr,
 638                                  bl->bufers[i].len, DMA_BIDIRECTIONAL);
 639
 640         dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 641         kfree(bl);
 642         if (blp != blpout) {
 643                 /* If out of place operation dma unmap only data */
 644                 int bufless = blout->num_bufs - blout->num_mapped_bufs;
 645
 646                 for (i = bufless; i < blout->num_bufs; i++) {
 647                         dma_unmap_single(dev, blout->bufers[i].addr,
 648                                          blout->bufers[i].len,
 649                                          DMA_BIDIRECTIONAL);
 650                 }
 651                 dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE);
 652                 kfree(blout);
 653         }
 654 }
 655
 656 static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
 657                                struct scatterlist *assoc,
 658                                struct scatterlist *sgl,
 659                                struct scatterlist *sglout, uint8_t *iv,
 660                                uint8_t ivlen,
 661                                struct qat_crypto_request *qat_req)
 662 {
 663         struct device *dev = &GET_DEV(inst->accel_dev);
 664         int i, bufs = 0, sg_nctr = 0;
 665         int n = sg_nents(sgl), assoc_n = sg_nents(assoc);
 666         struct qat_alg_buf_list *bufl;
 667         struct qat_alg_buf_list *buflout = NULL;
 668         dma_addr_t blp;
 669         dma_addr_t bloutp = 0;
 670         struct scatterlist *sg;
 671         size_t sz_out, sz = sizeof(struct qat_alg_buf_list) +
 672                         ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
 673
 674         if (unlikely(!n))
 675                 return -EINVAL;
 676
 677         bufl = kzalloc_node(sz, GFP_ATOMIC,
 678                             dev_to_node(&GET_DEV(inst->accel_dev)));
 679         if (unlikely(!bufl))
 680                 return -ENOMEM;
 681
 682         blp = dma_map_single(dev, bufl, sz, DMA_TO_DEVICE);
 683         if (unlikely(dma_mapping_error(dev, blp)))
 684                 goto err;
 685
 686         for_each_sg(assoc, sg, assoc_n, i) {
 687                 if (!sg->length)
 688                         continue;
 689                 bufl->bufers[bufs].addr = dma_map_single(dev,
 690                                                          sg_virt(sg),
 691                                                          sg->length,
 692                                                          DMA_BIDIRECTIONAL);
 693                 bufl->bufers[bufs].len = sg->length;
 694                 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
 695                         goto err;
 696                 bufs++;
 697         }
 698         if (ivlen) {
 699                 bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen,
 700                                                          DMA_BIDIRECTIONAL);
 701                 bufl->bufers[bufs].len = ivlen;
 702                 if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr)))
 703                         goto err;
 704                 bufs++;
 705         }
 706
 707         for_each_sg(sgl, sg, n, i) {
 708                 int y = sg_nctr + bufs;
 709
 710                 if (!sg->length)
 711                         continue;
 712
 713                 bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 714                                                       sg->length,
 715                                                       DMA_BIDIRECTIONAL);
 716                 bufl->bufers[y].len = sg->length;
 717                 if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr)))
 718                         goto err;
 719                 sg_nctr++;
 720         }
 721         bufl->num_bufs = sg_nctr + bufs;
 722         qat_req->buf.bl = bufl;
 723         qat_req->buf.blp = blp;
 724         qat_req->buf.sz = sz;
 725         /* Handle out of place operation */
 726         if (sgl != sglout) {
 727                 struct qat_alg_buf *bufers;
 728
 729                 n = sg_nents(sglout);
 730                 sz_out = sizeof(struct qat_alg_buf_list) +
 731                         ((1 + n + assoc_n) * sizeof(struct qat_alg_buf));
 732                 sg_nctr = 0;
 733                 buflout = kzalloc_node(sz_out, GFP_ATOMIC,
 734                                        dev_to_node(&GET_DEV(inst->accel_dev)));
 735                 if (unlikely(!buflout))
 736                         goto err;
 737                 bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE);
 738                 if (unlikely(dma_mapping_error(dev, bloutp)))
 739                         goto err;
 740                 bufers = buflout->bufers;
 741                 /* For out of place operation dma map only data and
 742                  * reuse assoc mapping and iv */
 743                 for (i = 0; i < bufs; i++) {
 744                         bufers[i].len = bufl->bufers[i].len;
 745                         bufers[i].addr = bufl->bufers[i].addr;
 746                 }
 747                 for_each_sg(sglout, sg, n, i) {
 748                         int y = sg_nctr + bufs;
 749
 750                         if (!sg->length)
 751                                 continue;
 752
 753                         bufers[y].addr = dma_map_single(dev, sg_virt(sg),
 754                                                         sg->length,
 755                                                         DMA_BIDIRECTIONAL);
 756                         if (unlikely(dma_mapping_error(dev, bufers[y].addr)))
 757                                 goto err;
 758                         bufers[y].len = sg->length;
 759                         sg_nctr++;
 760                 }
 761                 buflout->num_bufs = sg_nctr + bufs;
 762                 buflout->num_mapped_bufs = sg_nctr;
 763                 qat_req->buf.blout = buflout;
 764                 qat_req->buf.bloutp = bloutp;
 765                 qat_req->buf.sz_out = sz_out;
 766         } else {
 767                 /* Otherwise set the src and dst to the same address */
 768                 qat_req->buf.bloutp = qat_req->buf.blp;
 769                 qat_req->buf.sz_out = 0;
 770         }
 771         return 0;
 772 err:
 773         dev_err(dev, "Failed to map buf for dma\n");
 774         sg_nctr = 0;
 775         for (i = 0; i < n + bufs; i++)
 776                 if (!dma_mapping_error(dev, bufl->bufers[i].addr))
 777                         dma_unmap_single(dev, bufl->bufers[i].addr,
 778                                          bufl->bufers[i].len,
 779                                          DMA_BIDIRECTIONAL);
 780
 781         if (!dma_mapping_error(dev, blp))
 782                 dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE);
 783         kfree(bufl);
 784         if (sgl != sglout && buflout) {
 785                 n = sg_nents(sglout);
 786                 for (i = bufs; i < n + bufs; i++)
 787                         if (!dma_mapping_error(dev, buflout->bufers[i].addr))
 788                                 dma_unmap_single(dev, buflout->bufers[i].addr,
 789                                                  buflout->bufers[i].len,
 790                                                  DMA_BIDIRECTIONAL);
 791                 if (!dma_mapping_error(dev, bloutp))
 792                         dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE);
 793                 kfree(buflout);
 794         }
 795         return -ENOMEM;
 796 }
 797
 798 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 799                                   struct qat_crypto_request *qat_req)
 800 {
 801         struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
 802         struct qat_crypto_instance *inst = ctx->inst;
 803         struct aead_request *areq = qat_req->aead_req;
 804         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
 805         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 806
 807         qat_alg_free_bufl(inst, qat_req);
 808         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 809                 res = -EBADMSG;
 810         areq->base.complete(&areq->base, res);
 811 }
 812
 813 static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
 814                                         struct qat_crypto_request *qat_req)
 815 {
 816         struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx;
 817         struct qat_crypto_instance *inst = ctx->inst;
 818         struct ablkcipher_request *areq = qat_req->ablkcipher_req;
 819         uint8_t stat_filed = qat_resp->comn_resp.comn_status;
 820         int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
 821
 822         qat_alg_free_bufl(inst, qat_req);
 823         if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
 824                 res = -EINVAL;
 825         areq->base.complete(&areq->base, res);
 826 }
 827
 828 void qat_alg_callback(void *resp)
 829 {
 830         struct icp_qat_fw_la_resp *qat_resp = resp;
 831         struct qat_crypto_request *qat_req =
 832                                 (void *)(__force long)qat_resp->opaque_data;
 833
 834         qat_req->cb(qat_resp, qat_req);
 835 }
 836
 837 static int qat_alg_aead_dec(struct aead_request *areq)
 838 {
 839         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 840         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 841         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 842         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 843         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 844         struct icp_qat_fw_la_auth_req_params *auth_param;
 845         struct icp_qat_fw_la_bulk_req *msg;
 846         int digst_size = crypto_aead_crt(aead_tfm)->authsize;
 847         int ret, ctr = 0;
 848
 849         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,
 850                                   areq->iv, AES_BLOCK_SIZE, qat_req);
 851         if (unlikely(ret))
 852                 return ret;
 853
 854         msg = &qat_req->req;
 855         *msg = ctx->dec_fw_req;
 856         qat_req->aead_ctx = ctx;
 857         qat_req->aead_req = areq;
 858         qat_req->cb = qat_aead_alg_callback;
 859         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 860         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 861         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 862         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 863         cipher_param->cipher_length = areq->cryptlen - digst_size;
 864         cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
 865         memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
 866         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 867         auth_param->auth_off = 0;
 868         auth_param->auth_len = areq->assoclen +
 869                                 cipher_param->cipher_length + AES_BLOCK_SIZE;
 870         do {
 871                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 872         } while (ret == -EAGAIN && ctr++ < 10);
 873
 874         if (ret == -EAGAIN) {
 875                 qat_alg_free_bufl(ctx->inst, qat_req);
 876                 return -EBUSY;
 877         }
 878         return -EINPROGRESS;
 879 }
 880
 881 static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv,
 882                                      int enc_iv)
 883 {
 884         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
 885         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 886         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 887         struct qat_crypto_request *qat_req = aead_request_ctx(areq);
 888         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 889         struct icp_qat_fw_la_auth_req_params *auth_param;
 890         struct icp_qat_fw_la_bulk_req *msg;
 891         int ret, ctr = 0;
 892
 893         ret = qat_alg_sgl_to_bufl(ctx->inst, areq->assoc, areq->src, areq->dst,
 894                                   iv, AES_BLOCK_SIZE, qat_req);
 895         if (unlikely(ret))
 896                 return ret;
 897
 898         msg = &qat_req->req;
 899         *msg = ctx->enc_fw_req;
 900         qat_req->aead_ctx = ctx;
 901         qat_req->aead_req = areq;
 902         qat_req->cb = qat_aead_alg_callback;
 903         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
 904         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
 905         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
 906         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
 907         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 908
 909         if (enc_iv) {
 910                 cipher_param->cipher_length = areq->cryptlen + AES_BLOCK_SIZE;
 911                 cipher_param->cipher_offset = areq->assoclen;
 912         } else {
 913                 memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
 914                 cipher_param->cipher_length = areq->cryptlen;
 915                 cipher_param->cipher_offset = areq->assoclen + AES_BLOCK_SIZE;
 916         }
 917         auth_param->auth_off = 0;
 918         auth_param->auth_len = areq->assoclen + areq->cryptlen + AES_BLOCK_SIZE;
 919
 920         do {
 921                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
 922         } while (ret == -EAGAIN && ctr++ < 10);
 923
 924         if (ret == -EAGAIN) {
 925                 qat_alg_free_bufl(ctx->inst, qat_req);
 926                 return -EBUSY;
 927         }
 928         return -EINPROGRESS;
 929 }
 930
 931 static int qat_alg_aead_enc(struct aead_request *areq)
 932 {
 933         return qat_alg_aead_enc_internal(areq, areq->iv, 0);
 934 }
 935
 936 static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req)
 937 {
 938         struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq);
 939         struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
 940         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
 941         __be64 seq;
 942
 943         memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE);
 944         seq = cpu_to_be64(req->seq);
 945         memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t),
 946                &seq, sizeof(uint64_t));
 947         return qat_alg_aead_enc_internal(&req->areq, req->giv, 1);
 948 }
 949
 950 static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm,
 951                                      const uint8_t *key,
 952                                      unsigned int keylen)
 953 {
 954         struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm);
 955         struct device *dev;
 956
 957         spin_lock(&ctx->lock);
 958         if (ctx->enc_cd) {
 959                 /* rekeying */
 960                 dev = &GET_DEV(ctx->inst->accel_dev);
 961                 memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
 962                 memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
 963                 memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
 964                 memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
 965         } else {
 966                 /* new key */
 967                 int node = get_current_node();
 968                 struct qat_crypto_instance *inst =
 969                                 qat_crypto_get_instance_node(node);
 970                 if (!inst) {
 971                         spin_unlock(&ctx->lock);
 972                         return -EINVAL;
 973                 }
 974
 975                 dev = &GET_DEV(inst->accel_dev);
 976                 ctx->inst = inst;
 977                 ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd),
 978                                                   &ctx->enc_cd_paddr,
 979                                                   GFP_ATOMIC);
 980                 if (!ctx->enc_cd) {
 981                         spin_unlock(&ctx->lock);
 982                         return -ENOMEM;
 983                 }
 984                 ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd),
 985                                                   &ctx->dec_cd_paddr,
 986                                                   GFP_ATOMIC);
 987                 if (!ctx->dec_cd) {
 988                         spin_unlock(&ctx->lock);
 989                         goto out_free_enc;
 990                 }
 991         }
 992         spin_unlock(&ctx->lock);
 993         if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen))
 994                 goto out_free_all;
 995
 996         return 0;
 997
 998 out_free_all:
 999         memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd));
1000         dma_free_coherent(dev, sizeof(*ctx->enc_cd),
1001                           ctx->dec_cd, ctx->dec_cd_paddr);
1002         ctx->dec_cd = NULL;
1003 out_free_enc:
1004         memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd));
1005         dma_free_coherent(dev, sizeof(*ctx->dec_cd),
1006                           ctx->enc_cd, ctx->enc_cd_paddr);
1007         ctx->enc_cd = NULL;
1008         return -ENOMEM;
1009 }
1010
1011 static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req)
1012 {
1013         struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1014         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1015         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1016         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1017         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1018         struct icp_qat_fw_la_bulk_req *msg;
1019         int ret, ctr = 0;
1020
1021         ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
1022                                   NULL, 0, qat_req);
1023         if (unlikely(ret))
1024                 return ret;
1025
1026         msg = &qat_req->req;
1027         *msg = ctx->enc_fw_req;
1028         qat_req->ablkcipher_ctx = ctx;
1029         qat_req->ablkcipher_req = req;
1030         qat_req->cb = qat_ablkcipher_alg_callback;
1031         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1032         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1033         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1034         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1035         cipher_param->cipher_length = req->nbytes;
1036         cipher_param->cipher_offset = 0;
1037         memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1038         do {
1039                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1040         } while (ret == -EAGAIN && ctr++ < 10);
1041
1042         if (ret == -EAGAIN) {
1043                 qat_alg_free_bufl(ctx->inst, qat_req);
1044                 return -EBUSY;
1045         }
1046         return -EINPROGRESS;
1047 }
1048
1049 static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req)
1050 {
1051         struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req);
1052         struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm);
1053         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1054         struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req);
1055         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1056         struct icp_qat_fw_la_bulk_req *msg;
1057         int ret, ctr = 0;
1058
1059         ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst,
1060                                   NULL, 0, qat_req);
1061         if (unlikely(ret))
1062                 return ret;
1063
1064         msg = &qat_req->req;
1065         *msg = ctx->dec_fw_req;
1066         qat_req->ablkcipher_ctx = ctx;
1067         qat_req->ablkcipher_req = req;
1068         qat_req->cb = qat_ablkcipher_alg_callback;
1069         qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req;
1070         qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1071         qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1072         cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1073         cipher_param->cipher_length = req->nbytes;
1074         cipher_param->cipher_offset = 0;
1075         memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE);
1076         do {
1077                 ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg);
1078         } while (ret == -EAGAIN && ctr++ < 10);
1079
1080         if (ret == -EAGAIN) {
1081                 qat_alg_free_bufl(ctx->inst, qat_req);
1082                 return -EBUSY;
1083         }
1084         return -EINPROGRESS;
1085 }
1086
1087 static int qat_alg_aead_init(struct crypto_tfm *tfm,
1088                              enum icp_qat_hw_auth_algo hash,
1089                              const char *hash_name)
1090 {
1091         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1092
1093         ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1094         if (IS_ERR(ctx->hash_tfm))
1095                 return -EFAULT;
1096         spin_lock_init(&ctx->lock);
1097         ctx->qat_hash_alg = hash;
1098         tfm->crt_aead.reqsize = sizeof(struct aead_request) +
1099                                 sizeof(struct qat_crypto_request);
1100         ctx->tfm = tfm;
1101         return 0;
1102 }
1103
1104 static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm)
1105 {
1106         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1107 }
1108
1109 static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm)
1110 {
1111         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1112 }
1113
1114 static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm)
1115 {
1116         return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1117 }
1118
1119 static void qat_alg_aead_exit(struct crypto_tfm *tfm)
1120 {
1121         struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
1122         struct qat_crypto_instance *inst = ctx->inst;
1123         struct device *dev;
1124
1125         if (!IS_ERR(ctx->hash_tfm))
1126                 crypto_free_shash(ctx->hash_tfm);
1127
1128         if (!inst)
1129                 return;
1130
1131         dev = &GET_DEV(inst->accel_dev);
1132         if (ctx->enc_cd) {
1133                 memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1134                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1135                                   ctx->enc_cd, ctx->enc_cd_paddr);
1136         }
1137         if (ctx->dec_cd) {
1138                 memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1139                 dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1140                                   ctx->dec_cd, ctx->dec_cd_paddr);
1141         }
1142         qat_crypto_put_instance(inst);
1143 }
1144
1145 static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm)
1146 {
1147         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1148
1149         spin_lock_init(&ctx->lock);
1150         tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
1151                                         sizeof(struct qat_crypto_request);
1152         ctx->tfm = tfm;
1153         return 0;
1154 }
1155
1156 static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm)
1157 {
1158         struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1159         struct qat_crypto_instance *inst = ctx->inst;
1160         struct device *dev;
1161
1162         if (!inst)
1163                 return;
1164
1165         dev = &GET_DEV(inst->accel_dev);
1166         if (ctx->enc_cd) {
1167                 memset(ctx->enc_cd, 0,
1168                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1169                 dma_free_coherent(dev,
1170                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1171                                   ctx->enc_cd, ctx->enc_cd_paddr);
1172         }
1173         if (ctx->dec_cd) {
1174                 memset(ctx->dec_cd, 0,
1175                        sizeof(struct icp_qat_hw_cipher_algo_blk));
1176                 dma_free_coherent(dev,
1177                                   sizeof(struct icp_qat_hw_cipher_algo_blk),
1178                                   ctx->dec_cd, ctx->dec_cd_paddr);
1179         }
1180         qat_crypto_put_instance(inst);
1181 }
1182
1183 static struct crypto_alg qat_algs[] = { {
1184         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1185         .cra_driver_name = "qat_aes_cbc_hmac_sha1",
1186         .cra_priority = 4001,
1187         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1188         .cra_blocksize = AES_BLOCK_SIZE,
1189         .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1190         .cra_alignmask = 0,
1191         .cra_type = &crypto_aead_type,
1192         .cra_module = THIS_MODULE,
1193         .cra_init = qat_alg_aead_sha1_init,
1194         .cra_exit = qat_alg_aead_exit,
1195         .cra_u = {
1196                 .aead = {
1197                         .setkey = qat_alg_aead_setkey,
1198                         .decrypt = qat_alg_aead_dec,
1199                         .encrypt = qat_alg_aead_enc,
1200                         .givencrypt = qat_alg_aead_genivenc,
1201                         .ivsize = AES_BLOCK_SIZE,
1202                         .maxauthsize = SHA1_DIGEST_SIZE,
1203                 },
1204         },
1205 }, {
1206         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1207         .cra_driver_name = "qat_aes_cbc_hmac_sha256",
1208         .cra_priority = 4001,
1209         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1210         .cra_blocksize = AES_BLOCK_SIZE,
1211         .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1212         .cra_alignmask = 0,
1213         .cra_type = &crypto_aead_type,
1214         .cra_module = THIS_MODULE,
1215         .cra_init = qat_alg_aead_sha256_init,
1216         .cra_exit = qat_alg_aead_exit,
1217         .cra_u = {
1218                 .aead = {
1219                         .setkey = qat_alg_aead_setkey,
1220                         .decrypt = qat_alg_aead_dec,
1221                         .encrypt = qat_alg_aead_enc,
1222                         .givencrypt = qat_alg_aead_genivenc,
1223                         .ivsize = AES_BLOCK_SIZE,
1224                         .maxauthsize = SHA256_DIGEST_SIZE,
1225                 },
1226         },
1227 }, {
1228         .cra_name = "authenc(hmac(sha512),cbc(aes))",
1229         .cra_driver_name = "qat_aes_cbc_hmac_sha512",
1230         .cra_priority = 4001,
1231         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1232         .cra_blocksize = AES_BLOCK_SIZE,
1233         .cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1234         .cra_alignmask = 0,
1235         .cra_type = &crypto_aead_type,
1236         .cra_module = THIS_MODULE,
1237         .cra_init = qat_alg_aead_sha512_init,
1238         .cra_exit = qat_alg_aead_exit,
1239         .cra_u = {
1240                 .aead = {
1241                         .setkey = qat_alg_aead_setkey,
1242                         .decrypt = qat_alg_aead_dec,
1243                         .encrypt = qat_alg_aead_enc,
1244                         .givencrypt = qat_alg_aead_genivenc,
1245                         .ivsize = AES_BLOCK_SIZE,
1246                         .maxauthsize = SHA512_DIGEST_SIZE,
1247                 },
1248         },
1249 }, {
1250         .cra_name = "cbc(aes)",
1251         .cra_driver_name = "qat_aes_cbc",
1252         .cra_priority = 4001,
1253         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1254         .cra_blocksize = AES_BLOCK_SIZE,
1255         .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx),
1256         .cra_alignmask = 0,
1257         .cra_type = &crypto_ablkcipher_type,
1258         .cra_module = THIS_MODULE,
1259         .cra_init = qat_alg_ablkcipher_init,
1260         .cra_exit = qat_alg_ablkcipher_exit,
1261         .cra_u = {
1262                 .ablkcipher = {
1263                         .setkey = qat_alg_ablkcipher_setkey,
1264                         .decrypt = qat_alg_ablkcipher_decrypt,
1265                         .encrypt = qat_alg_ablkcipher_encrypt,
1266                         .min_keysize = AES_MIN_KEY_SIZE,
1267                         .max_keysize = AES_MAX_KEY_SIZE,
1268                         .ivsize = AES_BLOCK_SIZE,
1269                 },
1270         },
1271 } };
1272
1273 int qat_algs_register(void)
1274 {
1275         int ret = 0;
1276
1277         mutex_lock(&algs_lock);
1278         if (++active_devs == 1) {
1279                 int i;
1280
1281                 for (i = 0; i < ARRAY_SIZE(qat_algs); i++)
1282                         qat_algs[i].cra_flags =
1283                                 (qat_algs[i].cra_type == &crypto_aead_type) ?
1284                                 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC :
1285                                 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
1286
1287                 ret = crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs));
1288         }
1289         mutex_unlock(&algs_lock);
1290         return ret;
1291 }
1292
1293 int qat_algs_unregister(void)
1294 {
1295         int ret = 0;
1296
1297         mutex_lock(&algs_lock);
1298         if (--active_devs == 0)
1299                 ret = crypto_unregister_algs(qat_algs, ARRAY_SIZE(qat_algs));
1300         mutex_unlock(&algs_lock);
1301         return ret;
1302 }
1303
1304 int qat_algs_init(void)
1305 {
1306         crypto_get_default_rng();
1307         return 0;
1308 }
1309
1310 void qat_algs_exit(void)
1311 {
1312         crypto_put_default_rng();
1313 }