kernel/drivers/crypto/qat/qat_common/qat_hal.c

   1 /*
   2   This file is provided under a dual BSD/GPLv2 license.  When using or
   3   redistributing this file, you may do so under either license.
   4
   5   GPL LICENSE SUMMARY
   6   Copyright(c) 2014 Intel Corporation.
   7   This program is free software; you can redistribute it and/or modify
   8   it under the terms of version 2 of the GNU General Public License as
   9   published by the Free Software Foundation.
  10
  11   This program is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14   General Public License for more details.
  15
  16   Contact Information:
  17   qat-linux@intel.com
  18
  19   BSD LICENSE
  20   Copyright(c) 2014 Intel Corporation.
  21   Redistribution and use in source and binary forms, with or without
  22   modification, are permitted provided that the following conditions
  23   are met:
  24
  25     * Redistributions of source code must retain the above copyright
  26       notice, this list of conditions and the following disclaimer.
  27     * Redistributions in binary form must reproduce the above copyright
  28       notice, this list of conditions and the following disclaimer in
  29       the documentation and/or other materials provided with the
  30       distribution.
  31     * Neither the name of Intel Corporation nor the names of its
  32       contributors may be used to endorse or promote products derived
  33       from this software without specific prior written permission.
  34
  35   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46 */
  47 #include <linux/slab.h>
  48
  49 #include "adf_accel_devices.h"
  50 #include "adf_common_drv.h"
  51 #include "icp_qat_hal.h"
  52 #include "icp_qat_uclo.h"
  53
  54 #define BAD_REGADDR               0xffff
  55 #define MAX_RETRY_TIMES           10000
  56 #define INIT_CTX_ARB_VALUE        0x0
  57 #define INIT_CTX_ENABLE_VALUE     0x0
  58 #define INIT_PC_VALUE             0x0
  59 #define INIT_WAKEUP_EVENTS_VALUE  0x1
  60 #define INIT_SIG_EVENTS_VALUE     0x1
  61 #define INIT_CCENABLE_VALUE       0x2000
  62 #define RST_CSR_QAT_LSB           20
  63 #define RST_CSR_AE_LSB            0
  64 #define MC_TIMESTAMP_ENABLE       (0x1 << 7)
  65
  66 #define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
  67         (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
  68         (~(1 << CE_REG_PAR_ERR_BITPOS)))
  69 #define INSERT_IMMED_GPRA_CONST(inst, const_val) \
  70         (inst = ((inst & 0xFFFF00C03FFull) | \
  71                 ((((const_val) << 12) & 0x0FF00000ull) | \
  72                 (((const_val) << 10) & 0x0003FC00ull))))
  73 #define INSERT_IMMED_GPRB_CONST(inst, const_val) \
  74         (inst = ((inst & 0xFFFF00FFF00ull) | \
  75                 ((((const_val) << 12) & 0x0FF00000ull) | \
  76                 (((const_val) <<  0) & 0x000000FFull))))
  77
  78 #define AE(handle, ae) handle->hal_handle->aes[ae]
  79
  80 static const uint64_t inst_4b[] = {
  81         0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
  82         0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  83         0x0A021000000ull
  84 };
  85
  86 static const uint64_t inst[] = {
  87         0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
  88         0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  89         0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
  90         0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
  91         0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
  92         0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
  93         0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
  94         0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
  95         0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
  96         0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
  97         0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
  98         0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
  99         0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
 100         0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
 101         0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
 102         0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
 103         0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
 104         0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
 105         0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
 106         0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
 107         0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
 108         0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
 109 };
 110
 111 void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
 112                           unsigned char ae, unsigned int ctx_mask)
 113 {
 114         AE(handle, ae).live_ctx_mask = ctx_mask;
 115 }
 116
 117 #define CSR_RETRY_TIMES 500
 118 static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
 119                              unsigned char ae, unsigned int csr,
 120                              unsigned int *value)
 121 {
 122         unsigned int iterations = CSR_RETRY_TIMES;
 123
 124         do {
 125                 *value = GET_AE_CSR(handle, ae, csr);
 126                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 127                         return 0;
 128         } while (iterations--);
 129
 130         pr_err("QAT: Read CSR timeout\n");
 131         return -EFAULT;
 132 }
 133
 134 static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
 135                              unsigned char ae, unsigned int csr,
 136                              unsigned int value)
 137 {
 138         unsigned int iterations = CSR_RETRY_TIMES;
 139
 140         do {
 141                 SET_AE_CSR(handle, ae, csr, value);
 142                 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
 143                         return 0;
 144         } while (iterations--);
 145
 146         pr_err("QAT: Write CSR Timeout\n");
 147         return -EFAULT;
 148 }
 149
 150 static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 151                                      unsigned char ae, unsigned char ctx,
 152                                      unsigned int *events)
 153 {
 154         unsigned int cur_ctx;
 155
 156         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 157         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 158         qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
 159         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 160 }
 161
 162 static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
 163                                unsigned char ae, unsigned int cycles,
 164                                int chk_inactive)
 165 {
 166         unsigned int base_cnt = 0, cur_cnt = 0;
 167         unsigned int csr = (1 << ACS_ABO_BITPOS);
 168         int times = MAX_RETRY_TIMES;
 169         int elapsed_cycles = 0;
 170
 171         qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
 172         base_cnt &= 0xffff;
 173         while ((int)cycles > elapsed_cycles && times--) {
 174                 if (chk_inactive)
 175                         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
 176
 177                 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
 178                 cur_cnt &= 0xffff;
 179                 elapsed_cycles = cur_cnt - base_cnt;
 180
 181                 if (elapsed_cycles < 0)
 182                         elapsed_cycles += 0x10000;
 183
 184                 /* ensure at least 8 time cycles elapsed in wait_cycles */
 185                 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
 186                         return 0;
 187         }
 188         if (!times) {
 189                 pr_err("QAT: wait_num_cycles time out\n");
 190                 return -EFAULT;
 191         }
 192         return 0;
 193 }
 194
 195 #define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
 196 #define SET_BIT(wrd, bit) (wrd | 1 << bit)
 197
 198 int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
 199                             unsigned char ae, unsigned char mode)
 200 {
 201         unsigned int csr, new_csr;
 202
 203         if ((mode != 4) && (mode != 8)) {
 204                 pr_err("QAT: bad ctx mode=%d\n", mode);
 205                 return -EINVAL;
 206         }
 207
 208         /* Sets the accelaration engine context mode to either four or eight */
 209         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 210         csr = IGNORE_W1C_MASK & csr;
 211         new_csr = (mode == 4) ?
 212                 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
 213                 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
 214         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 215         return 0;
 216 }
 217
 218 int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
 219                            unsigned char ae, unsigned char mode)
 220 {
 221         unsigned int csr, new_csr;
 222
 223         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 224         csr &= IGNORE_W1C_MASK;
 225
 226         new_csr = (mode) ?
 227                 SET_BIT(csr, CE_NN_MODE_BITPOS) :
 228                 CLR_BIT(csr, CE_NN_MODE_BITPOS);
 229
 230         if (new_csr != csr)
 231                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 232
 233         return 0;
 234 }
 235
 236 int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
 237                            unsigned char ae, enum icp_qat_uof_regtype lm_type,
 238                            unsigned char mode)
 239 {
 240         unsigned int csr, new_csr;
 241
 242         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
 243         csr &= IGNORE_W1C_MASK;
 244         switch (lm_type) {
 245         case ICP_LMEM0:
 246                 new_csr = (mode) ?
 247                         SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
 248                         CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
 249                 break;
 250         case ICP_LMEM1:
 251                 new_csr = (mode) ?
 252                         SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
 253                         CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
 254                 break;
 255         default:
 256                 pr_err("QAT: lmType = 0x%x\n", lm_type);
 257                 return -EINVAL;
 258         }
 259
 260         if (new_csr != csr)
 261                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
 262         return 0;
 263 }
 264
 265 static unsigned short qat_hal_get_reg_addr(unsigned int type,
 266                                            unsigned short reg_num)
 267 {
 268         unsigned short reg_addr;
 269
 270         switch (type) {
 271         case ICP_GPA_ABS:
 272         case ICP_GPB_ABS:
 273                 reg_addr = 0x80 | (reg_num & 0x7f);
 274                 break;
 275         case ICP_GPA_REL:
 276         case ICP_GPB_REL:
 277                 reg_addr = reg_num & 0x1f;
 278                 break;
 279         case ICP_SR_RD_REL:
 280         case ICP_SR_WR_REL:
 281         case ICP_SR_REL:
 282                 reg_addr = 0x180 | (reg_num & 0x1f);
 283                 break;
 284         case ICP_SR_ABS:
 285                 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
 286                 break;
 287         case ICP_DR_RD_REL:
 288         case ICP_DR_WR_REL:
 289         case ICP_DR_REL:
 290                 reg_addr = 0x1c0 | (reg_num & 0x1f);
 291                 break;
 292         case ICP_DR_ABS:
 293                 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
 294                 break;
 295         case ICP_NEIGH_REL:
 296                 reg_addr = 0x280 | (reg_num & 0x1f);
 297                 break;
 298         case ICP_LMEM0:
 299                 reg_addr = 0x200;
 300                 break;
 301         case ICP_LMEM1:
 302                 reg_addr = 0x220;
 303                 break;
 304         case ICP_NO_DEST:
 305                 reg_addr = 0x300 | (reg_num & 0xff);
 306                 break;
 307         default:
 308                 reg_addr = BAD_REGADDR;
 309                 break;
 310         }
 311         return reg_addr;
 312 }
 313
 314 void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
 315 {
 316         unsigned int ae_reset_csr;
 317
 318         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 319         ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
 320         ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
 321         SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 322 }
 323
 324 static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
 325                                 unsigned char ae, unsigned int ctx_mask,
 326                                 unsigned int ae_csr, unsigned int csr_val)
 327 {
 328         unsigned int ctx, cur_ctx;
 329
 330         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 331
 332         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 333                 if (!(ctx_mask & (1 << ctx)))
 334                         continue;
 335                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 336                 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
 337         }
 338
 339         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 340 }
 341
 342 static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
 343                                 unsigned char ae, unsigned char ctx,
 344                                 unsigned int ae_csr, unsigned int *csr_val)
 345 {
 346         unsigned int cur_ctx;
 347
 348         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 349         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 350         qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
 351         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 352 }
 353
 354 static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
 355                                   unsigned char ae, unsigned int ctx_mask,
 356                                   unsigned int events)
 357 {
 358         unsigned int ctx, cur_ctx;
 359
 360         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 361         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 362                 if (!(ctx_mask & (1 << ctx)))
 363                         continue;
 364                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 365                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
 366         }
 367         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 368 }
 369
 370 static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
 371                                      unsigned char ae, unsigned int ctx_mask,
 372                                      unsigned int events)
 373 {
 374         unsigned int ctx, cur_ctx;
 375
 376         qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
 377         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
 378                 if (!(ctx_mask & (1 << ctx)))
 379                         continue;
 380                 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
 381                 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
 382                                   events);
 383         }
 384         qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
 385 }
 386
 387 static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
 388 {
 389         unsigned int base_cnt, cur_cnt;
 390         unsigned char ae;
 391         unsigned int times = MAX_RETRY_TIMES;
 392
 393         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 394                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 395                         continue;
 396
 397                 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
 398                                   (unsigned int *)&base_cnt);
 399                 base_cnt &= 0xffff;
 400
 401                 do {
 402                         qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
 403                                           (unsigned int *)&cur_cnt);
 404                         cur_cnt &= 0xffff;
 405                 } while (times-- && (cur_cnt == base_cnt));
 406
 407                 if (!times) {
 408                         pr_err("QAT: AE%d is inactive!!\n", ae);
 409                         return -EFAULT;
 410                 }
 411         }
 412
 413         return 0;
 414 }
 415
 416 static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
 417 {
 418         unsigned int misc_ctl;
 419         unsigned char ae;
 420
 421         /* stop the timestamp timers */
 422         misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
 423         if (misc_ctl & MC_TIMESTAMP_ENABLE)
 424                 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
 425                             (~MC_TIMESTAMP_ENABLE));
 426
 427         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 428                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 429                         continue;
 430                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
 431                 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
 432         }
 433         /* start timestamp timers */
 434         SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
 435 }
 436
 437 #define ESRAM_AUTO_TINIT        BIT(2)
 438 #define ESRAM_AUTO_TINIT_DONE   BIT(3)
 439 #define ESRAM_AUTO_INIT_USED_CYCLES (1640)
 440 #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
 441 static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
 442 {
 443         void __iomem *csr_addr = handle->hal_ep_csr_addr_v +
 444                                  ESRAM_AUTO_INIT_CSR_OFFSET;
 445         unsigned int csr_val, times = 30;
 446
 447         csr_val = ADF_CSR_RD(csr_addr, 0);
 448         if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
 449                 return 0;
 450
 451         csr_val = ADF_CSR_RD(csr_addr, 0);
 452         csr_val |= ESRAM_AUTO_TINIT;
 453         ADF_CSR_WR(csr_addr, 0, csr_val);
 454
 455         do {
 456                 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
 457                 csr_val = ADF_CSR_RD(csr_addr, 0);
 458         } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
 459         if ((!times)) {
 460                 pr_err("QAT: Fail to init eSram!\n");
 461                 return -EFAULT;
 462         }
 463         return 0;
 464 }
 465
 466 #define SHRAM_INIT_CYCLES 2060
 467 int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
 468 {
 469         unsigned int ae_reset_csr;
 470         unsigned char ae;
 471         unsigned int clk_csr;
 472         unsigned int times = 100;
 473         unsigned int csr;
 474
 475         /* write to the reset csr */
 476         ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
 477         ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
 478         ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
 479         do {
 480                 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
 481                 if (!(times--))
 482                         goto out_err;
 483                 csr = GET_GLB_CSR(handle, ICP_RESET);
 484         } while ((handle->hal_handle->ae_mask |
 485                  (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
 486         /* enable clock */
 487         clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
 488         clk_csr |= handle->hal_handle->ae_mask << 0;
 489         clk_csr |= handle->hal_handle->slice_mask << 20;
 490         SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
 491         if (qat_hal_check_ae_alive(handle))
 492                 goto out_err;
 493
 494         /* Set undefined power-up/reset states to reasonable default values */
 495         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 496                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 497                         continue;
 498                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 499                                   INIT_CTX_ENABLE_VALUE);
 500                 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
 501                                     CTX_STS_INDIRECT,
 502                                     handle->hal_handle->upc_mask &
 503                                     INIT_PC_VALUE);
 504                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 505                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 506                 qat_hal_put_wakeup_event(handle, ae,
 507                                          ICP_QAT_UCLO_AE_ALL_CTX,
 508                                          INIT_WAKEUP_EVENTS_VALUE);
 509                 qat_hal_put_sig_event(handle, ae,
 510                                       ICP_QAT_UCLO_AE_ALL_CTX,
 511                                       INIT_SIG_EVENTS_VALUE);
 512         }
 513         if (qat_hal_init_esram(handle))
 514                 goto out_err;
 515         if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
 516                 goto out_err;
 517         qat_hal_reset_timestamp(handle);
 518
 519         return 0;
 520 out_err:
 521         pr_err("QAT: failed to get device out of reset\n");
 522         return -EFAULT;
 523 }
 524
 525 static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
 526                                 unsigned char ae, unsigned int ctx_mask)
 527 {
 528         unsigned int ctx;
 529
 530         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
 531         ctx &= IGNORE_W1C_MASK &
 532                 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
 533         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 534 }
 535
 536 static uint64_t qat_hal_parity_64bit(uint64_t word)
 537 {
 538         word ^= word >> 1;
 539         word ^= word >> 2;
 540         word ^= word >> 4;
 541         word ^= word >> 8;
 542         word ^= word >> 16;
 543         word ^= word >> 32;
 544         return word & 1;
 545 }
 546
 547 static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
 548 {
 549         uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
 550                 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
 551                 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
 552                 bit6_mask = 0xdaf69a46910ULL;
 553
 554         /* clear the ecc bits */
 555         uword &= ~(0x7fULL << 0x2C);
 556         uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
 557         uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
 558         uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
 559         uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
 560         uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
 561         uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
 562         uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
 563         return uword;
 564 }
 565
 566 void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
 567                        unsigned char ae, unsigned int uaddr,
 568                        unsigned int words_num, uint64_t *uword)
 569 {
 570         unsigned int ustore_addr;
 571         unsigned int i;
 572
 573         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 574         uaddr |= UA_ECS;
 575         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 576         for (i = 0; i < words_num; i++) {
 577                 unsigned int uwrd_lo, uwrd_hi;
 578                 uint64_t tmp;
 579
 580                 tmp = qat_hal_set_uword_ecc(uword[i]);
 581                 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
 582                 uwrd_hi = (unsigned int)(tmp >> 0x20);
 583                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 584                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 585         }
 586         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 587 }
 588
 589 static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
 590                                unsigned char ae, unsigned int ctx_mask)
 591 {
 592         unsigned int ctx;
 593
 594         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
 595         ctx &= IGNORE_W1C_MASK;
 596         ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
 597         ctx |= (ctx_mask << CE_ENABLE_BITPOS);
 598         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
 599 }
 600
 601 static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
 602 {
 603         unsigned char ae;
 604         unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
 605         int times = MAX_RETRY_TIMES;
 606         unsigned int csr_val = 0;
 607         unsigned short reg;
 608         unsigned int savctx = 0;
 609         int ret = 0;
 610
 611         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 612                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 613                         continue;
 614                 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
 615                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
 616                                              reg, 0);
 617                         qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
 618                                              reg, 0);
 619                 }
 620                 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
 621                 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
 622                 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
 623                 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
 624                 csr_val &= IGNORE_W1C_MASK;
 625                 csr_val |= CE_NN_MODE;
 626                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
 627                 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
 628                                   (uint64_t *)inst);
 629                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 630                                     handle->hal_handle->upc_mask &
 631                                     INIT_PC_VALUE);
 632                 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 633                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
 634                 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
 635                 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
 636                                     CTX_SIG_EVENTS_INDIRECT, 0);
 637                 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 638                 qat_hal_enable_ctx(handle, ae, ctx_mask);
 639         }
 640         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 641                 if (!(handle->hal_handle->ae_mask & (1 << ae)))
 642                         continue;
 643                 /* wait for AE to finish */
 644                 do {
 645                         ret = qat_hal_wait_cycles(handle, ae, 20, 1);
 646                 } while (ret && times--);
 647
 648                 if (!times) {
 649                         pr_err("QAT: clear GPR of AE %d failed", ae);
 650                         return -EINVAL;
 651                 }
 652                 qat_hal_disable_ctx(handle, ae, ctx_mask);
 653                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 654                                   savctx & ACS_ACNO);
 655                 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
 656                                   INIT_CTX_ENABLE_VALUE);
 657                 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 658                                     handle->hal_handle->upc_mask &
 659                                     INIT_PC_VALUE);
 660                 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
 661                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
 662                 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
 663                                          INIT_WAKEUP_EVENTS_VALUE);
 664                 qat_hal_put_sig_event(handle, ae, ctx_mask,
 665                                       INIT_SIG_EVENTS_VALUE);
 666         }
 667         return 0;
 668 }
 669
 670 #define ICP_DH895XCC_AE_OFFSET      0x20000
 671 #define ICP_DH895XCC_CAP_OFFSET     (ICP_DH895XCC_AE_OFFSET + 0x10000)
 672 #define LOCAL_TO_XFER_REG_OFFSET    0x800
 673 #define ICP_DH895XCC_EP_OFFSET      0x3a000
 674 #define ICP_DH895XCC_PMISC_BAR 1
 675 int qat_hal_init(struct adf_accel_dev *accel_dev)
 676 {
 677         unsigned char ae;
 678         unsigned int max_en_ae_id = 0;
 679         struct icp_qat_fw_loader_handle *handle;
 680         struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
 681         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 682         struct adf_bar *bar =
 683                         &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
 684
 685         handle = kzalloc(sizeof(*handle), GFP_KERNEL);
 686         if (!handle)
 687                 return -ENOMEM;
 688
 689         handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr +
 690                                                 ICP_DH895XCC_CAP_OFFSET;
 691         handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr +
 692                                                 ICP_DH895XCC_AE_OFFSET;
 693         handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET;
 694         handle->hal_cap_ae_local_csr_addr_v =
 695                 handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET;
 696
 697         handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
 698         if (!handle->hal_handle)
 699                 goto out_hal_handle;
 700         handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
 701         handle->hal_handle->ae_mask = hw_data->ae_mask;
 702         handle->hal_handle->slice_mask = hw_data->accel_mask;
 703         /* create AE objects */
 704         handle->hal_handle->upc_mask = 0x1ffff;
 705         handle->hal_handle->max_ustore = 0x4000;
 706         for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
 707                 if (!(hw_data->ae_mask & (1 << ae)))
 708                         continue;
 709                 handle->hal_handle->aes[ae].free_addr = 0;
 710                 handle->hal_handle->aes[ae].free_size =
 711                     handle->hal_handle->max_ustore;
 712                 handle->hal_handle->aes[ae].ustore_size =
 713                     handle->hal_handle->max_ustore;
 714                 handle->hal_handle->aes[ae].live_ctx_mask =
 715                                                 ICP_QAT_UCLO_AE_ALL_CTX;
 716                 max_en_ae_id = ae;
 717         }
 718         handle->hal_handle->ae_max_num = max_en_ae_id + 1;
 719         /* take all AEs out of reset */
 720         if (qat_hal_clr_reset(handle)) {
 721                 dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n");
 722                 goto out_err;
 723         }
 724         if (qat_hal_clear_gpr(handle))
 725                 goto out_err;
 726         /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */
 727         for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
 728                 unsigned int csr_val = 0;
 729
 730                 if (!(hw_data->ae_mask & (1 << ae)))
 731                         continue;
 732                 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
 733                 csr_val |= 0x1;
 734                 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
 735         }
 736         accel_dev->fw_loader->fw_loader = handle;
 737         return 0;
 738
 739 out_err:
 740         kfree(handle->hal_handle);
 741 out_hal_handle:
 742         kfree(handle);
 743         return -EFAULT;
 744 }
 745
 746 void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
 747 {
 748         if (!handle)
 749                 return;
 750         kfree(handle->hal_handle);
 751         kfree(handle);
 752 }
 753
 754 void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 755                    unsigned int ctx_mask)
 756 {
 757         qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
 758                                  ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
 759         qat_hal_enable_ctx(handle, ae, ctx_mask);
 760 }
 761
 762 void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
 763                   unsigned int ctx_mask)
 764 {
 765         qat_hal_disable_ctx(handle, ae, ctx_mask);
 766 }
 767
 768 void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
 769                     unsigned char ae, unsigned int ctx_mask, unsigned int upc)
 770 {
 771         qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
 772                             handle->hal_handle->upc_mask & upc);
 773 }
 774
 775 static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
 776                                unsigned char ae, unsigned int uaddr,
 777                                unsigned int words_num, uint64_t *uword)
 778 {
 779         unsigned int i, uwrd_lo, uwrd_hi;
 780         unsigned int ustore_addr, misc_control;
 781
 782         qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
 783         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
 784                           misc_control & 0xfffffffb);
 785         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 786         uaddr |= UA_ECS;
 787         for (i = 0; i < words_num; i++) {
 788                 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 789                 uaddr++;
 790                 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
 791                 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
 792                 uword[i] = uwrd_hi;
 793                 uword[i] = (uword[i] << 0x20) | uwrd_lo;
 794         }
 795         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
 796         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 797 }
 798
 799 void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
 800                      unsigned char ae, unsigned int uaddr,
 801                      unsigned int words_num, unsigned int *data)
 802 {
 803         unsigned int i, ustore_addr;
 804
 805         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 806         uaddr |= UA_ECS;
 807         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 808         for (i = 0; i < words_num; i++) {
 809                 unsigned int uwrd_lo, uwrd_hi, tmp;
 810
 811                 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
 812                           ((data[i] & 0xff00) << 2) |
 813                           (0x3 << 8) | (data[i] & 0xff);
 814                 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
 815                 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
 816                 tmp = ((data[i] >> 0x10) & 0xffff);
 817                 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
 818                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 819                 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 820         }
 821         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 822 }
 823
 824 #define MAX_EXEC_INST 100
 825 static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
 826                                    unsigned char ae, unsigned char ctx,
 827                                    uint64_t *micro_inst, unsigned int inst_num,
 828                                    int code_off, unsigned int max_cycle,
 829                                    unsigned int *endpc)
 830 {
 831         uint64_t savuwords[MAX_EXEC_INST];
 832         unsigned int ind_lm_addr0, ind_lm_addr1;
 833         unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
 834         unsigned int ind_cnt_sig;
 835         unsigned int ind_sig, act_sig;
 836         unsigned int csr_val = 0, newcsr_val;
 837         unsigned int savctx;
 838         unsigned int savcc, wakeup_events, savpc;
 839         unsigned int ctxarb_ctl, ctx_enables;
 840
 841         if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
 842                 pr_err("QAT: invalid instruction num %d\n", inst_num);
 843                 return -EINVAL;
 844         }
 845         /* save current context */
 846         qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
 847         qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
 848         qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
 849                             &ind_lm_addr_byte0);
 850         qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
 851                             &ind_lm_addr_byte1);
 852         if (inst_num <= MAX_EXEC_INST)
 853                 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
 854         qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
 855         qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
 856         savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
 857         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
 858         ctx_enables &= IGNORE_W1C_MASK;
 859         qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
 860         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 861         qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
 862         qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
 863                             &ind_cnt_sig);
 864         qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
 865         qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
 866         /* execute micro codes */
 867         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 868         qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
 869         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
 870         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
 871         if (code_off)
 872                 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
 873         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
 874         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
 875         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
 876         qat_hal_enable_ctx(handle, ae, (1 << ctx));
 877         /* wait for micro codes to finish */
 878         if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
 879                 return -EFAULT;
 880         if (endpc) {
 881                 unsigned int ctx_status;
 882
 883                 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT,
 884                                     &ctx_status);
 885                 *endpc = ctx_status & handle->hal_handle->upc_mask;
 886         }
 887         /* retore to saved context */
 888         qat_hal_disable_ctx(handle, ae, (1 << ctx));
 889         if (inst_num <= MAX_EXEC_INST)
 890                 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
 891         qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
 892         qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
 893                             handle->hal_handle->upc_mask & savpc);
 894         qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
 895         newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
 896         qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
 897         qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
 898         qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
 899         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
 900         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 901                             LM_ADDR_0_INDIRECT, ind_lm_addr0);
 902         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 903                             LM_ADDR_1_INDIRECT, ind_lm_addr1);
 904         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 905                             INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
 906         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 907                             INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
 908         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 909                             FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
 910         qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
 911                             CTX_SIG_EVENTS_INDIRECT, ind_sig);
 912         qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
 913         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 914
 915         return 0;
 916 }
 917
 918 static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
 919                               unsigned char ae, unsigned char ctx,
 920                               enum icp_qat_uof_regtype reg_type,
 921                               unsigned short reg_num, unsigned int *data)
 922 {
 923         unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
 924         unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
 925         unsigned short reg_addr;
 926         int status = 0;
 927         uint64_t insts, savuword;
 928
 929         reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
 930         if (reg_addr == BAD_REGADDR) {
 931                 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
 932                 return -EINVAL;
 933         }
 934         switch (reg_type) {
 935         case ICP_GPA_REL:
 936                 insts = 0xA070000000ull | (reg_addr & 0x3ff);
 937                 break;
 938         default:
 939                 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
 940                 break;
 941         }
 942         qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
 943         qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
 944         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
 945         ctx_enables &= IGNORE_W1C_MASK;
 946         if (ctx != (savctx & ACS_ACNO))
 947                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 948                                   ctx & ACS_ACNO);
 949         qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
 950         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 951         qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
 952         uaddr = UA_ECS;
 953         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 954         insts = qat_hal_set_uword_ecc(insts);
 955         uwrd_lo = (unsigned int)(insts & 0xffffffff);
 956         uwrd_hi = (unsigned int)(insts >> 0x20);
 957         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
 958         qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
 959         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
 960         /* delay for at least 8 cycles */
 961         qat_hal_wait_cycles(handle, ae, 0x8, 0);
 962         /*
 963          * read ALU output
 964          * the instruction should have been executed
 965          * prior to clearing the ECS in putUwords
 966          */
 967         qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
 968         qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
 969         qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
 970         if (ctx != (savctx & ACS_ACNO))
 971                 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
 972                                   savctx & ACS_ACNO);
 973         qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
 974         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
 975
 976         return status;
 977 }
 978
 979 static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
 980                               unsigned char ae, unsigned char ctx,
 981                               enum icp_qat_uof_regtype reg_type,
 982                               unsigned short reg_num, unsigned int data)
 983 {
 984         unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
 985         uint64_t insts[] = {
 986                 0x0F440000000ull,
 987                 0x0F040000000ull,
 988                 0x0F0000C0300ull,
 989                 0x0E000010000ull
 990         };
 991         const int num_inst = ARRAY_SIZE(insts), code_off = 1;
 992         const int imm_w1 = 0, imm_w0 = 1;
 993
 994         dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
 995         if (dest_addr == BAD_REGADDR) {
 996                 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
 997                 return -EINVAL;
 998         }
 999
1000         data16lo = 0xffff & data;
1001         data16hi = 0xffff & (data >> 0x10);
1002         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1003                                           (0xff & data16hi));
1004         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1005                                            (0xff & data16lo));
1006         switch (reg_type) {
1007         case ICP_GPA_REL:
1008                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1009                     ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1010                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1011                     ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1012                 break;
1013         default:
1014                 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1015                     ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1016
1017                 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1018                     ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1019                 break;
1020         }
1021
1022         return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1023                                        code_off, num_inst * 0x5, NULL);
1024 }
1025
1026 int qat_hal_get_ins_num(void)
1027 {
1028         return ARRAY_SIZE(inst_4b);
1029 }
1030
1031 static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1032                                      unsigned int inst_num, unsigned int size,
1033                                      unsigned int addr, unsigned int *value)
1034 {
1035         int i, val_indx;
1036         unsigned int cur_value;
1037         const uint64_t *inst_arr;
1038         int fixup_offset;
1039         int usize = 0;
1040         int orig_num;
1041
1042         orig_num = inst_num;
1043         val_indx = 0;
1044         cur_value = value[val_indx++];
1045         inst_arr = inst_4b;
1046         usize = ARRAY_SIZE(inst_4b);
1047         fixup_offset = inst_num;
1048         for (i = 0; i < usize; i++)
1049                 micro_inst[inst_num++] = inst_arr[i];
1050         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1051         fixup_offset++;
1052         INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1053         fixup_offset++;
1054         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1055         fixup_offset++;
1056         INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1057
1058         return inst_num - orig_num;
1059 }
1060
1061 static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1062                                       unsigned char ae, unsigned char ctx,
1063                                       int *pfirst_exec, uint64_t *micro_inst,
1064                                       unsigned int inst_num)
1065 {
1066         int stat = 0;
1067         unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1068         unsigned int gprb0 = 0, gprb1 = 0;
1069
1070         if (*pfirst_exec) {
1071                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1072                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1073                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1074                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1075                 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1076                 *pfirst_exec = 0;
1077         }
1078         stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1079                                        inst_num * 0x5, NULL);
1080         if (stat != 0)
1081                 return -EFAULT;
1082         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1083         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1084         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1085         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1086         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1087
1088         return 0;
1089 }
1090
1091 int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1092                         unsigned char ae,
1093                         struct icp_qat_uof_batch_init *lm_init_header)
1094 {
1095         struct icp_qat_uof_batch_init *plm_init;
1096         uint64_t *micro_inst_arry;
1097         int micro_inst_num;
1098         int alloc_inst_size;
1099         int first_exec = 1;
1100         int stat = 0;
1101
1102         plm_init = lm_init_header->next;
1103         alloc_inst_size = lm_init_header->size;
1104         if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1105                 alloc_inst_size = handle->hal_handle->max_ustore;
1106         micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1107                                         GFP_KERNEL);
1108         if (!micro_inst_arry)
1109                 return -ENOMEM;
1110         micro_inst_num = 0;
1111         while (plm_init) {
1112                 unsigned int addr, *value, size;
1113
1114                 ae = plm_init->ae;
1115                 addr = plm_init->addr;
1116                 value = plm_init->value;
1117                 size = plm_init->size;
1118                 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1119                                                             micro_inst_num,
1120                                                             size, addr, value);
1121                 plm_init = plm_init->next;
1122         }
1123         /* exec micro codes */
1124         if (micro_inst_arry && (micro_inst_num > 0)) {
1125                 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1126                 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1127                                                   micro_inst_arry,
1128                                                   micro_inst_num);
1129         }
1130         kfree(micro_inst_arry);
1131         return stat;
1132 }
1133
1134 static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1135                                    unsigned char ae, unsigned char ctx,
1136                                    enum icp_qat_uof_regtype reg_type,
1137                                    unsigned short reg_num, unsigned int val)
1138 {
1139         int status = 0;
1140         unsigned int reg_addr;
1141         unsigned int ctx_enables;
1142         unsigned short mask;
1143         unsigned short dr_offset = 0x10;
1144
1145         status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1146         if (CE_INUSE_CONTEXTS & ctx_enables) {
1147                 if (ctx & 0x1) {
1148                         pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1149                         return -EINVAL;
1150                 }
1151                 mask = 0x1f;
1152                 dr_offset = 0x20;
1153         } else {
1154                 mask = 0x0f;
1155         }
1156         if (reg_num & ~mask)
1157                 return -EINVAL;
1158         reg_addr = reg_num + (ctx << 0x5);
1159         switch (reg_type) {
1160         case ICP_SR_RD_REL:
1161         case ICP_SR_REL:
1162                 SET_AE_XFER(handle, ae, reg_addr, val);
1163                 break;
1164         case ICP_DR_RD_REL:
1165         case ICP_DR_REL:
1166                 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1167                 break;
1168         default:
1169                 status = -EINVAL;
1170                 break;
1171         }
1172         return status;
1173 }
1174
1175 static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1176                                    unsigned char ae, unsigned char ctx,
1177                                    enum icp_qat_uof_regtype reg_type,
1178                                    unsigned short reg_num, unsigned int data)
1179 {
1180         unsigned int gprval, ctx_enables;
1181         unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1182             data16low;
1183         unsigned short reg_mask;
1184         int status = 0;
1185         uint64_t micro_inst[] = {
1186                 0x0F440000000ull,
1187                 0x0F040000000ull,
1188                 0x0A000000000ull,
1189                 0x0F0000C0300ull,
1190                 0x0E000010000ull
1191         };
1192         const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1193         const unsigned short gprnum = 0, dly = num_inst * 0x5;
1194
1195         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1196         if (CE_INUSE_CONTEXTS & ctx_enables) {
1197                 if (ctx & 0x1) {
1198                         pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1199                         return -EINVAL;
1200                 }
1201                 reg_mask = (unsigned short)~0x1f;
1202         } else {
1203                 reg_mask = (unsigned short)~0xf;
1204         }
1205         if (reg_num & reg_mask)
1206                 return -EINVAL;
1207         xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1208         if (xfr_addr == BAD_REGADDR) {
1209                 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1210                 return -EINVAL;
1211         }
1212         qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1213         gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1214         data16low = 0xffff & data;
1215         data16hi = 0xffff & (data >> 0x10);
1216         src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1217                                           (unsigned short)(0xff & data16hi));
1218         src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1219                                            (unsigned short)(0xff & data16low));
1220         micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1221             ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1222         micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1223             ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1224         micro_inst[0x2] = micro_inst[0x2] |
1225             ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1226         status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1227                                          code_off, dly, NULL);
1228         qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1229         return status;
1230 }
1231
1232 static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1233                               unsigned char ae, unsigned char ctx,
1234                               unsigned short nn, unsigned int val)
1235 {
1236         unsigned int ctx_enables;
1237         int stat = 0;
1238
1239         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1240         ctx_enables &= IGNORE_W1C_MASK;
1241         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1242
1243         stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1244         qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1245         return stat;
1246 }
1247
1248 static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1249                                       *handle, unsigned char ae,
1250                                       unsigned short absreg_num,
1251                                       unsigned short *relreg,
1252                                       unsigned char *ctx)
1253 {
1254         unsigned int ctx_enables;
1255
1256         qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1257         if (ctx_enables & CE_INUSE_CONTEXTS) {
1258                 /* 4-ctx mode */
1259                 *relreg = absreg_num & 0x1F;
1260                 *ctx = (absreg_num >> 0x4) & 0x6;
1261         } else {
1262                 /* 8-ctx mode */
1263                 *relreg = absreg_num & 0x0F;
1264                 *ctx = (absreg_num >> 0x4) & 0x7;
1265         }
1266         return 0;
1267 }
1268
1269 int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1270                      unsigned char ae, unsigned char ctx_mask,
1271                      enum icp_qat_uof_regtype reg_type,
1272                      unsigned short reg_num, unsigned int regdata)
1273 {
1274         int stat = 0;
1275         unsigned short reg;
1276         unsigned char ctx = 0;
1277         enum icp_qat_uof_regtype type;
1278
1279         if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1280                 return -EINVAL;
1281
1282         do {
1283                 if (ctx_mask == 0) {
1284                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1285                                                    &ctx);
1286                         type = reg_type - 1;
1287                 } else {
1288                         reg = reg_num;
1289                         type = reg_type;
1290                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1291                                 continue;
1292                 }
1293                 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1294                 if (stat) {
1295                         pr_err("QAT: write gpr fail\n");
1296                         return -EINVAL;
1297                 }
1298         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1299
1300         return 0;
1301 }
1302
1303 int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1304                          unsigned char ae, unsigned char ctx_mask,
1305                          enum icp_qat_uof_regtype reg_type,
1306                          unsigned short reg_num, unsigned int regdata)
1307 {
1308         int stat = 0;
1309         unsigned short reg;
1310         unsigned char ctx = 0;
1311         enum icp_qat_uof_regtype type;
1312
1313         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1314                 return -EINVAL;
1315
1316         do {
1317                 if (ctx_mask == 0) {
1318                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1319                                                    &ctx);
1320                         type = reg_type - 3;
1321                 } else {
1322                         reg = reg_num;
1323                         type = reg_type;
1324                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1325                                 continue;
1326                 }
1327                 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1328                                                regdata);
1329                 if (stat) {
1330                         pr_err("QAT: write wr xfer fail\n");
1331                         return -EINVAL;
1332                 }
1333         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1334
1335         return 0;
1336 }
1337
1338 int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1339                          unsigned char ae, unsigned char ctx_mask,
1340                          enum icp_qat_uof_regtype reg_type,
1341                          unsigned short reg_num, unsigned int regdata)
1342 {
1343         int stat = 0;
1344         unsigned short reg;
1345         unsigned char ctx = 0;
1346         enum icp_qat_uof_regtype type;
1347
1348         if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1349                 return -EINVAL;
1350
1351         do {
1352                 if (ctx_mask == 0) {
1353                         qat_hal_convert_abs_to_rel(handle, ae, reg_num, &reg,
1354                                                    &ctx);
1355                         type = reg_type - 3;
1356                 } else {
1357                         reg = reg_num;
1358                         type = reg_type;
1359                         if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1360                                 continue;
1361                 }
1362                 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1363                                                regdata);
1364                 if (stat) {
1365                         pr_err("QAT: write rd xfer fail\n");
1366                         return -EINVAL;
1367                 }
1368         } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1369
1370         return 0;
1371 }
1372
1373 int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1374                     unsigned char ae, unsigned char ctx_mask,
1375                     unsigned short reg_num, unsigned int regdata)
1376 {
1377         int stat = 0;
1378         unsigned char ctx;
1379
1380         if (ctx_mask == 0)
1381                 return -EINVAL;
1382
1383         for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1384                 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1385                         continue;
1386                 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1387                 if (stat) {
1388                         pr_err("QAT: write neigh error\n");
1389                         return -EINVAL;
1390                 }
1391         }
1392
1393         return 0;
1394 }