kernel/drivers/scsi/be2iscsi/be_main.c

   1 /**
   2  * Copyright (C) 2005 - 2015 Avago Technologies
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Written by: Jayamohan Kallickal (jayamohan.kallickal@avagotech.com)
  11  *
  12  * Contact Information:
  13  * linux-drivers@avagotech.com
  14  *
  15  * Avago Technologies
  16  * 3333 Susan Street
  17  * Costa Mesa, CA 92626
  18  */
  19
  20 #include <linux/reboot.h>
  21 #include <linux/delay.h>
  22 #include <linux/slab.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/blkdev.h>
  25 #include <linux/pci.h>
  26 #include <linux/string.h>
  27 #include <linux/kernel.h>
  28 #include <linux/semaphore.h>
  29 #include <linux/iscsi_boot_sysfs.h>
  30 #include <linux/module.h>
  31 #include <linux/bsg-lib.h>
  32
  33 #include <scsi/libiscsi.h>
  34 #include <scsi/scsi_bsg_iscsi.h>
  35 #include <scsi/scsi_netlink.h>
  36 #include <scsi/scsi_transport_iscsi.h>
  37 #include <scsi/scsi_transport.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_host.h>
  41 #include <scsi/scsi.h>
  42 #include "be_main.h"
  43 #include "be_iscsi.h"
  44 #include "be_mgmt.h"
  45 #include "be_cmds.h"
  46
  47 static unsigned int be_iopoll_budget = 10;
  48 static unsigned int be_max_phys_size = 64;
  49 static unsigned int enable_msix = 1;
  50
  51 MODULE_DESCRIPTION(DRV_DESC " " BUILD_STR);
  52 MODULE_VERSION(BUILD_STR);
  53 MODULE_AUTHOR("Avago Technologies");
  54 MODULE_LICENSE("GPL");
  55 module_param(be_iopoll_budget, int, 0);
  56 module_param(enable_msix, int, 0);
  57 module_param(be_max_phys_size, uint, S_IRUGO);
  58 MODULE_PARM_DESC(be_max_phys_size,
  59                 "Maximum Size (In Kilobytes) of physically contiguous "
  60                 "memory that can be allocated. Range is 16 - 128");
  61
  62 #define beiscsi_disp_param(_name)\
  63 ssize_t \
  64 beiscsi_##_name##_disp(struct device *dev,\
  65                         struct device_attribute *attrib, char *buf)     \
  66 {       \
  67         struct Scsi_Host *shost = class_to_shost(dev);\
  68         struct beiscsi_hba *phba = iscsi_host_priv(shost); \
  69         uint32_t param_val = 0; \
  70         param_val = phba->attr_##_name;\
  71         return snprintf(buf, PAGE_SIZE, "%d\n",\
  72                         phba->attr_##_name);\
  73 }
  74
  75 #define beiscsi_change_param(_name, _minval, _maxval, _defaval)\
  76 int \
  77 beiscsi_##_name##_change(struct beiscsi_hba *phba, uint32_t val)\
  78 {\
  79         if (val >= _minval && val <= _maxval) {\
  80                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
  81                             "BA_%d : beiscsi_"#_name" updated "\
  82                             "from 0x%x ==> 0x%x\n",\
  83                             phba->attr_##_name, val); \
  84                 phba->attr_##_name = val;\
  85                 return 0;\
  86         } \
  87         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, \
  88                     "BA_%d beiscsi_"#_name" attribute "\
  89                     "cannot be updated to 0x%x, "\
  90                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
  91                 return -EINVAL;\
  92 }
  93
  94 #define beiscsi_store_param(_name)  \
  95 ssize_t \
  96 beiscsi_##_name##_store(struct device *dev,\
  97                          struct device_attribute *attr, const char *buf,\
  98                          size_t count) \
  99 { \
 100         struct Scsi_Host  *shost = class_to_shost(dev);\
 101         struct beiscsi_hba *phba = iscsi_host_priv(shost);\
 102         uint32_t param_val = 0;\
 103         if (!isdigit(buf[0]))\
 104                 return -EINVAL;\
 105         if (sscanf(buf, "%i", &param_val) != 1)\
 106                 return -EINVAL;\
 107         if (beiscsi_##_name##_change(phba, param_val) == 0) \
 108                 return strlen(buf);\
 109         else \
 110                 return -EINVAL;\
 111 }
 112
 113 #define beiscsi_init_param(_name, _minval, _maxval, _defval) \
 114 int \
 115 beiscsi_##_name##_init(struct beiscsi_hba *phba, uint32_t val) \
 116 { \
 117         if (val >= _minval && val <= _maxval) {\
 118                 phba->attr_##_name = val;\
 119                 return 0;\
 120         } \
 121         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,\
 122                     "BA_%d beiscsi_"#_name" attribute " \
 123                     "cannot be updated to 0x%x, "\
 124                     "range allowed is ["#_minval" - "#_maxval"]\n", val);\
 125         phba->attr_##_name = _defval;\
 126         return -EINVAL;\
 127 }
 128
 129 #define BEISCSI_RW_ATTR(_name, _minval, _maxval, _defval, _descp) \
 130 static uint beiscsi_##_name = _defval;\
 131 module_param(beiscsi_##_name, uint, S_IRUGO);\
 132 MODULE_PARM_DESC(beiscsi_##_name, _descp);\
 133 beiscsi_disp_param(_name)\
 134 beiscsi_change_param(_name, _minval, _maxval, _defval)\
 135 beiscsi_store_param(_name)\
 136 beiscsi_init_param(_name, _minval, _maxval, _defval)\
 137 DEVICE_ATTR(beiscsi_##_name, S_IRUGO | S_IWUSR,\
 138               beiscsi_##_name##_disp, beiscsi_##_name##_store)
 139
 140 /*
 141  * When new log level added update the
 142  * the MAX allowed value for log_enable
 143  */
 144 BEISCSI_RW_ATTR(log_enable, 0x00,
 145                 0xFF, 0x00, "Enable logging Bit Mask\n"
 146                 "\t\t\t\tInitialization Events  : 0x01\n"
 147                 "\t\t\t\tMailbox Events         : 0x02\n"
 148                 "\t\t\t\tMiscellaneous Events   : 0x04\n"
 149                 "\t\t\t\tError Handling         : 0x08\n"
 150                 "\t\t\t\tIO Path Events         : 0x10\n"
 151                 "\t\t\t\tConfiguration Path     : 0x20\n"
 152                 "\t\t\t\tiSCSI Protocol         : 0x40\n");
 153
 154 DEVICE_ATTR(beiscsi_drvr_ver, S_IRUGO, beiscsi_drvr_ver_disp, NULL);
 155 DEVICE_ATTR(beiscsi_adapter_family, S_IRUGO, beiscsi_adap_family_disp, NULL);
 156 DEVICE_ATTR(beiscsi_fw_ver, S_IRUGO, beiscsi_fw_ver_disp, NULL);
 157 DEVICE_ATTR(beiscsi_phys_port, S_IRUGO, beiscsi_phys_port_disp, NULL);
 158 DEVICE_ATTR(beiscsi_active_session_count, S_IRUGO,
 159              beiscsi_active_session_disp, NULL);
 160 DEVICE_ATTR(beiscsi_free_session_count, S_IRUGO,
 161              beiscsi_free_session_disp, NULL);
 162 struct device_attribute *beiscsi_attrs[] = {
 163         &dev_attr_beiscsi_log_enable,
 164         &dev_attr_beiscsi_drvr_ver,
 165         &dev_attr_beiscsi_adapter_family,
 166         &dev_attr_beiscsi_fw_ver,
 167         &dev_attr_beiscsi_active_session_count,
 168         &dev_attr_beiscsi_free_session_count,
 169         &dev_attr_beiscsi_phys_port,
 170         NULL,
 171 };
 172
 173 static char const *cqe_desc[] = {
 174         "RESERVED_DESC",
 175         "SOL_CMD_COMPLETE",
 176         "SOL_CMD_KILLED_DATA_DIGEST_ERR",
 177         "CXN_KILLED_PDU_SIZE_EXCEEDS_DSL",
 178         "CXN_KILLED_BURST_LEN_MISMATCH",
 179         "CXN_KILLED_AHS_RCVD",
 180         "CXN_KILLED_HDR_DIGEST_ERR",
 181         "CXN_KILLED_UNKNOWN_HDR",
 182         "CXN_KILLED_STALE_ITT_TTT_RCVD",
 183         "CXN_KILLED_INVALID_ITT_TTT_RCVD",
 184         "CXN_KILLED_RST_RCVD",
 185         "CXN_KILLED_TIMED_OUT",
 186         "CXN_KILLED_RST_SENT",
 187         "CXN_KILLED_FIN_RCVD",
 188         "CXN_KILLED_BAD_UNSOL_PDU_RCVD",
 189         "CXN_KILLED_BAD_WRB_INDEX_ERROR",
 190         "CXN_KILLED_OVER_RUN_RESIDUAL",
 191         "CXN_KILLED_UNDER_RUN_RESIDUAL",
 192         "CMD_KILLED_INVALID_STATSN_RCVD",
 193         "CMD_KILLED_INVALID_R2T_RCVD",
 194         "CMD_CXN_KILLED_LUN_INVALID",
 195         "CMD_CXN_KILLED_ICD_INVALID",
 196         "CMD_CXN_KILLED_ITT_INVALID",
 197         "CMD_CXN_KILLED_SEQ_OUTOFORDER",
 198         "CMD_CXN_KILLED_INVALID_DATASN_RCVD",
 199         "CXN_INVALIDATE_NOTIFY",
 200         "CXN_INVALIDATE_INDEX_NOTIFY",
 201         "CMD_INVALIDATED_NOTIFY",
 202         "UNSOL_HDR_NOTIFY",
 203         "UNSOL_DATA_NOTIFY",
 204         "UNSOL_DATA_DIGEST_ERROR_NOTIFY",
 205         "DRIVERMSG_NOTIFY",
 206         "CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN",
 207         "SOL_CMD_KILLED_DIF_ERR",
 208         "CXN_KILLED_SYN_RCVD",
 209         "CXN_KILLED_IMM_DATA_RCVD"
 210 };
 211
 212 static int beiscsi_slave_configure(struct scsi_device *sdev)
 213 {
 214         blk_queue_max_segment_size(sdev->request_queue, 65536);
 215         return 0;
 216 }
 217
 218 static int beiscsi_eh_abort(struct scsi_cmnd *sc)
 219 {
 220         struct iscsi_cls_session *cls_session;
 221         struct iscsi_task *aborted_task = (struct iscsi_task *)sc->SCp.ptr;
 222         struct beiscsi_io_task *aborted_io_task;
 223         struct iscsi_conn *conn;
 224         struct beiscsi_conn *beiscsi_conn;
 225         struct beiscsi_hba *phba;
 226         struct iscsi_session *session;
 227         struct invalidate_command_table *inv_tbl;
 228         struct be_dma_mem nonemb_cmd;
 229         unsigned int cid, tag, num_invalidate;
 230         int rc;
 231
 232         cls_session = starget_to_session(scsi_target(sc->device));
 233         session = cls_session->dd_data;
 234
 235         spin_lock_bh(&session->frwd_lock);
 236         if (!aborted_task || !aborted_task->sc) {
 237                 /* we raced */
 238                 spin_unlock_bh(&session->frwd_lock);
 239                 return SUCCESS;
 240         }
 241
 242         aborted_io_task = aborted_task->dd_data;
 243         if (!aborted_io_task->scsi_cmnd) {
 244                 /* raced or invalid command */
 245                 spin_unlock_bh(&session->frwd_lock);
 246                 return SUCCESS;
 247         }
 248         spin_unlock_bh(&session->frwd_lock);
 249         /* Invalidate WRB Posted for this Task */
 250         AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 251                       aborted_io_task->pwrb_handle->pwrb,
 252                       1);
 253
 254         conn = aborted_task->conn;
 255         beiscsi_conn = conn->dd_data;
 256         phba = beiscsi_conn->phba;
 257
 258         /* invalidate iocb */
 259         cid = beiscsi_conn->beiscsi_conn_cid;
 260         inv_tbl = phba->inv_tbl;
 261         memset(inv_tbl, 0x0, sizeof(*inv_tbl));
 262         inv_tbl->cid = cid;
 263         inv_tbl->icd = aborted_io_task->psgl_handle->sgl_index;
 264         num_invalidate = 1;
 265         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 266                                 sizeof(struct invalidate_commands_params_in),
 267                                 &nonemb_cmd.dma);
 268         if (nonemb_cmd.va == NULL) {
 269                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 270                             "BM_%d : Failed to allocate memory for"
 271                             "mgmt_invalidate_icds\n");
 272                 return FAILED;
 273         }
 274         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 275
 276         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 277                                    cid, &nonemb_cmd);
 278         if (!tag) {
 279                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 280                             "BM_%d : mgmt_invalidate_icds could not be"
 281                             "submitted\n");
 282                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 283                                     nonemb_cmd.va, nonemb_cmd.dma);
 284
 285                 return FAILED;
 286         }
 287
 288         rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
 289         if (rc != -EBUSY)
 290                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 291                                     nonemb_cmd.va, nonemb_cmd.dma);
 292
 293         return iscsi_eh_abort(sc);
 294 }
 295
 296 static int beiscsi_eh_device_reset(struct scsi_cmnd *sc)
 297 {
 298         struct iscsi_task *abrt_task;
 299         struct beiscsi_io_task *abrt_io_task;
 300         struct iscsi_conn *conn;
 301         struct beiscsi_conn *beiscsi_conn;
 302         struct beiscsi_hba *phba;
 303         struct iscsi_session *session;
 304         struct iscsi_cls_session *cls_session;
 305         struct invalidate_command_table *inv_tbl;
 306         struct be_dma_mem nonemb_cmd;
 307         unsigned int cid, tag, i, num_invalidate;
 308         int rc;
 309
 310         /* invalidate iocbs */
 311         cls_session = starget_to_session(scsi_target(sc->device));
 312         session = cls_session->dd_data;
 313         spin_lock_bh(&session->frwd_lock);
 314         if (!session->leadconn || session->state != ISCSI_STATE_LOGGED_IN) {
 315                 spin_unlock_bh(&session->frwd_lock);
 316                 return FAILED;
 317         }
 318         conn = session->leadconn;
 319         beiscsi_conn = conn->dd_data;
 320         phba = beiscsi_conn->phba;
 321         cid = beiscsi_conn->beiscsi_conn_cid;
 322         inv_tbl = phba->inv_tbl;
 323         memset(inv_tbl, 0x0, sizeof(*inv_tbl) * BE2_CMDS_PER_CXN);
 324         num_invalidate = 0;
 325         for (i = 0; i < conn->session->cmds_max; i++) {
 326                 abrt_task = conn->session->cmds[i];
 327                 abrt_io_task = abrt_task->dd_data;
 328                 if (!abrt_task->sc || abrt_task->state == ISCSI_TASK_FREE)
 329                         continue;
 330
 331                 if (sc->device->lun != abrt_task->sc->device->lun)
 332                         continue;
 333
 334                 /* Invalidate WRB Posted for this Task */
 335                 AMAP_SET_BITS(struct amap_iscsi_wrb, invld,
 336                               abrt_io_task->pwrb_handle->pwrb,
 337                               1);
 338
 339                 inv_tbl->cid = cid;
 340                 inv_tbl->icd = abrt_io_task->psgl_handle->sgl_index;
 341                 num_invalidate++;
 342                 inv_tbl++;
 343         }
 344         spin_unlock_bh(&session->frwd_lock);
 345         inv_tbl = phba->inv_tbl;
 346
 347         nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
 348                                 sizeof(struct invalidate_commands_params_in),
 349                                 &nonemb_cmd.dma);
 350         if (nonemb_cmd.va == NULL) {
 351                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_EH,
 352                             "BM_%d : Failed to allocate memory for"
 353                             "mgmt_invalidate_icds\n");
 354                 return FAILED;
 355         }
 356         nonemb_cmd.size = sizeof(struct invalidate_commands_params_in);
 357         memset(nonemb_cmd.va, 0, nonemb_cmd.size);
 358         tag = mgmt_invalidate_icds(phba, inv_tbl, num_invalidate,
 359                                    cid, &nonemb_cmd);
 360         if (!tag) {
 361                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_EH,
 362                             "BM_%d : mgmt_invalidate_icds could not be"
 363                             " submitted\n");
 364                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 365                                     nonemb_cmd.va, nonemb_cmd.dma);
 366                 return FAILED;
 367         }
 368
 369         rc = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
 370         if (rc != -EBUSY)
 371                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
 372                                     nonemb_cmd.va, nonemb_cmd.dma);
 373         return iscsi_eh_device_reset(sc);
 374 }
 375
 376 static ssize_t beiscsi_show_boot_tgt_info(void *data, int type, char *buf)
 377 {
 378         struct beiscsi_hba *phba = data;
 379         struct mgmt_session_info *boot_sess = &phba->boot_sess;
 380         struct mgmt_conn_info *boot_conn = &boot_sess->conn_list[0];
 381         char *str = buf;
 382         int rc;
 383
 384         switch (type) {
 385         case ISCSI_BOOT_TGT_NAME:
 386                 rc = sprintf(buf, "%.*s\n",
 387                             (int)strlen(boot_sess->target_name),
 388                             (char *)&boot_sess->target_name);
 389                 break;
 390         case ISCSI_BOOT_TGT_IP_ADDR:
 391                 if (boot_conn->dest_ipaddr.ip_type == 0x1)
 392                         rc = sprintf(buf, "%pI4\n",
 393                                 (char *)&boot_conn->dest_ipaddr.addr);
 394                 else
 395                         rc = sprintf(str, "%pI6\n",
 396                                 (char *)&boot_conn->dest_ipaddr.addr);
 397                 break;
 398         case ISCSI_BOOT_TGT_PORT:
 399                 rc = sprintf(str, "%d\n", boot_conn->dest_port);
 400                 break;
 401
 402         case ISCSI_BOOT_TGT_CHAP_NAME:
 403                 rc = sprintf(str,  "%.*s\n",
 404                              boot_conn->negotiated_login_options.auth_data.chap.
 405                              target_chap_name_length,
 406                              (char *)&boot_conn->negotiated_login_options.
 407                              auth_data.chap.target_chap_name);
 408                 break;
 409         case ISCSI_BOOT_TGT_CHAP_SECRET:
 410                 rc = sprintf(str,  "%.*s\n",
 411                              boot_conn->negotiated_login_options.auth_data.chap.
 412                              target_secret_length,
 413                              (char *)&boot_conn->negotiated_login_options.
 414                              auth_data.chap.target_secret);
 415                 break;
 416         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 417                 rc = sprintf(str,  "%.*s\n",
 418                              boot_conn->negotiated_login_options.auth_data.chap.
 419                              intr_chap_name_length,
 420                              (char *)&boot_conn->negotiated_login_options.
 421                              auth_data.chap.intr_chap_name);
 422                 break;
 423         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 424                 rc = sprintf(str,  "%.*s\n",
 425                              boot_conn->negotiated_login_options.auth_data.chap.
 426                              intr_secret_length,
 427                              (char *)&boot_conn->negotiated_login_options.
 428                              auth_data.chap.intr_secret);
 429                 break;
 430         case ISCSI_BOOT_TGT_FLAGS:
 431                 rc = sprintf(str, "2\n");
 432                 break;
 433         case ISCSI_BOOT_TGT_NIC_ASSOC:
 434                 rc = sprintf(str, "0\n");
 435                 break;
 436         default:
 437                 rc = -ENOSYS;
 438                 break;
 439         }
 440         return rc;
 441 }
 442
 443 static ssize_t beiscsi_show_boot_ini_info(void *data, int type, char *buf)
 444 {
 445         struct beiscsi_hba *phba = data;
 446         char *str = buf;
 447         int rc;
 448
 449         switch (type) {
 450         case ISCSI_BOOT_INI_INITIATOR_NAME:
 451                 rc = sprintf(str, "%s\n", phba->boot_sess.initiator_iscsiname);
 452                 break;
 453         default:
 454                 rc = -ENOSYS;
 455                 break;
 456         }
 457         return rc;
 458 }
 459
 460 static ssize_t beiscsi_show_boot_eth_info(void *data, int type, char *buf)
 461 {
 462         struct beiscsi_hba *phba = data;
 463         char *str = buf;
 464         int rc;
 465
 466         switch (type) {
 467         case ISCSI_BOOT_ETH_FLAGS:
 468                 rc = sprintf(str, "2\n");
 469                 break;
 470         case ISCSI_BOOT_ETH_INDEX:
 471                 rc = sprintf(str, "0\n");
 472                 break;
 473         case ISCSI_BOOT_ETH_MAC:
 474                 rc  = beiscsi_get_macaddr(str, phba);
 475                 break;
 476         default:
 477                 rc = -ENOSYS;
 478                 break;
 479         }
 480         return rc;
 481 }
 482
 483
 484 static umode_t beiscsi_tgt_get_attr_visibility(void *data, int type)
 485 {
 486         umode_t rc;
 487
 488         switch (type) {
 489         case ISCSI_BOOT_TGT_NAME:
 490         case ISCSI_BOOT_TGT_IP_ADDR:
 491         case ISCSI_BOOT_TGT_PORT:
 492         case ISCSI_BOOT_TGT_CHAP_NAME:
 493         case ISCSI_BOOT_TGT_CHAP_SECRET:
 494         case ISCSI_BOOT_TGT_REV_CHAP_NAME:
 495         case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
 496         case ISCSI_BOOT_TGT_NIC_ASSOC:
 497         case ISCSI_BOOT_TGT_FLAGS:
 498                 rc = S_IRUGO;
 499                 break;
 500         default:
 501                 rc = 0;
 502                 break;
 503         }
 504         return rc;
 505 }
 506
 507 static umode_t beiscsi_ini_get_attr_visibility(void *data, int type)
 508 {
 509         umode_t rc;
 510
 511         switch (type) {
 512         case ISCSI_BOOT_INI_INITIATOR_NAME:
 513                 rc = S_IRUGO;
 514                 break;
 515         default:
 516                 rc = 0;
 517                 break;
 518         }
 519         return rc;
 520 }
 521
 522
 523 static umode_t beiscsi_eth_get_attr_visibility(void *data, int type)
 524 {
 525         umode_t rc;
 526
 527         switch (type) {
 528         case ISCSI_BOOT_ETH_FLAGS:
 529         case ISCSI_BOOT_ETH_MAC:
 530         case ISCSI_BOOT_ETH_INDEX:
 531                 rc = S_IRUGO;
 532                 break;
 533         default:
 534                 rc = 0;
 535                 break;
 536         }
 537         return rc;
 538 }
 539
 540 /*------------------- PCI Driver operations and data ----------------- */
 541 static const struct pci_device_id beiscsi_pci_id_table[] = {
 542         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
 543         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
 544         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
 545         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
 546         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID3) },
 547         { PCI_DEVICE(ELX_VENDOR_ID, OC_SKH_ID1) },
 548         { 0 }
 549 };
 550 MODULE_DEVICE_TABLE(pci, beiscsi_pci_id_table);
 551
 552
 553 static struct scsi_host_template beiscsi_sht = {
 554         .module = THIS_MODULE,
 555         .name = "Avago Technologies 10Gbe open-iscsi Initiator Driver",
 556         .proc_name = DRV_NAME,
 557         .queuecommand = iscsi_queuecommand,
 558         .change_queue_depth = scsi_change_queue_depth,
 559         .slave_configure = beiscsi_slave_configure,
 560         .target_alloc = iscsi_target_alloc,
 561         .eh_abort_handler = beiscsi_eh_abort,
 562         .eh_device_reset_handler = beiscsi_eh_device_reset,
 563         .eh_target_reset_handler = iscsi_eh_session_reset,
 564         .shost_attrs = beiscsi_attrs,
 565         .sg_tablesize = BEISCSI_SGLIST_ELEMENTS,
 566         .can_queue = BE2_IO_DEPTH,
 567         .this_id = -1,
 568         .max_sectors = BEISCSI_MAX_SECTORS,
 569         .cmd_per_lun = BEISCSI_CMD_PER_LUN,
 570         .use_clustering = ENABLE_CLUSTERING,
 571         .vendor_id = SCSI_NL_VID_TYPE_PCI | BE_VENDOR_ID,
 572         .track_queue_depth = 1,
 573 };
 574
 575 static struct scsi_transport_template *beiscsi_scsi_transport;
 576
 577 static struct beiscsi_hba *beiscsi_hba_alloc(struct pci_dev *pcidev)
 578 {
 579         struct beiscsi_hba *phba;
 580         struct Scsi_Host *shost;
 581
 582         shost = iscsi_host_alloc(&beiscsi_sht, sizeof(*phba), 0);
 583         if (!shost) {
 584                 dev_err(&pcidev->dev,
 585                         "beiscsi_hba_alloc - iscsi_host_alloc failed\n");
 586                 return NULL;
 587         }
 588         shost->max_id = BE2_MAX_SESSIONS;
 589         shost->max_channel = 0;
 590         shost->max_cmd_len = BEISCSI_MAX_CMD_LEN;
 591         shost->max_lun = BEISCSI_NUM_MAX_LUN;
 592         shost->transportt = beiscsi_scsi_transport;
 593         phba = iscsi_host_priv(shost);
 594         memset(phba, 0, sizeof(*phba));
 595         phba->shost = shost;
 596         phba->pcidev = pci_dev_get(pcidev);
 597         pci_set_drvdata(pcidev, phba);
 598         phba->interface_handle = 0xFFFFFFFF;
 599
 600         return phba;
 601 }
 602
 603 static void beiscsi_unmap_pci_function(struct beiscsi_hba *phba)
 604 {
 605         if (phba->csr_va) {
 606                 iounmap(phba->csr_va);
 607                 phba->csr_va = NULL;
 608         }
 609         if (phba->db_va) {
 610                 iounmap(phba->db_va);
 611                 phba->db_va = NULL;
 612         }
 613         if (phba->pci_va) {
 614                 iounmap(phba->pci_va);
 615                 phba->pci_va = NULL;
 616         }
 617 }
 618
 619 static int beiscsi_map_pci_bars(struct beiscsi_hba *phba,
 620                                 struct pci_dev *pcidev)
 621 {
 622         u8 __iomem *addr;
 623         int pcicfg_reg;
 624
 625         addr = ioremap_nocache(pci_resource_start(pcidev, 2),
 626                                pci_resource_len(pcidev, 2));
 627         if (addr == NULL)
 628                 return -ENOMEM;
 629         phba->ctrl.csr = addr;
 630         phba->csr_va = addr;
 631         phba->csr_pa.u.a64.address = pci_resource_start(pcidev, 2);
 632
 633         addr = ioremap_nocache(pci_resource_start(pcidev, 4), 128 * 1024);
 634         if (addr == NULL)
 635                 goto pci_map_err;
 636         phba->ctrl.db = addr;
 637         phba->db_va = addr;
 638         phba->db_pa.u.a64.address =  pci_resource_start(pcidev, 4);
 639
 640         if (phba->generation == BE_GEN2)
 641                 pcicfg_reg = 1;
 642         else
 643                 pcicfg_reg = 0;
 644
 645         addr = ioremap_nocache(pci_resource_start(pcidev, pcicfg_reg),
 646                                pci_resource_len(pcidev, pcicfg_reg));
 647
 648         if (addr == NULL)
 649                 goto pci_map_err;
 650         phba->ctrl.pcicfg = addr;
 651         phba->pci_va = addr;
 652         phba->pci_pa.u.a64.address = pci_resource_start(pcidev, pcicfg_reg);
 653         return 0;
 654
 655 pci_map_err:
 656         beiscsi_unmap_pci_function(phba);
 657         return -ENOMEM;
 658 }
 659
 660 static int beiscsi_enable_pci(struct pci_dev *pcidev)
 661 {
 662         int ret;
 663
 664         ret = pci_enable_device(pcidev);
 665         if (ret) {
 666                 dev_err(&pcidev->dev,
 667                         "beiscsi_enable_pci - enable device failed\n");
 668                 return ret;
 669         }
 670
 671         pci_set_master(pcidev);
 672         ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64));
 673         if (ret) {
 674                 ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32));
 675                 if (ret) {
 676                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 677                         pci_disable_device(pcidev);
 678                         return ret;
 679                 } else {
 680                         ret = pci_set_consistent_dma_mask(pcidev,
 681                                                           DMA_BIT_MASK(32));
 682                 }
 683         } else {
 684                 ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64));
 685                 if (ret) {
 686                         dev_err(&pcidev->dev, "Could not set PCI DMA Mask\n");
 687                         pci_disable_device(pcidev);
 688                         return ret;
 689                 }
 690         }
 691         return 0;
 692 }
 693
 694 static int be_ctrl_init(struct beiscsi_hba *phba, struct pci_dev *pdev)
 695 {
 696         struct be_ctrl_info *ctrl = &phba->ctrl;
 697         struct be_dma_mem *mbox_mem_alloc = &ctrl->mbox_mem_alloced;
 698         struct be_dma_mem *mbox_mem_align = &ctrl->mbox_mem;
 699         int status = 0;
 700
 701         ctrl->pdev = pdev;
 702         status = beiscsi_map_pci_bars(phba, pdev);
 703         if (status)
 704                 return status;
 705         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
 706         mbox_mem_alloc->va = pci_alloc_consistent(pdev,
 707                                                   mbox_mem_alloc->size,
 708                                                   &mbox_mem_alloc->dma);
 709         if (!mbox_mem_alloc->va) {
 710                 beiscsi_unmap_pci_function(phba);
 711                 return -ENOMEM;
 712         }
 713
 714         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
 715         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
 716         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
 717         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
 718         spin_lock_init(&ctrl->mbox_lock);
 719         spin_lock_init(&phba->ctrl.mcc_lock);
 720         spin_lock_init(&phba->ctrl.mcc_cq_lock);
 721
 722         return status;
 723 }
 724
 725 /**
 726  * beiscsi_get_params()- Set the config paramters
 727  * @phba: ptr  device priv structure
 728  **/
 729 static void beiscsi_get_params(struct beiscsi_hba *phba)
 730 {
 731         uint32_t total_cid_count = 0;
 732         uint32_t total_icd_count = 0;
 733         uint8_t ulp_num = 0;
 734
 735         total_cid_count = BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP0) +
 736                           BEISCSI_GET_CID_COUNT(phba, BEISCSI_ULP1);
 737
 738         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
 739                 uint32_t align_mask = 0;
 740                 uint32_t icd_post_per_page = 0;
 741                 uint32_t icd_count_unavailable = 0;
 742                 uint32_t icd_start = 0, icd_count = 0;
 743                 uint32_t icd_start_align = 0, icd_count_align = 0;
 744
 745                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
 746                         icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
 747                         icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 748
 749                         /* Get ICD count that can be posted on each page */
 750                         icd_post_per_page = (PAGE_SIZE / (BE2_SGE *
 751                                              sizeof(struct iscsi_sge)));
 752                         align_mask = (icd_post_per_page - 1);
 753
 754                         /* Check if icd_start is aligned ICD per page posting */
 755                         if (icd_start % icd_post_per_page) {
 756                                 icd_start_align = ((icd_start +
 757                                                     icd_post_per_page) &
 758                                                     ~(align_mask));
 759                                 phba->fw_config.
 760                                         iscsi_icd_start[ulp_num] =
 761                                         icd_start_align;
 762                         }
 763
 764                         icd_count_align = (icd_count & ~align_mask);
 765
 766                         /* ICD discarded in the process of alignment */
 767                         if (icd_start_align)
 768                                 icd_count_unavailable = ((icd_start_align -
 769                                                           icd_start) +
 770                                                          (icd_count -
 771                                                           icd_count_align));
 772
 773                         /* Updated ICD count available */
 774                         phba->fw_config.iscsi_icd_count[ulp_num] = (icd_count -
 775                                         icd_count_unavailable);
 776
 777                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
 778                                         "BM_%d : Aligned ICD values\n"
 779                                         "\t ICD Start : %d\n"
 780                                         "\t ICD Count : %d\n"
 781                                         "\t ICD Discarded : %d\n",
 782                                         phba->fw_config.
 783                                         iscsi_icd_start[ulp_num],
 784                                         phba->fw_config.
 785                                         iscsi_icd_count[ulp_num],
 786                                         icd_count_unavailable);
 787                         break;
 788                 }
 789         }
 790
 791         total_icd_count = phba->fw_config.iscsi_icd_count[ulp_num];
 792         phba->params.ios_per_ctrl = (total_icd_count -
 793                                     (total_cid_count +
 794                                      BE2_TMFS + BE2_NOPOUT_REQ));
 795         phba->params.cxns_per_ctrl = total_cid_count;
 796         phba->params.asyncpdus_per_ctrl = total_cid_count;
 797         phba->params.icds_per_ctrl = total_icd_count;
 798         phba->params.num_sge_per_io = BE2_SGE;
 799         phba->params.defpdu_hdr_sz = BE2_DEFPDU_HDR_SZ;
 800         phba->params.defpdu_data_sz = BE2_DEFPDU_DATA_SZ;
 801         phba->params.eq_timer = 64;
 802         phba->params.num_eq_entries = 1024;
 803         phba->params.num_cq_entries = 1024;
 804         phba->params.wrbs_per_cxn = 256;
 805 }
 806
 807 static void hwi_ring_eq_db(struct beiscsi_hba *phba,
 808                            unsigned int id, unsigned int clr_interrupt,
 809                            unsigned int num_processed,
 810                            unsigned char rearm, unsigned char event)
 811 {
 812         u32 val = 0;
 813
 814         if (rearm)
 815                 val |= 1 << DB_EQ_REARM_SHIFT;
 816         if (clr_interrupt)
 817                 val |= 1 << DB_EQ_CLR_SHIFT;
 818         if (event)
 819                 val |= 1 << DB_EQ_EVNT_SHIFT;
 820
 821         val |= num_processed << DB_EQ_NUM_POPPED_SHIFT;
 822         /* Setting lower order EQ_ID Bits */
 823         val |= (id & DB_EQ_RING_ID_LOW_MASK);
 824
 825         /* Setting Higher order EQ_ID Bits */
 826         val |= (((id >> DB_EQ_HIGH_FEILD_SHIFT) &
 827                   DB_EQ_RING_ID_HIGH_MASK)
 828                   << DB_EQ_HIGH_SET_SHIFT);
 829
 830         iowrite32(val, phba->db_va + DB_EQ_OFFSET);
 831 }
 832
 833 /**
 834  * be_isr_mcc - The isr routine of the driver.
 835  * @irq: Not used
 836  * @dev_id: Pointer to host adapter structure
 837  */
 838 static irqreturn_t be_isr_mcc(int irq, void *dev_id)
 839 {
 840         struct beiscsi_hba *phba;
 841         struct be_eq_entry *eqe = NULL;
 842         struct be_queue_info *eq;
 843         struct be_queue_info *mcc;
 844         unsigned int num_eq_processed;
 845         struct be_eq_obj *pbe_eq;
 846         unsigned long flags;
 847
 848         pbe_eq = dev_id;
 849         eq = &pbe_eq->q;
 850         phba =  pbe_eq->phba;
 851         mcc = &phba->ctrl.mcc_obj.cq;
 852         eqe = queue_tail_node(eq);
 853
 854         num_eq_processed = 0;
 855
 856         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 857                                 & EQE_VALID_MASK) {
 858                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 859                      resource_id) / 32] &
 860                      EQE_RESID_MASK) >> 16) == mcc->id) {
 861                         spin_lock_irqsave(&phba->isr_lock, flags);
 862                         pbe_eq->todo_mcc_cq = true;
 863                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 864                 }
 865                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 866                 queue_tail_inc(eq);
 867                 eqe = queue_tail_node(eq);
 868                 num_eq_processed++;
 869         }
 870         if (pbe_eq->todo_mcc_cq)
 871                 queue_work(phba->wq, &pbe_eq->work_cqs);
 872         if (num_eq_processed)
 873                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 1, 1);
 874
 875         return IRQ_HANDLED;
 876 }
 877
 878 /**
 879  * be_isr_msix - The isr routine of the driver.
 880  * @irq: Not used
 881  * @dev_id: Pointer to host adapter structure
 882  */
 883 static irqreturn_t be_isr_msix(int irq, void *dev_id)
 884 {
 885         struct beiscsi_hba *phba;
 886         struct be_eq_entry *eqe = NULL;
 887         struct be_queue_info *eq;
 888         struct be_queue_info *cq;
 889         unsigned int num_eq_processed;
 890         struct be_eq_obj *pbe_eq;
 891
 892         pbe_eq = dev_id;
 893         eq = &pbe_eq->q;
 894         cq = pbe_eq->cq;
 895         eqe = queue_tail_node(eq);
 896
 897         phba = pbe_eq->phba;
 898         num_eq_processed = 0;
 899         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 900                                 & EQE_VALID_MASK) {
 901                 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 902                         blk_iopoll_sched(&pbe_eq->iopoll);
 903
 904                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 905                 queue_tail_inc(eq);
 906                 eqe = queue_tail_node(eq);
 907                 num_eq_processed++;
 908         }
 909
 910         if (num_eq_processed)
 911                 hwi_ring_eq_db(phba, eq->id, 1, num_eq_processed, 0, 1);
 912
 913         return IRQ_HANDLED;
 914 }
 915
 916 /**
 917  * be_isr - The isr routine of the driver.
 918  * @irq: Not used
 919  * @dev_id: Pointer to host adapter structure
 920  */
 921 static irqreturn_t be_isr(int irq, void *dev_id)
 922 {
 923         struct beiscsi_hba *phba;
 924         struct hwi_controller *phwi_ctrlr;
 925         struct hwi_context_memory *phwi_context;
 926         struct be_eq_entry *eqe = NULL;
 927         struct be_queue_info *eq;
 928         struct be_queue_info *mcc;
 929         unsigned long flags, index;
 930         unsigned int num_mcceq_processed, num_ioeq_processed;
 931         struct be_ctrl_info *ctrl;
 932         struct be_eq_obj *pbe_eq;
 933         int isr;
 934
 935         phba = dev_id;
 936         ctrl = &phba->ctrl;
 937         isr = ioread32(ctrl->csr + CEV_ISR0_OFFSET +
 938                        (PCI_FUNC(ctrl->pdev->devfn) * CEV_ISR_SIZE));
 939         if (!isr)
 940                 return IRQ_NONE;
 941
 942         phwi_ctrlr = phba->phwi_ctrlr;
 943         phwi_context = phwi_ctrlr->phwi_ctxt;
 944         pbe_eq = &phwi_context->be_eq[0];
 945
 946         eq = &phwi_context->be_eq[0].q;
 947         mcc = &phba->ctrl.mcc_obj.cq;
 948         index = 0;
 949         eqe = queue_tail_node(eq);
 950
 951         num_ioeq_processed = 0;
 952         num_mcceq_processed = 0;
 953         while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
 954                                 & EQE_VALID_MASK) {
 955                 if (((eqe->dw[offsetof(struct amap_eq_entry,
 956                      resource_id) / 32] &
 957                      EQE_RESID_MASK) >> 16) == mcc->id) {
 958                         spin_lock_irqsave(&phba->isr_lock, flags);
 959                         pbe_eq->todo_mcc_cq = true;
 960                         spin_unlock_irqrestore(&phba->isr_lock, flags);
 961                         num_mcceq_processed++;
 962                 } else {
 963                         if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
 964                                 blk_iopoll_sched(&pbe_eq->iopoll);
 965                         num_ioeq_processed++;
 966                 }
 967                 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
 968                 queue_tail_inc(eq);
 969                 eqe = queue_tail_node(eq);
 970         }
 971         if (num_ioeq_processed || num_mcceq_processed) {
 972                 if (pbe_eq->todo_mcc_cq)
 973                         queue_work(phba->wq, &pbe_eq->work_cqs);
 974
 975                 if ((num_mcceq_processed) && (!num_ioeq_processed))
 976                         hwi_ring_eq_db(phba, eq->id, 0,
 977                                       (num_ioeq_processed +
 978                                        num_mcceq_processed) , 1, 1);
 979                 else
 980                         hwi_ring_eq_db(phba, eq->id, 0,
 981                                        (num_ioeq_processed +
 982                                         num_mcceq_processed), 0, 1);
 983
 984                 return IRQ_HANDLED;
 985         } else
 986                 return IRQ_NONE;
 987 }
 988
 989 static int beiscsi_init_irqs(struct beiscsi_hba *phba)
 990 {
 991         struct pci_dev *pcidev = phba->pcidev;
 992         struct hwi_controller *phwi_ctrlr;
 993         struct hwi_context_memory *phwi_context;
 994         int ret, msix_vec, i, j;
 995
 996         phwi_ctrlr = phba->phwi_ctrlr;
 997         phwi_context = phwi_ctrlr->phwi_ctxt;
 998
 999         if (phba->msix_enabled) {
1000                 for (i = 0; i < phba->num_cpus; i++) {
1001                         phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME,
1002                                                     GFP_KERNEL);
1003                         if (!phba->msi_name[i]) {
1004                                 ret = -ENOMEM;
1005                                 goto free_msix_irqs;
1006                         }
1007
1008                         sprintf(phba->msi_name[i], "beiscsi_%02x_%02x",
1009                                 phba->shost->host_no, i);
1010                         msix_vec = phba->msix_entries[i].vector;
1011                         ret = request_irq(msix_vec, be_isr_msix, 0,
1012                                           phba->msi_name[i],
1013                                           &phwi_context->be_eq[i]);
1014                         if (ret) {
1015                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1016                                             "BM_%d : beiscsi_init_irqs-Failed to"
1017                                             "register msix for i = %d\n",
1018                                             i);
1019                                 kfree(phba->msi_name[i]);
1020                                 goto free_msix_irqs;
1021                         }
1022                 }
1023                 phba->msi_name[i] = kzalloc(BEISCSI_MSI_NAME, GFP_KERNEL);
1024                 if (!phba->msi_name[i]) {
1025                         ret = -ENOMEM;
1026                         goto free_msix_irqs;
1027                 }
1028                 sprintf(phba->msi_name[i], "beiscsi_mcc_%02x",
1029                         phba->shost->host_no);
1030                 msix_vec = phba->msix_entries[i].vector;
1031                 ret = request_irq(msix_vec, be_isr_mcc, 0, phba->msi_name[i],
1032                                   &phwi_context->be_eq[i]);
1033                 if (ret) {
1034                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT ,
1035                                     "BM_%d : beiscsi_init_irqs-"
1036                                     "Failed to register beiscsi_msix_mcc\n");
1037                         kfree(phba->msi_name[i]);
1038                         goto free_msix_irqs;
1039                 }
1040
1041         } else {
1042                 ret = request_irq(pcidev->irq, be_isr, IRQF_SHARED,
1043                                   "beiscsi", phba);
1044                 if (ret) {
1045                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
1046                                     "BM_%d : beiscsi_init_irqs-"
1047                                     "Failed to register irq\\n");
1048                         return ret;
1049                 }
1050         }
1051         return 0;
1052 free_msix_irqs:
1053         for (j = i - 1; j >= 0; j--) {
1054                 kfree(phba->msi_name[j]);
1055                 msix_vec = phba->msix_entries[j].vector;
1056                 free_irq(msix_vec, &phwi_context->be_eq[j]);
1057         }
1058         return ret;
1059 }
1060
1061 void hwi_ring_cq_db(struct beiscsi_hba *phba,
1062                            unsigned int id, unsigned int num_processed,
1063                            unsigned char rearm, unsigned char event)
1064 {
1065         u32 val = 0;
1066
1067         if (rearm)
1068                 val |= 1 << DB_CQ_REARM_SHIFT;
1069
1070         val |= num_processed << DB_CQ_NUM_POPPED_SHIFT;
1071
1072         /* Setting lower order CQ_ID Bits */
1073         val |= (id & DB_CQ_RING_ID_LOW_MASK);
1074
1075         /* Setting Higher order CQ_ID Bits */
1076         val |= (((id >> DB_CQ_HIGH_FEILD_SHIFT) &
1077                   DB_CQ_RING_ID_HIGH_MASK)
1078                   << DB_CQ_HIGH_SET_SHIFT);
1079
1080         iowrite32(val, phba->db_va + DB_CQ_OFFSET);
1081 }
1082
1083 static unsigned int
1084 beiscsi_process_async_pdu(struct beiscsi_conn *beiscsi_conn,
1085                           struct beiscsi_hba *phba,
1086                           struct pdu_base *ppdu,
1087                           unsigned long pdu_len,
1088                           void *pbuffer, unsigned long buf_len)
1089 {
1090         struct iscsi_conn *conn = beiscsi_conn->conn;
1091         struct iscsi_session *session = conn->session;
1092         struct iscsi_task *task;
1093         struct beiscsi_io_task *io_task;
1094         struct iscsi_hdr *login_hdr;
1095
1096         switch (ppdu->dw[offsetof(struct amap_pdu_base, opcode) / 32] &
1097                                                 PDUBASE_OPCODE_MASK) {
1098         case ISCSI_OP_NOOP_IN:
1099                 pbuffer = NULL;
1100                 buf_len = 0;
1101                 break;
1102         case ISCSI_OP_ASYNC_EVENT:
1103                 break;
1104         case ISCSI_OP_REJECT:
1105                 WARN_ON(!pbuffer);
1106                 WARN_ON(!(buf_len == 48));
1107                 beiscsi_log(phba, KERN_ERR,
1108                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1109                             "BM_%d : In ISCSI_OP_REJECT\n");
1110                 break;
1111         case ISCSI_OP_LOGIN_RSP:
1112         case ISCSI_OP_TEXT_RSP:
1113                 task = conn->login_task;
1114                 io_task = task->dd_data;
1115                 login_hdr = (struct iscsi_hdr *)ppdu;
1116                 login_hdr->itt = io_task->libiscsi_itt;
1117                 break;
1118         default:
1119                 beiscsi_log(phba, KERN_WARNING,
1120                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1121                             "BM_%d : Unrecognized opcode 0x%x in async msg\n",
1122                             (ppdu->
1123                              dw[offsetof(struct amap_pdu_base, opcode) / 32]
1124                              & PDUBASE_OPCODE_MASK));
1125                 return 1;
1126         }
1127
1128         spin_lock_bh(&session->back_lock);
1129         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)ppdu, pbuffer, buf_len);
1130         spin_unlock_bh(&session->back_lock);
1131         return 0;
1132 }
1133
1134 static struct sgl_handle *alloc_io_sgl_handle(struct beiscsi_hba *phba)
1135 {
1136         struct sgl_handle *psgl_handle;
1137
1138         if (phba->io_sgl_hndl_avbl) {
1139                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1140                             "BM_%d : In alloc_io_sgl_handle,"
1141                             " io_sgl_alloc_index=%d\n",
1142                             phba->io_sgl_alloc_index);
1143
1144                 psgl_handle = phba->io_sgl_hndl_base[phba->
1145                                                 io_sgl_alloc_index];
1146                 phba->io_sgl_hndl_base[phba->io_sgl_alloc_index] = NULL;
1147                 phba->io_sgl_hndl_avbl--;
1148                 if (phba->io_sgl_alloc_index == (phba->params.
1149                                                  ios_per_ctrl - 1))
1150                         phba->io_sgl_alloc_index = 0;
1151                 else
1152                         phba->io_sgl_alloc_index++;
1153         } else
1154                 psgl_handle = NULL;
1155         return psgl_handle;
1156 }
1157
1158 static void
1159 free_io_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1160 {
1161         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1162                     "BM_%d : In free_,io_sgl_free_index=%d\n",
1163                     phba->io_sgl_free_index);
1164
1165         if (phba->io_sgl_hndl_base[phba->io_sgl_free_index]) {
1166                 /*
1167                  * this can happen if clean_task is called on a task that
1168                  * failed in xmit_task or alloc_pdu.
1169                  */
1170                  beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_IO,
1171                              "BM_%d : Double Free in IO SGL io_sgl_free_index=%d,"
1172                              "value there=%p\n", phba->io_sgl_free_index,
1173                              phba->io_sgl_hndl_base
1174                              [phba->io_sgl_free_index]);
1175                 return;
1176         }
1177         phba->io_sgl_hndl_base[phba->io_sgl_free_index] = psgl_handle;
1178         phba->io_sgl_hndl_avbl++;
1179         if (phba->io_sgl_free_index == (phba->params.ios_per_ctrl - 1))
1180                 phba->io_sgl_free_index = 0;
1181         else
1182                 phba->io_sgl_free_index++;
1183 }
1184
1185 /**
1186  * alloc_wrb_handle - To allocate a wrb handle
1187  * @phba: The hba pointer
1188  * @cid: The cid to use for allocation
1189  *
1190  * This happens under session_lock until submission to chip
1191  */
1192 struct wrb_handle *alloc_wrb_handle(struct beiscsi_hba *phba, unsigned int cid)
1193 {
1194         struct hwi_wrb_context *pwrb_context;
1195         struct hwi_controller *phwi_ctrlr;
1196         struct wrb_handle *pwrb_handle, *pwrb_handle_tmp;
1197         uint16_t cri_index = BE_GET_CRI_FROM_CID(cid);
1198
1199         phwi_ctrlr = phba->phwi_ctrlr;
1200         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1201         if (pwrb_context->wrb_handles_available >= 2) {
1202                 pwrb_handle = pwrb_context->pwrb_handle_base[
1203                                             pwrb_context->alloc_index];
1204                 pwrb_context->wrb_handles_available--;
1205                 if (pwrb_context->alloc_index ==
1206                                                 (phba->params.wrbs_per_cxn - 1))
1207                         pwrb_context->alloc_index = 0;
1208                 else
1209                         pwrb_context->alloc_index++;
1210                 pwrb_handle_tmp = pwrb_context->pwrb_handle_base[
1211                                                 pwrb_context->alloc_index];
1212                 pwrb_handle->nxt_wrb_index = pwrb_handle_tmp->wrb_index;
1213         } else
1214                 pwrb_handle = NULL;
1215         return pwrb_handle;
1216 }
1217
1218 /**
1219  * free_wrb_handle - To free the wrb handle back to pool
1220  * @phba: The hba pointer
1221  * @pwrb_context: The context to free from
1222  * @pwrb_handle: The wrb_handle to free
1223  *
1224  * This happens under session_lock until submission to chip
1225  */
1226 static void
1227 free_wrb_handle(struct beiscsi_hba *phba, struct hwi_wrb_context *pwrb_context,
1228                 struct wrb_handle *pwrb_handle)
1229 {
1230         pwrb_context->pwrb_handle_base[pwrb_context->free_index] = pwrb_handle;
1231         pwrb_context->wrb_handles_available++;
1232         if (pwrb_context->free_index == (phba->params.wrbs_per_cxn - 1))
1233                 pwrb_context->free_index = 0;
1234         else
1235                 pwrb_context->free_index++;
1236
1237         beiscsi_log(phba, KERN_INFO,
1238                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1239                     "BM_%d : FREE WRB: pwrb_handle=%p free_index=0x%x"
1240                     "wrb_handles_available=%d\n",
1241                     pwrb_handle, pwrb_context->free_index,
1242                     pwrb_context->wrb_handles_available);
1243 }
1244
1245 static struct sgl_handle *alloc_mgmt_sgl_handle(struct beiscsi_hba *phba)
1246 {
1247         struct sgl_handle *psgl_handle;
1248
1249         if (phba->eh_sgl_hndl_avbl) {
1250                 psgl_handle = phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index];
1251                 phba->eh_sgl_hndl_base[phba->eh_sgl_alloc_index] = NULL;
1252                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1253                             "BM_%d : mgmt_sgl_alloc_index=%d=0x%x\n",
1254                             phba->eh_sgl_alloc_index,
1255                             phba->eh_sgl_alloc_index);
1256
1257                 phba->eh_sgl_hndl_avbl--;
1258                 if (phba->eh_sgl_alloc_index ==
1259                     (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl -
1260                      1))
1261                         phba->eh_sgl_alloc_index = 0;
1262                 else
1263                         phba->eh_sgl_alloc_index++;
1264         } else
1265                 psgl_handle = NULL;
1266         return psgl_handle;
1267 }
1268
1269 void
1270 free_mgmt_sgl_handle(struct beiscsi_hba *phba, struct sgl_handle *psgl_handle)
1271 {
1272
1273         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_CONFIG,
1274                     "BM_%d : In  free_mgmt_sgl_handle,"
1275                     "eh_sgl_free_index=%d\n",
1276                     phba->eh_sgl_free_index);
1277
1278         if (phba->eh_sgl_hndl_base[phba->eh_sgl_free_index]) {
1279                 /*
1280                  * this can happen if clean_task is called on a task that
1281                  * failed in xmit_task or alloc_pdu.
1282                  */
1283                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_CONFIG,
1284                             "BM_%d : Double Free in eh SGL ,"
1285                             "eh_sgl_free_index=%d\n",
1286                             phba->eh_sgl_free_index);
1287                 return;
1288         }
1289         phba->eh_sgl_hndl_base[phba->eh_sgl_free_index] = psgl_handle;
1290         phba->eh_sgl_hndl_avbl++;
1291         if (phba->eh_sgl_free_index ==
1292             (phba->params.icds_per_ctrl - phba->params.ios_per_ctrl - 1))
1293                 phba->eh_sgl_free_index = 0;
1294         else
1295                 phba->eh_sgl_free_index++;
1296 }
1297
1298 static void
1299 be_complete_io(struct beiscsi_conn *beiscsi_conn,
1300                 struct iscsi_task *task,
1301                 struct common_sol_cqe *csol_cqe)
1302 {
1303         struct beiscsi_io_task *io_task = task->dd_data;
1304         struct be_status_bhs *sts_bhs =
1305                                 (struct be_status_bhs *)io_task->cmd_bhs;
1306         struct iscsi_conn *conn = beiscsi_conn->conn;
1307         unsigned char *sense;
1308         u32 resid = 0, exp_cmdsn, max_cmdsn;
1309         u8 rsp, status, flags;
1310
1311         exp_cmdsn = csol_cqe->exp_cmdsn;
1312         max_cmdsn = (csol_cqe->exp_cmdsn +
1313                      csol_cqe->cmd_wnd - 1);
1314         rsp = csol_cqe->i_resp;
1315         status = csol_cqe->i_sts;
1316         flags = csol_cqe->i_flags;
1317         resid = csol_cqe->res_cnt;
1318
1319         if (!task->sc) {
1320                 if (io_task->scsi_cmnd) {
1321                         scsi_dma_unmap(io_task->scsi_cmnd);
1322                         io_task->scsi_cmnd = NULL;
1323                 }
1324
1325                 return;
1326         }
1327         task->sc->result = (DID_OK << 16) | status;
1328         if (rsp != ISCSI_STATUS_CMD_COMPLETED) {
1329                 task->sc->result = DID_ERROR << 16;
1330                 goto unmap;
1331         }
1332
1333         /* bidi not initially supported */
1334         if (flags & (ISCSI_FLAG_CMD_UNDERFLOW | ISCSI_FLAG_CMD_OVERFLOW)) {
1335                 if (!status && (flags & ISCSI_FLAG_CMD_OVERFLOW))
1336                         task->sc->result = DID_ERROR << 16;
1337
1338                 if (flags & ISCSI_FLAG_CMD_UNDERFLOW) {
1339                         scsi_set_resid(task->sc, resid);
1340                         if (!status && (scsi_bufflen(task->sc) - resid <
1341                             task->sc->underflow))
1342                                 task->sc->result = DID_ERROR << 16;
1343                 }
1344         }
1345
1346         if (status == SAM_STAT_CHECK_CONDITION) {
1347                 u16 sense_len;
1348                 unsigned short *slen = (unsigned short *)sts_bhs->sense_info;
1349
1350                 sense = sts_bhs->sense_info + sizeof(unsigned short);
1351                 sense_len = be16_to_cpu(*slen);
1352                 memcpy(task->sc->sense_buffer, sense,
1353                        min_t(u16, sense_len, SCSI_SENSE_BUFFERSIZE));
1354         }
1355
1356         if (io_task->cmd_bhs->iscsi_hdr.flags & ISCSI_FLAG_CMD_READ)
1357                 conn->rxdata_octets += resid;
1358 unmap:
1359         scsi_dma_unmap(io_task->scsi_cmnd);
1360         io_task->scsi_cmnd = NULL;
1361         iscsi_complete_scsi_task(task, exp_cmdsn, max_cmdsn);
1362 }
1363
1364 static void
1365 be_complete_logout(struct beiscsi_conn *beiscsi_conn,
1366                     struct iscsi_task *task,
1367                     struct common_sol_cqe *csol_cqe)
1368 {
1369         struct iscsi_logout_rsp *hdr;
1370         struct beiscsi_io_task *io_task = task->dd_data;
1371         struct iscsi_conn *conn = beiscsi_conn->conn;
1372
1373         hdr = (struct iscsi_logout_rsp *)task->hdr;
1374         hdr->opcode = ISCSI_OP_LOGOUT_RSP;
1375         hdr->t2wait = 5;
1376         hdr->t2retain = 0;
1377         hdr->flags = csol_cqe->i_flags;
1378         hdr->response = csol_cqe->i_resp;
1379         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1380         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1381                                      csol_cqe->cmd_wnd - 1);
1382
1383         hdr->dlength[0] = 0;
1384         hdr->dlength[1] = 0;
1385         hdr->dlength[2] = 0;
1386         hdr->hlength = 0;
1387         hdr->itt = io_task->libiscsi_itt;
1388         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1389 }
1390
1391 static void
1392 be_complete_tmf(struct beiscsi_conn *beiscsi_conn,
1393                  struct iscsi_task *task,
1394                  struct common_sol_cqe *csol_cqe)
1395 {
1396         struct iscsi_tm_rsp *hdr;
1397         struct iscsi_conn *conn = beiscsi_conn->conn;
1398         struct beiscsi_io_task *io_task = task->dd_data;
1399
1400         hdr = (struct iscsi_tm_rsp *)task->hdr;
1401         hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP;
1402         hdr->flags = csol_cqe->i_flags;
1403         hdr->response = csol_cqe->i_resp;
1404         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1405         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1406                                      csol_cqe->cmd_wnd - 1);
1407
1408         hdr->itt = io_task->libiscsi_itt;
1409         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1410 }
1411
1412 static void
1413 hwi_complete_drvr_msgs(struct beiscsi_conn *beiscsi_conn,
1414                        struct beiscsi_hba *phba, struct sol_cqe *psol)
1415 {
1416         struct hwi_wrb_context *pwrb_context;
1417         struct wrb_handle *pwrb_handle = NULL;
1418         struct hwi_controller *phwi_ctrlr;
1419         struct iscsi_task *task;
1420         struct beiscsi_io_task *io_task;
1421         uint16_t wrb_index, cid, cri_index;
1422
1423         phwi_ctrlr = phba->phwi_ctrlr;
1424         if (is_chip_be2_be3r(phba)) {
1425                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1426                                           wrb_idx, psol);
1427                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe,
1428                                     cid, psol);
1429         } else {
1430                 wrb_index = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1431                                           wrb_idx, psol);
1432                 cid = AMAP_GET_BITS(struct amap_it_dmsg_cqe_v2,
1433                                     cid, psol);
1434         }
1435
1436         cri_index = BE_GET_CRI_FROM_CID(cid);
1437         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1438         pwrb_handle = pwrb_context->pwrb_handle_basestd[wrb_index];
1439         task = pwrb_handle->pio_handle;
1440
1441         io_task = task->dd_data;
1442         memset(io_task->pwrb_handle->pwrb, 0, sizeof(struct iscsi_wrb));
1443         iscsi_put_task(task);
1444 }
1445
1446 static void
1447 be_complete_nopin_resp(struct beiscsi_conn *beiscsi_conn,
1448                         struct iscsi_task *task,
1449                         struct common_sol_cqe *csol_cqe)
1450 {
1451         struct iscsi_nopin *hdr;
1452         struct iscsi_conn *conn = beiscsi_conn->conn;
1453         struct beiscsi_io_task *io_task = task->dd_data;
1454
1455         hdr = (struct iscsi_nopin *)task->hdr;
1456         hdr->flags = csol_cqe->i_flags;
1457         hdr->exp_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn);
1458         hdr->max_cmdsn = cpu_to_be32(csol_cqe->exp_cmdsn +
1459                                      csol_cqe->cmd_wnd - 1);
1460
1461         hdr->opcode = ISCSI_OP_NOOP_IN;
1462         hdr->itt = io_task->libiscsi_itt;
1463         __iscsi_complete_pdu(conn, (struct iscsi_hdr *)hdr, NULL, 0);
1464 }
1465
1466 static void adapter_get_sol_cqe(struct beiscsi_hba *phba,
1467                 struct sol_cqe *psol,
1468                 struct common_sol_cqe *csol_cqe)
1469 {
1470         if (is_chip_be2_be3r(phba)) {
1471                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe,
1472                                                     i_exp_cmd_sn, psol);
1473                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe,
1474                                                   i_res_cnt, psol);
1475                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe,
1476                                                   i_cmd_wnd, psol);
1477                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe,
1478                                                     wrb_index, psol);
1479                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe,
1480                                               cid, psol);
1481                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1482                                                  hw_sts, psol);
1483                 csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe,
1484                                                  i_resp, psol);
1485                 csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe,
1486                                                 i_sts, psol);
1487                 csol_cqe->i_flags = AMAP_GET_BITS(struct amap_sol_cqe,
1488                                                   i_flags, psol);
1489         } else {
1490                 csol_cqe->exp_cmdsn = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1491                                                     i_exp_cmd_sn, psol);
1492                 csol_cqe->res_cnt = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1493                                                   i_res_cnt, psol);
1494                 csol_cqe->wrb_index = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1495                                                     wrb_index, psol);
1496                 csol_cqe->cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1497                                               cid, psol);
1498                 csol_cqe->hw_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1499                                                  hw_sts, psol);
1500                 csol_cqe->cmd_wnd = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1501                                                   i_cmd_wnd, psol);
1502                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1503                                   cmd_cmpl, psol))
1504                         csol_cqe->i_sts = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1505                                                         i_sts, psol);
1506                 else
1507                         csol_cqe->i_resp = AMAP_GET_BITS(struct amap_sol_cqe_v2,
1508                                                          i_sts, psol);
1509                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1510                                   u, psol))
1511                         csol_cqe->i_flags = ISCSI_FLAG_CMD_UNDERFLOW;
1512
1513                 if (AMAP_GET_BITS(struct amap_sol_cqe_v2,
1514                                   o, psol))
1515                         csol_cqe->i_flags |= ISCSI_FLAG_CMD_OVERFLOW;
1516         }
1517 }
1518
1519
1520 static void hwi_complete_cmd(struct beiscsi_conn *beiscsi_conn,
1521                              struct beiscsi_hba *phba, struct sol_cqe *psol)
1522 {
1523         struct hwi_wrb_context *pwrb_context;
1524         struct wrb_handle *pwrb_handle;
1525         struct iscsi_wrb *pwrb = NULL;
1526         struct hwi_controller *phwi_ctrlr;
1527         struct iscsi_task *task;
1528         unsigned int type;
1529         struct iscsi_conn *conn = beiscsi_conn->conn;
1530         struct iscsi_session *session = conn->session;
1531         struct common_sol_cqe csol_cqe = {0};
1532         uint16_t cri_index = 0;
1533
1534         phwi_ctrlr = phba->phwi_ctrlr;
1535
1536         /* Copy the elements to a common structure */
1537         adapter_get_sol_cqe(phba, psol, &csol_cqe);
1538
1539         cri_index = BE_GET_CRI_FROM_CID(csol_cqe.cid);
1540         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
1541
1542         pwrb_handle = pwrb_context->pwrb_handle_basestd[
1543                       csol_cqe.wrb_index];
1544
1545         task = pwrb_handle->pio_handle;
1546         pwrb = pwrb_handle->pwrb;
1547         type = ((struct beiscsi_io_task *)task->dd_data)->wrb_type;
1548
1549         spin_lock_bh(&session->back_lock);
1550         switch (type) {
1551         case HWH_TYPE_IO:
1552         case HWH_TYPE_IO_RD:
1553                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) ==
1554                      ISCSI_OP_NOOP_OUT)
1555                         be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1556                 else
1557                         be_complete_io(beiscsi_conn, task, &csol_cqe);
1558                 break;
1559
1560         case HWH_TYPE_LOGOUT:
1561                 if ((task->hdr->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGOUT)
1562                         be_complete_logout(beiscsi_conn, task, &csol_cqe);
1563                 else
1564                         be_complete_tmf(beiscsi_conn, task, &csol_cqe);
1565                 break;
1566
1567         case HWH_TYPE_LOGIN:
1568                 beiscsi_log(phba, KERN_ERR,
1569                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1570                             "BM_%d :\t\t No HWH_TYPE_LOGIN Expected in"
1571                             " hwi_complete_cmd- Solicited path\n");
1572                 break;
1573
1574         case HWH_TYPE_NOP:
1575                 be_complete_nopin_resp(beiscsi_conn, task, &csol_cqe);
1576                 break;
1577
1578         default:
1579                 beiscsi_log(phba, KERN_WARNING,
1580                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1581                             "BM_%d : In hwi_complete_cmd, unknown type = %d"
1582                             "wrb_index 0x%x CID 0x%x\n", type,
1583                             csol_cqe.wrb_index,
1584                             csol_cqe.cid);
1585                 break;
1586         }
1587
1588         spin_unlock_bh(&session->back_lock);
1589 }
1590
1591 static struct list_head *hwi_get_async_busy_list(struct hwi_async_pdu_context
1592                                           *pasync_ctx, unsigned int is_header,
1593                                           unsigned int host_write_ptr)
1594 {
1595         if (is_header)
1596                 return &pasync_ctx->async_entry[host_write_ptr].
1597                     header_busy_list;
1598         else
1599                 return &pasync_ctx->async_entry[host_write_ptr].data_busy_list;
1600 }
1601
1602 static struct async_pdu_handle *
1603 hwi_get_async_handle(struct beiscsi_hba *phba,
1604                      struct beiscsi_conn *beiscsi_conn,
1605                      struct hwi_async_pdu_context *pasync_ctx,
1606                      struct i_t_dpdu_cqe *pdpdu_cqe, unsigned int *pcq_index)
1607 {
1608         struct be_bus_address phys_addr;
1609         struct list_head *pbusy_list;
1610         struct async_pdu_handle *pasync_handle = NULL;
1611         unsigned char is_header = 0;
1612         unsigned int index, dpl;
1613
1614         if (is_chip_be2_be3r(phba)) {
1615                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1616                                     dpl, pdpdu_cqe);
1617                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe,
1618                                       index, pdpdu_cqe);
1619         } else {
1620                 dpl = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1621                                     dpl, pdpdu_cqe);
1622                 index = AMAP_GET_BITS(struct amap_i_t_dpdu_cqe_v2,
1623                                       index, pdpdu_cqe);
1624         }
1625
1626         phys_addr.u.a32.address_lo =
1627                 (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1628                                         db_addr_lo) / 32] - dpl);
1629         phys_addr.u.a32.address_hi =
1630                 pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1631                                        db_addr_hi) / 32];
1632
1633         phys_addr.u.a64.address =
1634                         *((unsigned long long *)(&phys_addr.u.a64.address));
1635
1636         switch (pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe, code) / 32]
1637                         & PDUCQE_CODE_MASK) {
1638         case UNSOL_HDR_NOTIFY:
1639                 is_header = 1;
1640
1641                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1642                                                       is_header, index);
1643                 break;
1644         case UNSOL_DATA_NOTIFY:
1645                  pbusy_list = hwi_get_async_busy_list(pasync_ctx,
1646                                                       is_header, index);
1647                 break;
1648         default:
1649                 pbusy_list = NULL;
1650                 beiscsi_log(phba, KERN_WARNING,
1651                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
1652                             "BM_%d : Unexpected code=%d\n",
1653                             pdpdu_cqe->dw[offsetof(struct amap_i_t_dpdu_cqe,
1654                             code) / 32] & PDUCQE_CODE_MASK);
1655                 return NULL;
1656         }
1657
1658         WARN_ON(list_empty(pbusy_list));
1659         list_for_each_entry(pasync_handle, pbusy_list, link) {
1660                 if (pasync_handle->pa.u.a64.address == phys_addr.u.a64.address)
1661                         break;
1662         }
1663
1664         WARN_ON(!pasync_handle);
1665
1666         pasync_handle->cri = BE_GET_ASYNC_CRI_FROM_CID(
1667                              beiscsi_conn->beiscsi_conn_cid);
1668         pasync_handle->is_header = is_header;
1669         pasync_handle->buffer_len = dpl;
1670         *pcq_index = index;
1671
1672         return pasync_handle;
1673 }
1674
1675 static unsigned int
1676 hwi_update_async_writables(struct beiscsi_hba *phba,
1677                             struct hwi_async_pdu_context *pasync_ctx,
1678                             unsigned int is_header, unsigned int cq_index)
1679 {
1680         struct list_head *pbusy_list;
1681         struct async_pdu_handle *pasync_handle;
1682         unsigned int num_entries, writables = 0;
1683         unsigned int *pep_read_ptr, *pwritables;
1684
1685         num_entries = pasync_ctx->num_entries;
1686         if (is_header) {
1687                 pep_read_ptr = &pasync_ctx->async_header.ep_read_ptr;
1688                 pwritables = &pasync_ctx->async_header.writables;
1689         } else {
1690                 pep_read_ptr = &pasync_ctx->async_data.ep_read_ptr;
1691                 pwritables = &pasync_ctx->async_data.writables;
1692         }
1693
1694         while ((*pep_read_ptr) != cq_index) {
1695                 (*pep_read_ptr)++;
1696                 *pep_read_ptr = (*pep_read_ptr) % num_entries;
1697
1698                 pbusy_list = hwi_get_async_busy_list(pasync_ctx, is_header,
1699                                                      *pep_read_ptr);
1700                 if (writables == 0)
1701                         WARN_ON(list_empty(pbusy_list));
1702
1703                 if (!list_empty(pbusy_list)) {
1704                         pasync_handle = list_entry(pbusy_list->next,
1705                                                    struct async_pdu_handle,
1706                                                    link);
1707                         WARN_ON(!pasync_handle);
1708                         pasync_handle->consumed = 1;
1709                 }
1710
1711                 writables++;
1712         }
1713
1714         if (!writables) {
1715                 beiscsi_log(phba, KERN_ERR,
1716                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
1717                             "BM_%d : Duplicate notification received - index 0x%x!!\n",
1718                             cq_index);
1719                 WARN_ON(1);
1720         }
1721
1722         *pwritables = *pwritables + writables;
1723         return 0;
1724 }
1725
1726 static void hwi_free_async_msg(struct beiscsi_hba *phba,
1727                                struct hwi_async_pdu_context *pasync_ctx,
1728                                unsigned int cri)
1729 {
1730         struct async_pdu_handle *pasync_handle, *tmp_handle;
1731         struct list_head *plist;
1732
1733         plist  = &pasync_ctx->async_entry[cri].wait_queue.list;
1734         list_for_each_entry_safe(pasync_handle, tmp_handle, plist, link) {
1735                 list_del(&pasync_handle->link);
1736
1737                 if (pasync_handle->is_header) {
1738                         list_add_tail(&pasync_handle->link,
1739                                       &pasync_ctx->async_header.free_list);
1740                         pasync_ctx->async_header.free_entries++;
1741                 } else {
1742                         list_add_tail(&pasync_handle->link,
1743                                       &pasync_ctx->async_data.free_list);
1744                         pasync_ctx->async_data.free_entries++;
1745                 }
1746         }
1747
1748         INIT_LIST_HEAD(&pasync_ctx->async_entry[cri].wait_queue.list);
1749         pasync_ctx->async_entry[cri].wait_queue.hdr_received = 0;
1750         pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1751 }
1752
1753 static struct phys_addr *
1754 hwi_get_ring_address(struct hwi_async_pdu_context *pasync_ctx,
1755                      unsigned int is_header, unsigned int host_write_ptr)
1756 {
1757         struct phys_addr *pasync_sge = NULL;
1758
1759         if (is_header)
1760                 pasync_sge = pasync_ctx->async_header.ring_base;
1761         else
1762                 pasync_sge = pasync_ctx->async_data.ring_base;
1763
1764         return pasync_sge + host_write_ptr;
1765 }
1766
1767 static void hwi_post_async_buffers(struct beiscsi_hba *phba,
1768                                     unsigned int is_header, uint8_t ulp_num)
1769 {
1770         struct hwi_controller *phwi_ctrlr;
1771         struct hwi_async_pdu_context *pasync_ctx;
1772         struct async_pdu_handle *pasync_handle;
1773         struct list_head *pfree_link, *pbusy_list;
1774         struct phys_addr *pasync_sge;
1775         unsigned int ring_id, num_entries;
1776         unsigned int host_write_num, doorbell_offset;
1777         unsigned int writables;
1778         unsigned int i = 0;
1779         u32 doorbell = 0;
1780
1781         phwi_ctrlr = phba->phwi_ctrlr;
1782         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr, ulp_num);
1783         num_entries = pasync_ctx->num_entries;
1784
1785         if (is_header) {
1786                 writables = min(pasync_ctx->async_header.writables,
1787                                 pasync_ctx->async_header.free_entries);
1788                 pfree_link = pasync_ctx->async_header.free_list.next;
1789                 host_write_num = pasync_ctx->async_header.host_write_ptr;
1790                 ring_id = phwi_ctrlr->default_pdu_hdr[ulp_num].id;
1791                 doorbell_offset = phwi_ctrlr->default_pdu_hdr[ulp_num].
1792                                   doorbell_offset;
1793         } else {
1794                 writables = min(pasync_ctx->async_data.writables,
1795                                 pasync_ctx->async_data.free_entries);
1796                 pfree_link = pasync_ctx->async_data.free_list.next;
1797                 host_write_num = pasync_ctx->async_data.host_write_ptr;
1798                 ring_id = phwi_ctrlr->default_pdu_data[ulp_num].id;
1799                 doorbell_offset = phwi_ctrlr->default_pdu_data[ulp_num].
1800                                   doorbell_offset;
1801         }
1802
1803         writables = (writables / 8) * 8;
1804         if (writables) {
1805                 for (i = 0; i < writables; i++) {
1806                         pbusy_list =
1807                             hwi_get_async_busy_list(pasync_ctx, is_header,
1808                                                     host_write_num);
1809                         pasync_handle =
1810                             list_entry(pfree_link, struct async_pdu_handle,
1811                                                                 link);
1812                         WARN_ON(!pasync_handle);
1813                         pasync_handle->consumed = 0;
1814
1815                         pfree_link = pfree_link->next;
1816
1817                         pasync_sge = hwi_get_ring_address(pasync_ctx,
1818                                                 is_header, host_write_num);
1819
1820                         pasync_sge->hi = pasync_handle->pa.u.a32.address_lo;
1821                         pasync_sge->lo = pasync_handle->pa.u.a32.address_hi;
1822
1823                         list_move(&pasync_handle->link, pbusy_list);
1824
1825                         host_write_num++;
1826                         host_write_num = host_write_num % num_entries;
1827                 }
1828
1829                 if (is_header) {
1830                         pasync_ctx->async_header.host_write_ptr =
1831                                                         host_write_num;
1832                         pasync_ctx->async_header.free_entries -= writables;
1833                         pasync_ctx->async_header.writables -= writables;
1834                         pasync_ctx->async_header.busy_entries += writables;
1835                 } else {
1836                         pasync_ctx->async_data.host_write_ptr = host_write_num;
1837                         pasync_ctx->async_data.free_entries -= writables;
1838                         pasync_ctx->async_data.writables -= writables;
1839                         pasync_ctx->async_data.busy_entries += writables;
1840                 }
1841
1842                 doorbell |= ring_id & DB_DEF_PDU_RING_ID_MASK;
1843                 doorbell |= 1 << DB_DEF_PDU_REARM_SHIFT;
1844                 doorbell |= 0 << DB_DEF_PDU_EVENT_SHIFT;
1845                 doorbell |= (writables & DB_DEF_PDU_CQPROC_MASK)
1846                                         << DB_DEF_PDU_CQPROC_SHIFT;
1847
1848                 iowrite32(doorbell, phba->db_va + doorbell_offset);
1849         }
1850 }
1851
1852 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba *phba,
1853                                          struct beiscsi_conn *beiscsi_conn,
1854                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1855 {
1856         struct hwi_controller *phwi_ctrlr;
1857         struct hwi_async_pdu_context *pasync_ctx;
1858         struct async_pdu_handle *pasync_handle = NULL;
1859         unsigned int cq_index = -1;
1860         uint16_t cri_index = BE_GET_CRI_FROM_CID(
1861                              beiscsi_conn->beiscsi_conn_cid);
1862
1863         phwi_ctrlr = phba->phwi_ctrlr;
1864         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1865                      BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1866                      cri_index));
1867
1868         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
1869                                              pdpdu_cqe, &cq_index);
1870         BUG_ON(pasync_handle->is_header != 0);
1871         if (pasync_handle->consumed == 0)
1872                 hwi_update_async_writables(phba, pasync_ctx,
1873                                            pasync_handle->is_header, cq_index);
1874
1875         hwi_free_async_msg(phba, pasync_ctx, pasync_handle->cri);
1876         hwi_post_async_buffers(phba, pasync_handle->is_header,
1877                                BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1878                                cri_index));
1879 }
1880
1881 static unsigned int
1882 hwi_fwd_async_msg(struct beiscsi_conn *beiscsi_conn,
1883                   struct beiscsi_hba *phba,
1884                   struct hwi_async_pdu_context *pasync_ctx, unsigned short cri)
1885 {
1886         struct list_head *plist;
1887         struct async_pdu_handle *pasync_handle;
1888         void *phdr = NULL;
1889         unsigned int hdr_len = 0, buf_len = 0;
1890         unsigned int status, index = 0, offset = 0;
1891         void *pfirst_buffer = NULL;
1892         unsigned int num_buf = 0;
1893
1894         plist = &pasync_ctx->async_entry[cri].wait_queue.list;
1895
1896         list_for_each_entry(pasync_handle, plist, link) {
1897                 if (index == 0) {
1898                         phdr = pasync_handle->pbuffer;
1899                         hdr_len = pasync_handle->buffer_len;
1900                 } else {
1901                         buf_len = pasync_handle->buffer_len;
1902                         if (!num_buf) {
1903                                 pfirst_buffer = pasync_handle->pbuffer;
1904                                 num_buf++;
1905                         }
1906                         memcpy(pfirst_buffer + offset,
1907                                pasync_handle->pbuffer, buf_len);
1908                         offset += buf_len;
1909                 }
1910                 index++;
1911         }
1912
1913         status = beiscsi_process_async_pdu(beiscsi_conn, phba,
1914                                             phdr, hdr_len, pfirst_buffer,
1915                                             offset);
1916
1917         hwi_free_async_msg(phba, pasync_ctx, cri);
1918         return 0;
1919 }
1920
1921 static unsigned int
1922 hwi_gather_async_pdu(struct beiscsi_conn *beiscsi_conn,
1923                      struct beiscsi_hba *phba,
1924                      struct async_pdu_handle *pasync_handle)
1925 {
1926         struct hwi_async_pdu_context *pasync_ctx;
1927         struct hwi_controller *phwi_ctrlr;
1928         unsigned int bytes_needed = 0, status = 0;
1929         unsigned short cri = pasync_handle->cri;
1930         struct pdu_base *ppdu;
1931
1932         phwi_ctrlr = phba->phwi_ctrlr;
1933         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
1934                      BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
1935                      BE_GET_CRI_FROM_CID(beiscsi_conn->
1936                                  beiscsi_conn_cid)));
1937
1938         list_del(&pasync_handle->link);
1939         if (pasync_handle->is_header) {
1940                 pasync_ctx->async_header.busy_entries--;
1941                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1942                         hwi_free_async_msg(phba, pasync_ctx, cri);
1943                         BUG();
1944                 }
1945
1946                 pasync_ctx->async_entry[cri].wait_queue.bytes_received = 0;
1947                 pasync_ctx->async_entry[cri].wait_queue.hdr_received = 1;
1948                 pasync_ctx->async_entry[cri].wait_queue.hdr_len =
1949                                 (unsigned short)pasync_handle->buffer_len;
1950                 list_add_tail(&pasync_handle->link,
1951                               &pasync_ctx->async_entry[cri].wait_queue.list);
1952
1953                 ppdu = pasync_handle->pbuffer;
1954                 bytes_needed = ((((ppdu->dw[offsetof(struct amap_pdu_base,
1955                         data_len_hi) / 32] & PDUBASE_DATALENHI_MASK) << 8) &
1956                         0xFFFF0000) | ((be16_to_cpu((ppdu->
1957                         dw[offsetof(struct amap_pdu_base, data_len_lo) / 32]
1958                         & PDUBASE_DATALENLO_MASK) >> 16)) & 0x0000FFFF));
1959
1960                 if (status == 0) {
1961                         pasync_ctx->async_entry[cri].wait_queue.bytes_needed =
1962                             bytes_needed;
1963
1964                         if (bytes_needed == 0)
1965                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1966                                                            pasync_ctx, cri);
1967                 }
1968         } else {
1969                 pasync_ctx->async_data.busy_entries--;
1970                 if (pasync_ctx->async_entry[cri].wait_queue.hdr_received) {
1971                         list_add_tail(&pasync_handle->link,
1972                                       &pasync_ctx->async_entry[cri].wait_queue.
1973                                       list);
1974                         pasync_ctx->async_entry[cri].wait_queue.
1975                                 bytes_received +=
1976                                 (unsigned short)pasync_handle->buffer_len;
1977
1978                         if (pasync_ctx->async_entry[cri].wait_queue.
1979                             bytes_received >=
1980                             pasync_ctx->async_entry[cri].wait_queue.
1981                             bytes_needed)
1982                                 status = hwi_fwd_async_msg(beiscsi_conn, phba,
1983                                                            pasync_ctx, cri);
1984                 }
1985         }
1986         return status;
1987 }
1988
1989 static void hwi_process_default_pdu_ring(struct beiscsi_conn *beiscsi_conn,
1990                                          struct beiscsi_hba *phba,
1991                                          struct i_t_dpdu_cqe *pdpdu_cqe)
1992 {
1993         struct hwi_controller *phwi_ctrlr;
1994         struct hwi_async_pdu_context *pasync_ctx;
1995         struct async_pdu_handle *pasync_handle = NULL;
1996         unsigned int cq_index = -1;
1997         uint16_t cri_index = BE_GET_CRI_FROM_CID(
1998                              beiscsi_conn->beiscsi_conn_cid);
1999
2000         phwi_ctrlr = phba->phwi_ctrlr;
2001         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr,
2002                      BEISCSI_GET_ULP_FROM_CRI(phwi_ctrlr,
2003                      cri_index));
2004
2005         pasync_handle = hwi_get_async_handle(phba, beiscsi_conn, pasync_ctx,
2006                                              pdpdu_cqe, &cq_index);
2007
2008         if (pasync_handle->consumed == 0)
2009                 hwi_update_async_writables(phba, pasync_ctx,
2010                                            pasync_handle->is_header, cq_index);
2011
2012         hwi_gather_async_pdu(beiscsi_conn, phba, pasync_handle);
2013         hwi_post_async_buffers(phba, pasync_handle->is_header,
2014                                BEISCSI_GET_ULP_FROM_CRI(
2015                                phwi_ctrlr, cri_index));
2016 }
2017
2018 static void  beiscsi_process_mcc_isr(struct beiscsi_hba *phba)
2019 {
2020         struct be_queue_info *mcc_cq;
2021         struct  be_mcc_compl *mcc_compl;
2022         unsigned int num_processed = 0;
2023
2024         mcc_cq = &phba->ctrl.mcc_obj.cq;
2025         mcc_compl = queue_tail_node(mcc_cq);
2026         mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2027         while (mcc_compl->flags & CQE_FLAGS_VALID_MASK) {
2028
2029                 if (num_processed >= 32) {
2030                         hwi_ring_cq_db(phba, mcc_cq->id,
2031                                         num_processed, 0, 0);
2032                         num_processed = 0;
2033                 }
2034                 if (mcc_compl->flags & CQE_FLAGS_ASYNC_MASK) {
2035                         /* Interpret flags as an async trailer */
2036                         if (is_link_state_evt(mcc_compl->flags))
2037                                 /* Interpret compl as a async link evt */
2038                                 beiscsi_async_link_state_process(phba,
2039                                 (struct be_async_event_link_state *) mcc_compl);
2040                         else
2041                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_MBOX,
2042                                             "BM_%d :  Unsupported Async Event, flags"
2043                                             " = 0x%08x\n",
2044                                             mcc_compl->flags);
2045                 } else if (mcc_compl->flags & CQE_FLAGS_COMPLETED_MASK) {
2046                         be_mcc_compl_process_isr(&phba->ctrl, mcc_compl);
2047                         atomic_dec(&phba->ctrl.mcc_obj.q.used);
2048                 }
2049
2050                 mcc_compl->flags = 0;
2051                 queue_tail_inc(mcc_cq);
2052                 mcc_compl = queue_tail_node(mcc_cq);
2053                 mcc_compl->flags = le32_to_cpu(mcc_compl->flags);
2054                 num_processed++;
2055         }
2056
2057         if (num_processed > 0)
2058                 hwi_ring_cq_db(phba, mcc_cq->id, num_processed, 1, 0);
2059
2060 }
2061
2062 /**
2063  * beiscsi_process_cq()- Process the Completion Queue
2064  * @pbe_eq: Event Q on which the Completion has come
2065  *
2066  * return
2067  *     Number of Completion Entries processed.
2068  **/
2069 unsigned int beiscsi_process_cq(struct be_eq_obj *pbe_eq)
2070 {
2071         struct be_queue_info *cq;
2072         struct sol_cqe *sol;
2073         struct dmsg_cqe *dmsg;
2074         unsigned int num_processed = 0;
2075         unsigned int tot_nump = 0;
2076         unsigned short code = 0, cid = 0;
2077         uint16_t cri_index = 0;
2078         struct beiscsi_conn *beiscsi_conn;
2079         struct beiscsi_endpoint *beiscsi_ep;
2080         struct iscsi_endpoint *ep;
2081         struct beiscsi_hba *phba;
2082
2083         cq = pbe_eq->cq;
2084         sol = queue_tail_node(cq);
2085         phba = pbe_eq->phba;
2086
2087         while (sol->dw[offsetof(struct amap_sol_cqe, valid) / 32] &
2088                CQE_VALID_MASK) {
2089                 be_dws_le_to_cpu(sol, sizeof(struct sol_cqe));
2090
2091                  code = (sol->dw[offsetof(struct amap_sol_cqe, code) /
2092                          32] & CQE_CODE_MASK);
2093
2094                  /* Get the CID */
2095                 if (is_chip_be2_be3r(phba)) {
2096                         cid = AMAP_GET_BITS(struct amap_sol_cqe, cid, sol);
2097                 } else {
2098                         if ((code == DRIVERMSG_NOTIFY) ||
2099                             (code == UNSOL_HDR_NOTIFY) ||
2100                             (code == UNSOL_DATA_NOTIFY))
2101                                 cid = AMAP_GET_BITS(
2102                                                     struct amap_i_t_dpdu_cqe_v2,
2103                                                     cid, sol);
2104                          else
2105                                  cid = AMAP_GET_BITS(struct amap_sol_cqe_v2,
2106                                                      cid, sol);
2107                 }
2108
2109                 cri_index = BE_GET_CRI_FROM_CID(cid);
2110                 ep = phba->ep_array[cri_index];
2111
2112                 if (ep == NULL) {
2113                         /* connection has already been freed
2114                          * just move on to next one
2115                          */
2116                         beiscsi_log(phba, KERN_WARNING,
2117                                     BEISCSI_LOG_INIT,
2118                                     "BM_%d : proc cqe of disconn ep: cid %d\n",
2119                                     cid);
2120                         goto proc_next_cqe;
2121                 }
2122
2123                 beiscsi_ep = ep->dd_data;
2124                 beiscsi_conn = beiscsi_ep->conn;
2125
2126                 if (num_processed >= 32) {
2127                         hwi_ring_cq_db(phba, cq->id,
2128                                         num_processed, 0, 0);
2129                         tot_nump += num_processed;
2130                         num_processed = 0;
2131                 }
2132
2133                 switch (code) {
2134                 case SOL_CMD_COMPLETE:
2135                         hwi_complete_cmd(beiscsi_conn, phba, sol);
2136                         break;
2137                 case DRIVERMSG_NOTIFY:
2138                         beiscsi_log(phba, KERN_INFO,
2139                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2140                                     "BM_%d : Received %s[%d] on CID : %d\n",
2141                                     cqe_desc[code], code, cid);
2142
2143                         dmsg = (struct dmsg_cqe *)sol;
2144                         hwi_complete_drvr_msgs(beiscsi_conn, phba, sol);
2145                         break;
2146                 case UNSOL_HDR_NOTIFY:
2147                         beiscsi_log(phba, KERN_INFO,
2148                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2149                                     "BM_%d : Received %s[%d] on CID : %d\n",
2150                                     cqe_desc[code], code, cid);
2151
2152                         spin_lock_bh(&phba->async_pdu_lock);
2153                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2154                                              (struct i_t_dpdu_cqe *)sol);
2155                         spin_unlock_bh(&phba->async_pdu_lock);
2156                         break;
2157                 case UNSOL_DATA_NOTIFY:
2158                         beiscsi_log(phba, KERN_INFO,
2159                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2160                                     "BM_%d : Received %s[%d] on CID : %d\n",
2161                                     cqe_desc[code], code, cid);
2162
2163                         spin_lock_bh(&phba->async_pdu_lock);
2164                         hwi_process_default_pdu_ring(beiscsi_conn, phba,
2165                                              (struct i_t_dpdu_cqe *)sol);
2166                         spin_unlock_bh(&phba->async_pdu_lock);
2167                         break;
2168                 case CXN_INVALIDATE_INDEX_NOTIFY:
2169                 case CMD_INVALIDATED_NOTIFY:
2170                 case CXN_INVALIDATE_NOTIFY:
2171                         beiscsi_log(phba, KERN_ERR,
2172                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2173                                     "BM_%d : Ignoring %s[%d] on CID : %d\n",
2174                                     cqe_desc[code], code, cid);
2175                         break;
2176                 case SOL_CMD_KILLED_DATA_DIGEST_ERR:
2177                 case CMD_KILLED_INVALID_STATSN_RCVD:
2178                 case CMD_KILLED_INVALID_R2T_RCVD:
2179                 case CMD_CXN_KILLED_LUN_INVALID:
2180                 case CMD_CXN_KILLED_ICD_INVALID:
2181                 case CMD_CXN_KILLED_ITT_INVALID:
2182                 case CMD_CXN_KILLED_SEQ_OUTOFORDER:
2183                 case CMD_CXN_KILLED_INVALID_DATASN_RCVD:
2184                         beiscsi_log(phba, KERN_ERR,
2185                                     BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2186                                     "BM_%d : Cmd Notification %s[%d] on CID : %d\n",
2187                                     cqe_desc[code], code,  cid);
2188                         break;
2189                 case UNSOL_DATA_DIGEST_ERROR_NOTIFY:
2190                         beiscsi_log(phba, KERN_ERR,
2191                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2192                                     "BM_%d :  Dropping %s[%d] on DPDU ring on CID : %d\n",
2193                                     cqe_desc[code], code, cid);
2194                         spin_lock_bh(&phba->async_pdu_lock);
2195                         hwi_flush_default_pdu_buffer(phba, beiscsi_conn,
2196                                              (struct i_t_dpdu_cqe *) sol);
2197                         spin_unlock_bh(&phba->async_pdu_lock);
2198                         break;
2199                 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL:
2200                 case CXN_KILLED_BURST_LEN_MISMATCH:
2201                 case CXN_KILLED_AHS_RCVD:
2202                 case CXN_KILLED_HDR_DIGEST_ERR:
2203                 case CXN_KILLED_UNKNOWN_HDR:
2204                 case CXN_KILLED_STALE_ITT_TTT_RCVD:
2205                 case CXN_KILLED_INVALID_ITT_TTT_RCVD:
2206                 case CXN_KILLED_TIMED_OUT:
2207                 case CXN_KILLED_FIN_RCVD:
2208                 case CXN_KILLED_RST_SENT:
2209                 case CXN_KILLED_RST_RCVD:
2210                 case CXN_KILLED_BAD_UNSOL_PDU_RCVD:
2211                 case CXN_KILLED_BAD_WRB_INDEX_ERROR:
2212                 case CXN_KILLED_OVER_RUN_RESIDUAL:
2213                 case CXN_KILLED_UNDER_RUN_RESIDUAL:
2214                 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN:
2215                         beiscsi_log(phba, KERN_ERR,
2216                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2217                                     "BM_%d : Event %s[%d] received on CID : %d\n",
2218                                     cqe_desc[code], code, cid);
2219                         if (beiscsi_conn)
2220                                 iscsi_conn_failure(beiscsi_conn->conn,
2221                                                    ISCSI_ERR_CONN_FAILED);
2222                         break;
2223                 default:
2224                         beiscsi_log(phba, KERN_ERR,
2225                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
2226                                     "BM_%d : Invalid CQE Event Received Code : %d"
2227                                     "CID 0x%x...\n",
2228                                     code, cid);
2229                         break;
2230                 }
2231
2232 proc_next_cqe:
2233                 AMAP_SET_BITS(struct amap_sol_cqe, valid, sol, 0);
2234                 queue_tail_inc(cq);
2235                 sol = queue_tail_node(cq);
2236                 num_processed++;
2237         }
2238
2239         if (num_processed > 0) {
2240                 tot_nump += num_processed;
2241                 hwi_ring_cq_db(phba, cq->id, num_processed, 1, 0);
2242         }
2243         return tot_nump;
2244 }
2245
2246 void beiscsi_process_all_cqs(struct work_struct *work)
2247 {
2248         unsigned long flags;
2249         struct hwi_controller *phwi_ctrlr;
2250         struct hwi_context_memory *phwi_context;
2251         struct beiscsi_hba *phba;
2252         struct be_eq_obj *pbe_eq =
2253             container_of(work, struct be_eq_obj, work_cqs);
2254
2255         phba = pbe_eq->phba;
2256         phwi_ctrlr = phba->phwi_ctrlr;
2257         phwi_context = phwi_ctrlr->phwi_ctxt;
2258
2259         if (pbe_eq->todo_mcc_cq) {
2260                 spin_lock_irqsave(&phba->isr_lock, flags);
2261                 pbe_eq->todo_mcc_cq = false;
2262                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2263                 beiscsi_process_mcc_isr(phba);
2264         }
2265
2266         if (pbe_eq->todo_cq) {
2267                 spin_lock_irqsave(&phba->isr_lock, flags);
2268                 pbe_eq->todo_cq = false;
2269                 spin_unlock_irqrestore(&phba->isr_lock, flags);
2270                 beiscsi_process_cq(pbe_eq);
2271         }
2272
2273         /* rearm EQ for further interrupts */
2274         hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2275 }
2276
2277 static int be_iopoll(struct blk_iopoll *iop, int budget)
2278 {
2279         unsigned int ret;
2280         struct beiscsi_hba *phba;
2281         struct be_eq_obj *pbe_eq;
2282
2283         pbe_eq = container_of(iop, struct be_eq_obj, iopoll);
2284         ret = beiscsi_process_cq(pbe_eq);
2285         pbe_eq->cq_count += ret;
2286         if (ret < budget) {
2287                 phba = pbe_eq->phba;
2288                 blk_iopoll_complete(iop);
2289                 beiscsi_log(phba, KERN_INFO,
2290                             BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
2291                             "BM_%d : rearm pbe_eq->q.id =%d\n",
2292                             pbe_eq->q.id);
2293                 hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
2294         }
2295         return ret;
2296 }
2297
2298 static void
2299 hwi_write_sgl_v2(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2300                   unsigned int num_sg, struct beiscsi_io_task *io_task)
2301 {
2302         struct iscsi_sge *psgl;
2303         unsigned int sg_len, index;
2304         unsigned int sge_len = 0;
2305         unsigned long long addr;
2306         struct scatterlist *l_sg;
2307         unsigned int offset;
2308
2309         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_lo, pwrb,
2310                       io_task->bhs_pa.u.a32.address_lo);
2311         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, iscsi_bhs_addr_hi, pwrb,
2312                       io_task->bhs_pa.u.a32.address_hi);
2313
2314         l_sg = sg;
2315         for (index = 0; (index < num_sg) && (index < 2); index++,
2316                         sg = sg_next(sg)) {
2317                 if (index == 0) {
2318                         sg_len = sg_dma_len(sg);
2319                         addr = (u64) sg_dma_address(sg);
2320                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2321                                       sge0_addr_lo, pwrb,
2322                                       lower_32_bits(addr));
2323                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2324                                       sge0_addr_hi, pwrb,
2325                                       upper_32_bits(addr));
2326                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2327                                       sge0_len, pwrb,
2328                                       sg_len);
2329                         sge_len = sg_len;
2330                 } else {
2331                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_r2t_offset,
2332                                       pwrb, sge_len);
2333                         sg_len = sg_dma_len(sg);
2334                         addr = (u64) sg_dma_address(sg);
2335                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2336                                       sge1_addr_lo, pwrb,
2337                                       lower_32_bits(addr));
2338                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2339                                       sge1_addr_hi, pwrb,
2340                                       upper_32_bits(addr));
2341                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
2342                                       sge1_len, pwrb,
2343                                       sg_len);
2344                 }
2345         }
2346         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2347         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2348
2349         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2350
2351         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2352                       io_task->bhs_pa.u.a32.address_hi);
2353         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2354                       io_task->bhs_pa.u.a32.address_lo);
2355
2356         if (num_sg == 1) {
2357                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2358                               1);
2359                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2360                               0);
2361         } else if (num_sg == 2) {
2362                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2363                               0);
2364                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2365                               1);
2366         } else {
2367                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge0_last, pwrb,
2368                               0);
2369                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sge1_last, pwrb,
2370                               0);
2371         }
2372
2373         sg = l_sg;
2374         psgl++;
2375         psgl++;
2376         offset = 0;
2377         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2378                 sg_len = sg_dma_len(sg);
2379                 addr = (u64) sg_dma_address(sg);
2380                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2381                               lower_32_bits(addr));
2382                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2383                               upper_32_bits(addr));
2384                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2385                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2386                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2387                 offset += sg_len;
2388         }
2389         psgl--;
2390         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2391 }
2392
2393 static void
2394 hwi_write_sgl(struct iscsi_wrb *pwrb, struct scatterlist *sg,
2395               unsigned int num_sg, struct beiscsi_io_task *io_task)
2396 {
2397         struct iscsi_sge *psgl;
2398         unsigned int sg_len, index;
2399         unsigned int sge_len = 0;
2400         unsigned long long addr;
2401         struct scatterlist *l_sg;
2402         unsigned int offset;
2403
2404         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2405                                       io_task->bhs_pa.u.a32.address_lo);
2406         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2407                                       io_task->bhs_pa.u.a32.address_hi);
2408
2409         l_sg = sg;
2410         for (index = 0; (index < num_sg) && (index < 2); index++,
2411                                                          sg = sg_next(sg)) {
2412                 if (index == 0) {
2413                         sg_len = sg_dma_len(sg);
2414                         addr = (u64) sg_dma_address(sg);
2415                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2416                                                 ((u32)(addr & 0xFFFFFFFF)));
2417                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2418                                                         ((u32)(addr >> 32)));
2419                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2420                                                         sg_len);
2421                         sge_len = sg_len;
2422                 } else {
2423                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_r2t_offset,
2424                                                         pwrb, sge_len);
2425                         sg_len = sg_dma_len(sg);
2426                         addr = (u64) sg_dma_address(sg);
2427                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_lo, pwrb,
2428                                                 ((u32)(addr & 0xFFFFFFFF)));
2429                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_addr_hi, pwrb,
2430                                                         ((u32)(addr >> 32)));
2431                         AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_len, pwrb,
2432                                                         sg_len);
2433                 }
2434         }
2435         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2436         memset(psgl, 0, sizeof(*psgl) * BE2_SGE);
2437
2438         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len - 2);
2439
2440         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2441                         io_task->bhs_pa.u.a32.address_hi);
2442         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2443                         io_task->bhs_pa.u.a32.address_lo);
2444
2445         if (num_sg == 1) {
2446                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2447                                                                 1);
2448                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2449                                                                 0);
2450         } else if (num_sg == 2) {
2451                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2452                                                                 0);
2453                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2454                                                                 1);
2455         } else {
2456                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb,
2457                                                                 0);
2458                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge1_last, pwrb,
2459                                                                 0);
2460         }
2461         sg = l_sg;
2462         psgl++;
2463         psgl++;
2464         offset = 0;
2465         for (index = 0; index < num_sg; index++, sg = sg_next(sg), psgl++) {
2466                 sg_len = sg_dma_len(sg);
2467                 addr = (u64) sg_dma_address(sg);
2468                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2469                                                 (addr & 0xFFFFFFFF));
2470                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2471                                                 (addr >> 32));
2472                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, sg_len);
2473                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, offset);
2474                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2475                 offset += sg_len;
2476         }
2477         psgl--;
2478         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2479 }
2480
2481 /**
2482  * hwi_write_buffer()- Populate the WRB with task info
2483  * @pwrb: ptr to the WRB entry
2484  * @task: iscsi task which is to be executed
2485  **/
2486 static void hwi_write_buffer(struct iscsi_wrb *pwrb, struct iscsi_task *task)
2487 {
2488         struct iscsi_sge *psgl;
2489         struct beiscsi_io_task *io_task = task->dd_data;
2490         struct beiscsi_conn *beiscsi_conn = io_task->conn;
2491         struct beiscsi_hba *phba = beiscsi_conn->phba;
2492         uint8_t dsp_value = 0;
2493
2494         io_task->bhs_len = sizeof(struct be_nonio_bhs) - 2;
2495         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_lo, pwrb,
2496                                 io_task->bhs_pa.u.a32.address_lo);
2497         AMAP_SET_BITS(struct amap_iscsi_wrb, iscsi_bhs_addr_hi, pwrb,
2498                                 io_task->bhs_pa.u.a32.address_hi);
2499
2500         if (task->data) {
2501
2502                 /* Check for the data_count */
2503                 dsp_value = (task->data_count) ? 1 : 0;
2504
2505                 if (is_chip_be2_be3r(phba))
2506                         AMAP_SET_BITS(struct amap_iscsi_wrb, dsp,
2507                                       pwrb, dsp_value);
2508                 else
2509                         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp,
2510                                       pwrb, dsp_value);
2511
2512                 /* Map addr only if there is data_count */
2513                 if (dsp_value) {
2514                         io_task->mtask_addr = pci_map_single(phba->pcidev,
2515                                                              task->data,
2516                                                              task->data_count,
2517                                                              PCI_DMA_TODEVICE);
2518                         io_task->mtask_data_count = task->data_count;
2519                 } else
2520                         io_task->mtask_addr = 0;
2521
2522                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_lo, pwrb,
2523                               lower_32_bits(io_task->mtask_addr));
2524                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_addr_hi, pwrb,
2525                               upper_32_bits(io_task->mtask_addr));
2526                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_len, pwrb,
2527                                                 task->data_count);
2528
2529                 AMAP_SET_BITS(struct amap_iscsi_wrb, sge0_last, pwrb, 1);
2530         } else {
2531                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
2532                 io_task->mtask_addr = 0;
2533         }
2534
2535         psgl = (struct iscsi_sge *)io_task->psgl_handle->pfrag;
2536
2537         AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, io_task->bhs_len);
2538
2539         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2540                       io_task->bhs_pa.u.a32.address_hi);
2541         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2542                       io_task->bhs_pa.u.a32.address_lo);
2543         if (task->data) {
2544                 psgl++;
2545                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl, 0);
2546                 AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl, 0);
2547                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0);
2548                 AMAP_SET_BITS(struct amap_iscsi_sge, sge_offset, psgl, 0);
2549                 AMAP_SET_BITS(struct amap_iscsi_sge, rsvd0, psgl, 0);
2550                 AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 0);
2551
2552                 psgl++;
2553                 if (task->data) {
2554                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, psgl,
2555                                       lower_32_bits(io_task->mtask_addr));
2556                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, psgl,
2557                                       upper_32_bits(io_task->mtask_addr));
2558                 }
2559                 AMAP_SET_BITS(struct amap_iscsi_sge, len, psgl, 0x106);
2560         }
2561         AMAP_SET_BITS(struct amap_iscsi_sge, last_sge, psgl, 1);
2562 }
2563
2564 /**
2565  * beiscsi_find_mem_req()- Find mem needed
2566  * @phba: ptr to HBA struct
2567  **/
2568 static void beiscsi_find_mem_req(struct beiscsi_hba *phba)
2569 {
2570         uint8_t mem_descr_index, ulp_num;
2571         unsigned int num_cq_pages, num_async_pdu_buf_pages;
2572         unsigned int num_async_pdu_data_pages, wrb_sz_per_cxn;
2573         unsigned int num_async_pdu_buf_sgl_pages, num_async_pdu_data_sgl_pages;
2574
2575         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
2576                                       sizeof(struct sol_cqe));
2577
2578         phba->params.hwi_ws_sz = sizeof(struct hwi_controller);
2579
2580         phba->mem_req[ISCSI_MEM_GLOBAL_HEADER] = 2 *
2581                                                  BE_ISCSI_PDU_HEADER_SIZE;
2582         phba->mem_req[HWI_MEM_ADDN_CONTEXT] =
2583                                             sizeof(struct hwi_context_memory);
2584
2585
2586         phba->mem_req[HWI_MEM_WRB] = sizeof(struct iscsi_wrb)
2587             * (phba->params.wrbs_per_cxn)
2588             * phba->params.cxns_per_ctrl;
2589         wrb_sz_per_cxn =  sizeof(struct wrb_handle) *
2590                                  (phba->params.wrbs_per_cxn);
2591         phba->mem_req[HWI_MEM_WRBH] = roundup_pow_of_two((wrb_sz_per_cxn) *
2592                                 phba->params.cxns_per_ctrl);
2593
2594         phba->mem_req[HWI_MEM_SGLH] = sizeof(struct sgl_handle) *
2595                 phba->params.icds_per_ctrl;
2596         phba->mem_req[HWI_MEM_SGE] = sizeof(struct iscsi_sge) *
2597                 phba->params.num_sge_per_io * phba->params.icds_per_ctrl;
2598         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2599                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2600
2601                         num_async_pdu_buf_sgl_pages =
2602                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2603                                                phba, ulp_num) *
2604                                                sizeof(struct phys_addr));
2605
2606                         num_async_pdu_buf_pages =
2607                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2608                                                phba, ulp_num) *
2609                                                phba->params.defpdu_hdr_sz);
2610
2611                         num_async_pdu_data_pages =
2612                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2613                                                phba, ulp_num) *
2614                                                phba->params.defpdu_data_sz);
2615
2616                         num_async_pdu_data_sgl_pages =
2617                                 PAGES_REQUIRED(BEISCSI_GET_CID_COUNT(
2618                                                phba, ulp_num) *
2619                                                sizeof(struct phys_addr));
2620
2621                         mem_descr_index = (HWI_MEM_TEMPLATE_HDR_ULP0 +
2622                                           (ulp_num * MEM_DESCR_OFFSET));
2623                         phba->mem_req[mem_descr_index] =
2624                                         BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2625                                         BEISCSI_TEMPLATE_HDR_PER_CXN_SIZE;
2626
2627                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2628                                           (ulp_num * MEM_DESCR_OFFSET));
2629                         phba->mem_req[mem_descr_index] =
2630                                           num_async_pdu_buf_pages *
2631                                           PAGE_SIZE;
2632
2633                         mem_descr_index = (HWI_MEM_ASYNC_DATA_BUF_ULP0 +
2634                                           (ulp_num * MEM_DESCR_OFFSET));
2635                         phba->mem_req[mem_descr_index] =
2636                                           num_async_pdu_data_pages *
2637                                           PAGE_SIZE;
2638
2639                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2640                                           (ulp_num * MEM_DESCR_OFFSET));
2641                         phba->mem_req[mem_descr_index] =
2642                                           num_async_pdu_buf_sgl_pages *
2643                                           PAGE_SIZE;
2644
2645                         mem_descr_index = (HWI_MEM_ASYNC_DATA_RING_ULP0 +
2646                                           (ulp_num * MEM_DESCR_OFFSET));
2647                         phba->mem_req[mem_descr_index] =
2648                                           num_async_pdu_data_sgl_pages *
2649                                           PAGE_SIZE;
2650
2651                         mem_descr_index = (HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2652                                           (ulp_num * MEM_DESCR_OFFSET));
2653                         phba->mem_req[mem_descr_index] =
2654                                           BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2655                                           sizeof(struct async_pdu_handle);
2656
2657                         mem_descr_index = (HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
2658                                           (ulp_num * MEM_DESCR_OFFSET));
2659                         phba->mem_req[mem_descr_index] =
2660                                           BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2661                                           sizeof(struct async_pdu_handle);
2662
2663                         mem_descr_index = (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2664                                           (ulp_num * MEM_DESCR_OFFSET));
2665                         phba->mem_req[mem_descr_index] =
2666                                           sizeof(struct hwi_async_pdu_context) +
2667                                          (BEISCSI_GET_CID_COUNT(phba, ulp_num) *
2668                                           sizeof(struct hwi_async_entry));
2669                 }
2670         }
2671 }
2672
2673 static int beiscsi_alloc_mem(struct beiscsi_hba *phba)
2674 {
2675         dma_addr_t bus_add;
2676         struct hwi_controller *phwi_ctrlr;
2677         struct be_mem_descriptor *mem_descr;
2678         struct mem_array *mem_arr, *mem_arr_orig;
2679         unsigned int i, j, alloc_size, curr_alloc_size;
2680
2681         phba->phwi_ctrlr = kzalloc(phba->params.hwi_ws_sz, GFP_KERNEL);
2682         if (!phba->phwi_ctrlr)
2683                 return -ENOMEM;
2684
2685         /* Allocate memory for wrb_context */
2686         phwi_ctrlr = phba->phwi_ctrlr;
2687         phwi_ctrlr->wrb_context = kzalloc(sizeof(struct hwi_wrb_context) *
2688                                           phba->params.cxns_per_ctrl,
2689                                           GFP_KERNEL);
2690         if (!phwi_ctrlr->wrb_context)
2691                 return -ENOMEM;
2692
2693         phba->init_mem = kcalloc(SE_MEM_MAX, sizeof(*mem_descr),
2694                                  GFP_KERNEL);
2695         if (!phba->init_mem) {
2696                 kfree(phwi_ctrlr->wrb_context);
2697                 kfree(phba->phwi_ctrlr);
2698                 return -ENOMEM;
2699         }
2700
2701         mem_arr_orig = kmalloc(sizeof(*mem_arr_orig) * BEISCSI_MAX_FRAGS_INIT,
2702                                GFP_KERNEL);
2703         if (!mem_arr_orig) {
2704                 kfree(phba->init_mem);
2705                 kfree(phwi_ctrlr->wrb_context);
2706                 kfree(phba->phwi_ctrlr);
2707                 return -ENOMEM;
2708         }
2709
2710         mem_descr = phba->init_mem;
2711         for (i = 0; i < SE_MEM_MAX; i++) {
2712                 if (!phba->mem_req[i]) {
2713                         mem_descr->mem_array = NULL;
2714                         mem_descr++;
2715                         continue;
2716                 }
2717
2718                 j = 0;
2719                 mem_arr = mem_arr_orig;
2720                 alloc_size = phba->mem_req[i];
2721                 memset(mem_arr, 0, sizeof(struct mem_array) *
2722                        BEISCSI_MAX_FRAGS_INIT);
2723                 curr_alloc_size = min(be_max_phys_size * 1024, alloc_size);
2724                 do {
2725                         mem_arr->virtual_address = pci_alloc_consistent(
2726                                                         phba->pcidev,
2727                                                         curr_alloc_size,
2728                                                         &bus_add);
2729                         if (!mem_arr->virtual_address) {
2730                                 if (curr_alloc_size <= BE_MIN_MEM_SIZE)
2731                                         goto free_mem;
2732                                 if (curr_alloc_size -
2733                                         rounddown_pow_of_two(curr_alloc_size))
2734                                         curr_alloc_size = rounddown_pow_of_two
2735                                                              (curr_alloc_size);
2736                                 else
2737                                         curr_alloc_size = curr_alloc_size / 2;
2738                         } else {
2739                                 mem_arr->bus_address.u.
2740                                     a64.address = (__u64) bus_add;
2741                                 mem_arr->size = curr_alloc_size;
2742                                 alloc_size -= curr_alloc_size;
2743                                 curr_alloc_size = min(be_max_phys_size *
2744                                                       1024, alloc_size);
2745                                 j++;
2746                                 mem_arr++;
2747                         }
2748                 } while (alloc_size);
2749                 mem_descr->num_elements = j;
2750                 mem_descr->size_in_bytes = phba->mem_req[i];
2751                 mem_descr->mem_array = kmalloc(sizeof(*mem_arr) * j,
2752                                                GFP_KERNEL);
2753                 if (!mem_descr->mem_array)
2754                         goto free_mem;
2755
2756                 memcpy(mem_descr->mem_array, mem_arr_orig,
2757                        sizeof(struct mem_array) * j);
2758                 mem_descr++;
2759         }
2760         kfree(mem_arr_orig);
2761         return 0;
2762 free_mem:
2763         mem_descr->num_elements = j;
2764         while ((i) || (j)) {
2765                 for (j = mem_descr->num_elements; j > 0; j--) {
2766                         pci_free_consistent(phba->pcidev,
2767                                             mem_descr->mem_array[j - 1].size,
2768                                             mem_descr->mem_array[j - 1].
2769                                             virtual_address,
2770                                             (unsigned long)mem_descr->
2771                                             mem_array[j - 1].
2772                                             bus_address.u.a64.address);
2773                 }
2774                 if (i) {
2775                         i--;
2776                         kfree(mem_descr->mem_array);
2777                         mem_descr--;
2778                 }
2779         }
2780         kfree(mem_arr_orig);
2781         kfree(phba->init_mem);
2782         kfree(phba->phwi_ctrlr->wrb_context);
2783         kfree(phba->phwi_ctrlr);
2784         return -ENOMEM;
2785 }
2786
2787 static int beiscsi_get_memory(struct beiscsi_hba *phba)
2788 {
2789         beiscsi_find_mem_req(phba);
2790         return beiscsi_alloc_mem(phba);
2791 }
2792
2793 static void iscsi_init_global_templates(struct beiscsi_hba *phba)
2794 {
2795         struct pdu_data_out *pdata_out;
2796         struct pdu_nop_out *pnop_out;
2797         struct be_mem_descriptor *mem_descr;
2798
2799         mem_descr = phba->init_mem;
2800         mem_descr += ISCSI_MEM_GLOBAL_HEADER;
2801         pdata_out =
2802             (struct pdu_data_out *)mem_descr->mem_array[0].virtual_address;
2803         memset(pdata_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2804
2805         AMAP_SET_BITS(struct amap_pdu_data_out, opcode, pdata_out,
2806                       IIOC_SCSI_DATA);
2807
2808         pnop_out =
2809             (struct pdu_nop_out *)((unsigned char *)mem_descr->mem_array[0].
2810                                    virtual_address + BE_ISCSI_PDU_HEADER_SIZE);
2811
2812         memset(pnop_out, 0, BE_ISCSI_PDU_HEADER_SIZE);
2813         AMAP_SET_BITS(struct amap_pdu_nop_out, ttt, pnop_out, 0xFFFFFFFF);
2814         AMAP_SET_BITS(struct amap_pdu_nop_out, f_bit, pnop_out, 1);
2815         AMAP_SET_BITS(struct amap_pdu_nop_out, i_bit, pnop_out, 0);
2816 }
2817
2818 static int beiscsi_init_wrb_handle(struct beiscsi_hba *phba)
2819 {
2820         struct be_mem_descriptor *mem_descr_wrbh, *mem_descr_wrb;
2821         struct hwi_context_memory *phwi_ctxt;
2822         struct wrb_handle *pwrb_handle = NULL;
2823         struct hwi_controller *phwi_ctrlr;
2824         struct hwi_wrb_context *pwrb_context;
2825         struct iscsi_wrb *pwrb = NULL;
2826         unsigned int num_cxn_wrbh = 0;
2827         unsigned int num_cxn_wrb = 0, j, idx = 0, index;
2828
2829         mem_descr_wrbh = phba->init_mem;
2830         mem_descr_wrbh += HWI_MEM_WRBH;
2831
2832         mem_descr_wrb = phba->init_mem;
2833         mem_descr_wrb += HWI_MEM_WRB;
2834         phwi_ctrlr = phba->phwi_ctrlr;
2835
2836         /* Allocate memory for WRBQ */
2837         phwi_ctxt = phwi_ctrlr->phwi_ctxt;
2838         phwi_ctxt->be_wrbq = kzalloc(sizeof(struct be_queue_info) *
2839                                      phba->params.cxns_per_ctrl,
2840                                      GFP_KERNEL);
2841         if (!phwi_ctxt->be_wrbq) {
2842                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2843                             "BM_%d : WRBQ Mem Alloc Failed\n");
2844                 return -ENOMEM;
2845         }
2846
2847         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2848                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2849                 pwrb_context->pwrb_handle_base =
2850                                 kzalloc(sizeof(struct wrb_handle *) *
2851                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2852                 if (!pwrb_context->pwrb_handle_base) {
2853                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2854                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2855                         goto init_wrb_hndl_failed;
2856                 }
2857                 pwrb_context->pwrb_handle_basestd =
2858                                 kzalloc(sizeof(struct wrb_handle *) *
2859                                         phba->params.wrbs_per_cxn, GFP_KERNEL);
2860                 if (!pwrb_context->pwrb_handle_basestd) {
2861                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
2862                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
2863                         goto init_wrb_hndl_failed;
2864                 }
2865                 if (!num_cxn_wrbh) {
2866                         pwrb_handle =
2867                                 mem_descr_wrbh->mem_array[idx].virtual_address;
2868                         num_cxn_wrbh = ((mem_descr_wrbh->mem_array[idx].size) /
2869                                         ((sizeof(struct wrb_handle)) *
2870                                          phba->params.wrbs_per_cxn));
2871                         idx++;
2872                 }
2873                 pwrb_context->alloc_index = 0;
2874                 pwrb_context->wrb_handles_available = 0;
2875                 pwrb_context->free_index = 0;
2876
2877                 if (num_cxn_wrbh) {
2878                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2879                                 pwrb_context->pwrb_handle_base[j] = pwrb_handle;
2880                                 pwrb_context->pwrb_handle_basestd[j] =
2881                                                                 pwrb_handle;
2882                                 pwrb_context->wrb_handles_available++;
2883                                 pwrb_handle->wrb_index = j;
2884                                 pwrb_handle++;
2885                         }
2886                         num_cxn_wrbh--;
2887                 }
2888         }
2889         idx = 0;
2890         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
2891                 pwrb_context = &phwi_ctrlr->wrb_context[index];
2892                 if (!num_cxn_wrb) {
2893                         pwrb = mem_descr_wrb->mem_array[idx].virtual_address;
2894                         num_cxn_wrb = (mem_descr_wrb->mem_array[idx].size) /
2895                                 ((sizeof(struct iscsi_wrb) *
2896                                   phba->params.wrbs_per_cxn));
2897                         idx++;
2898                 }
2899
2900                 if (num_cxn_wrb) {
2901                         for (j = 0; j < phba->params.wrbs_per_cxn; j++) {
2902                                 pwrb_handle = pwrb_context->pwrb_handle_base[j];
2903                                 pwrb_handle->pwrb = pwrb;
2904                                 pwrb++;
2905                         }
2906                         num_cxn_wrb--;
2907                 }
2908         }
2909         return 0;
2910 init_wrb_hndl_failed:
2911         for (j = index; j > 0; j--) {
2912                 pwrb_context = &phwi_ctrlr->wrb_context[j];
2913                 kfree(pwrb_context->pwrb_handle_base);
2914                 kfree(pwrb_context->pwrb_handle_basestd);
2915         }
2916         return -ENOMEM;
2917 }
2918
2919 static int hwi_init_async_pdu_ctx(struct beiscsi_hba *phba)
2920 {
2921         uint8_t ulp_num;
2922         struct hwi_controller *phwi_ctrlr;
2923         struct hba_parameters *p = &phba->params;
2924         struct hwi_async_pdu_context *pasync_ctx;
2925         struct async_pdu_handle *pasync_header_h, *pasync_data_h;
2926         unsigned int index, idx, num_per_mem, num_async_data;
2927         struct be_mem_descriptor *mem_descr;
2928
2929         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
2930                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
2931
2932                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2933                         mem_descr += (HWI_MEM_ASYNC_PDU_CONTEXT_ULP0 +
2934                                      (ulp_num * MEM_DESCR_OFFSET));
2935
2936                         phwi_ctrlr = phba->phwi_ctrlr;
2937                         phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num] =
2938                                 (struct hwi_async_pdu_context *)
2939                                  mem_descr->mem_array[0].virtual_address;
2940
2941                         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
2942                         memset(pasync_ctx, 0, sizeof(*pasync_ctx));
2943
2944                         pasync_ctx->async_entry =
2945                                         (struct hwi_async_entry *)
2946                                         ((long unsigned int)pasync_ctx +
2947                                         sizeof(struct hwi_async_pdu_context));
2948
2949                         pasync_ctx->num_entries = BEISCSI_GET_CID_COUNT(phba,
2950                                                   ulp_num);
2951                         pasync_ctx->buffer_size = p->defpdu_hdr_sz;
2952
2953                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2954                         mem_descr += HWI_MEM_ASYNC_HEADER_BUF_ULP0 +
2955                                 (ulp_num * MEM_DESCR_OFFSET);
2956                         if (mem_descr->mem_array[0].virtual_address) {
2957                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2958                                             "BM_%d : hwi_init_async_pdu_ctx"
2959                                             " HWI_MEM_ASYNC_HEADER_BUF_ULP%d va=%p\n",
2960                                             ulp_num,
2961                                             mem_descr->mem_array[0].
2962                                             virtual_address);
2963                         } else
2964                                 beiscsi_log(phba, KERN_WARNING,
2965                                             BEISCSI_LOG_INIT,
2966                                             "BM_%d : No Virtual address for ULP : %d\n",
2967                                             ulp_num);
2968
2969                         pasync_ctx->async_header.va_base =
2970                                 mem_descr->mem_array[0].virtual_address;
2971
2972                         pasync_ctx->async_header.pa_base.u.a64.address =
2973                                 mem_descr->mem_array[0].
2974                                 bus_address.u.a64.address;
2975
2976                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2977                         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
2978                                      (ulp_num * MEM_DESCR_OFFSET);
2979                         if (mem_descr->mem_array[0].virtual_address) {
2980                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
2981                                             "BM_%d : hwi_init_async_pdu_ctx"
2982                                             " HWI_MEM_ASYNC_HEADER_RING_ULP%d va=%p\n",
2983                                             ulp_num,
2984                                             mem_descr->mem_array[0].
2985                                             virtual_address);
2986                         } else
2987                                 beiscsi_log(phba, KERN_WARNING,
2988                                             BEISCSI_LOG_INIT,
2989                                             "BM_%d : No Virtual address for ULP : %d\n",
2990                                             ulp_num);
2991
2992                         pasync_ctx->async_header.ring_base =
2993                                 mem_descr->mem_array[0].virtual_address;
2994
2995                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
2996                         mem_descr += HWI_MEM_ASYNC_HEADER_HANDLE_ULP0 +
2997                                      (ulp_num * MEM_DESCR_OFFSET);
2998                         if (mem_descr->mem_array[0].virtual_address) {
2999                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3000                                             "BM_%d : hwi_init_async_pdu_ctx"
3001                                             " HWI_MEM_ASYNC_HEADER_HANDLE_ULP%d va=%p\n",
3002                                             ulp_num,
3003                                             mem_descr->mem_array[0].
3004                                             virtual_address);
3005                         } else
3006                                 beiscsi_log(phba, KERN_WARNING,
3007                                             BEISCSI_LOG_INIT,
3008                                             "BM_%d : No Virtual address for ULP : %d\n",
3009                                             ulp_num);
3010
3011                         pasync_ctx->async_header.handle_base =
3012                                 mem_descr->mem_array[0].virtual_address;
3013                         pasync_ctx->async_header.writables = 0;
3014                         INIT_LIST_HEAD(&pasync_ctx->async_header.free_list);
3015
3016                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3017                         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3018                                      (ulp_num * MEM_DESCR_OFFSET);
3019                         if (mem_descr->mem_array[0].virtual_address) {
3020                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3021                                             "BM_%d : hwi_init_async_pdu_ctx"
3022                                             " HWI_MEM_ASYNC_DATA_RING_ULP%d va=%p\n",
3023                                             ulp_num,
3024                                             mem_descr->mem_array[0].
3025                                             virtual_address);
3026                         } else
3027                                 beiscsi_log(phba, KERN_WARNING,
3028                                             BEISCSI_LOG_INIT,
3029                                             "BM_%d : No Virtual address for ULP : %d\n",
3030                                             ulp_num);
3031
3032                         pasync_ctx->async_data.ring_base =
3033                                 mem_descr->mem_array[0].virtual_address;
3034
3035                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3036                         mem_descr += HWI_MEM_ASYNC_DATA_HANDLE_ULP0 +
3037                                      (ulp_num * MEM_DESCR_OFFSET);
3038                         if (!mem_descr->mem_array[0].virtual_address)
3039                                 beiscsi_log(phba, KERN_WARNING,
3040                                             BEISCSI_LOG_INIT,
3041                                             "BM_%d : No Virtual address for ULP : %d\n",
3042                                             ulp_num);
3043
3044                         pasync_ctx->async_data.handle_base =
3045                                 mem_descr->mem_array[0].virtual_address;
3046                         pasync_ctx->async_data.writables = 0;
3047                         INIT_LIST_HEAD(&pasync_ctx->async_data.free_list);
3048
3049                         pasync_header_h =
3050                                 (struct async_pdu_handle *)
3051                                 pasync_ctx->async_header.handle_base;
3052                         pasync_data_h =
3053                                 (struct async_pdu_handle *)
3054                                 pasync_ctx->async_data.handle_base;
3055
3056                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3057                         mem_descr += HWI_MEM_ASYNC_DATA_BUF_ULP0 +
3058                                      (ulp_num * MEM_DESCR_OFFSET);
3059                         if (mem_descr->mem_array[0].virtual_address) {
3060                                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3061                                             "BM_%d : hwi_init_async_pdu_ctx"
3062                                             " HWI_MEM_ASYNC_DATA_BUF_ULP%d va=%p\n",
3063                                             ulp_num,
3064                                             mem_descr->mem_array[0].
3065                                             virtual_address);
3066                         } else
3067                                 beiscsi_log(phba, KERN_WARNING,
3068                                             BEISCSI_LOG_INIT,
3069                                             "BM_%d : No Virtual address for ULP : %d\n",
3070                                             ulp_num);
3071
3072                         idx = 0;
3073                         pasync_ctx->async_data.va_base =
3074                                 mem_descr->mem_array[idx].virtual_address;
3075                         pasync_ctx->async_data.pa_base.u.a64.address =
3076                                 mem_descr->mem_array[idx].
3077                                 bus_address.u.a64.address;
3078
3079                         num_async_data = ((mem_descr->mem_array[idx].size) /
3080                                         phba->params.defpdu_data_sz);
3081                         num_per_mem = 0;
3082
3083                         for (index = 0; index < BEISCSI_GET_CID_COUNT
3084                                         (phba, ulp_num); index++) {
3085                                 pasync_header_h->cri = -1;
3086                                 pasync_header_h->index = (char)index;
3087                                 INIT_LIST_HEAD(&pasync_header_h->link);
3088                                 pasync_header_h->pbuffer =
3089                                         (void *)((unsigned long)
3090                                                  (pasync_ctx->
3091                                                   async_header.va_base) +
3092                                                  (p->defpdu_hdr_sz * index));
3093
3094                                 pasync_header_h->pa.u.a64.address =
3095                                         pasync_ctx->async_header.pa_base.u.a64.
3096                                         address + (p->defpdu_hdr_sz * index);
3097
3098                                 list_add_tail(&pasync_header_h->link,
3099                                               &pasync_ctx->async_header.
3100                                               free_list);
3101                                 pasync_header_h++;
3102                                 pasync_ctx->async_header.free_entries++;
3103                                 pasync_ctx->async_header.writables++;
3104
3105                                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3106                                                wait_queue.list);
3107                                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3108                                                header_busy_list);
3109                                 pasync_data_h->cri = -1;
3110                                 pasync_data_h->index = (char)index;
3111                                 INIT_LIST_HEAD(&pasync_data_h->link);
3112
3113                                 if (!num_async_data) {
3114                                         num_per_mem = 0;
3115                                         idx++;
3116                                         pasync_ctx->async_data.va_base =
3117                                                 mem_descr->mem_array[idx].
3118                                                 virtual_address;
3119                                         pasync_ctx->async_data.pa_base.u.
3120                                                 a64.address =
3121                                                 mem_descr->mem_array[idx].
3122                                                 bus_address.u.a64.address;
3123                                         num_async_data =
3124                                                 ((mem_descr->mem_array[idx].
3125                                                   size) /
3126                                                  phba->params.defpdu_data_sz);
3127                                 }
3128                                 pasync_data_h->pbuffer =
3129                                         (void *)((unsigned long)
3130                                         (pasync_ctx->async_data.va_base) +
3131                                         (p->defpdu_data_sz * num_per_mem));
3132
3133                                 pasync_data_h->pa.u.a64.address =
3134                                         pasync_ctx->async_data.pa_base.u.a64.
3135                                         address + (p->defpdu_data_sz *
3136                                         num_per_mem);
3137                                 num_per_mem++;
3138                                 num_async_data--;
3139
3140                                 list_add_tail(&pasync_data_h->link,
3141                                               &pasync_ctx->async_data.
3142                                               free_list);
3143                                 pasync_data_h++;
3144                                 pasync_ctx->async_data.free_entries++;
3145                                 pasync_ctx->async_data.writables++;
3146
3147                                 INIT_LIST_HEAD(&pasync_ctx->async_entry[index].
3148                                                data_busy_list);
3149                         }
3150
3151                         pasync_ctx->async_header.host_write_ptr = 0;
3152                         pasync_ctx->async_header.ep_read_ptr = -1;
3153                         pasync_ctx->async_data.host_write_ptr = 0;
3154                         pasync_ctx->async_data.ep_read_ptr = -1;
3155                 }
3156         }
3157
3158         return 0;
3159 }
3160
3161 static int
3162 be_sgl_create_contiguous(void *virtual_address,
3163                          u64 physical_address, u32 length,
3164                          struct be_dma_mem *sgl)
3165 {
3166         WARN_ON(!virtual_address);
3167         WARN_ON(!physical_address);
3168         WARN_ON(!length > 0);
3169         WARN_ON(!sgl);
3170
3171         sgl->va = virtual_address;
3172         sgl->dma = (unsigned long)physical_address;
3173         sgl->size = length;
3174
3175         return 0;
3176 }
3177
3178 static void be_sgl_destroy_contiguous(struct be_dma_mem *sgl)
3179 {
3180         memset(sgl, 0, sizeof(*sgl));
3181 }
3182
3183 static void
3184 hwi_build_be_sgl_arr(struct beiscsi_hba *phba,
3185                      struct mem_array *pmem, struct be_dma_mem *sgl)
3186 {
3187         if (sgl->va)
3188                 be_sgl_destroy_contiguous(sgl);
3189
3190         be_sgl_create_contiguous(pmem->virtual_address,
3191                                  pmem->bus_address.u.a64.address,
3192                                  pmem->size, sgl);
3193 }
3194
3195 static void
3196 hwi_build_be_sgl_by_offset(struct beiscsi_hba *phba,
3197                            struct mem_array *pmem, struct be_dma_mem *sgl)
3198 {
3199         if (sgl->va)
3200                 be_sgl_destroy_contiguous(sgl);
3201
3202         be_sgl_create_contiguous((unsigned char *)pmem->virtual_address,
3203                                  pmem->bus_address.u.a64.address,
3204                                  pmem->size, sgl);
3205 }
3206
3207 static int be_fill_queue(struct be_queue_info *q,
3208                 u16 len, u16 entry_size, void *vaddress)
3209 {
3210         struct be_dma_mem *mem = &q->dma_mem;
3211
3212         memset(q, 0, sizeof(*q));
3213         q->len = len;
3214         q->entry_size = entry_size;
3215         mem->size = len * entry_size;
3216         mem->va = vaddress;
3217         if (!mem->va)
3218                 return -ENOMEM;
3219         memset(mem->va, 0, mem->size);
3220         return 0;
3221 }
3222
3223 static int beiscsi_create_eqs(struct beiscsi_hba *phba,
3224                              struct hwi_context_memory *phwi_context)
3225 {
3226         unsigned int i, num_eq_pages;
3227         int ret = 0, eq_for_mcc;
3228         struct be_queue_info *eq;
3229         struct be_dma_mem *mem;
3230         void *eq_vaddress;
3231         dma_addr_t paddr;
3232
3233         num_eq_pages = PAGES_REQUIRED(phba->params.num_eq_entries * \
3234                                       sizeof(struct be_eq_entry));
3235
3236         if (phba->msix_enabled)
3237                 eq_for_mcc = 1;
3238         else
3239                 eq_for_mcc = 0;
3240         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3241                 eq = &phwi_context->be_eq[i].q;
3242                 mem = &eq->dma_mem;
3243                 phwi_context->be_eq[i].phba = phba;
3244                 eq_vaddress = pci_alloc_consistent(phba->pcidev,
3245                                                      num_eq_pages * PAGE_SIZE,
3246                                                      &paddr);
3247                 if (!eq_vaddress)
3248                         goto create_eq_error;
3249
3250                 mem->va = eq_vaddress;
3251                 ret = be_fill_queue(eq, phba->params.num_eq_entries,
3252                                     sizeof(struct be_eq_entry), eq_vaddress);
3253                 if (ret) {
3254                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3255                                     "BM_%d : be_fill_queue Failed for EQ\n");
3256                         goto create_eq_error;
3257                 }
3258
3259                 mem->dma = paddr;
3260                 ret = beiscsi_cmd_eq_create(&phba->ctrl, eq,
3261                                             phwi_context->cur_eqd);
3262                 if (ret) {
3263                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3264                                     "BM_%d : beiscsi_cmd_eq_create"
3265                                     "Failed for EQ\n");
3266                         goto create_eq_error;
3267                 }
3268
3269                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3270                             "BM_%d : eqid = %d\n",
3271                             phwi_context->be_eq[i].q.id);
3272         }
3273         return 0;
3274 create_eq_error:
3275         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3276                 eq = &phwi_context->be_eq[i].q;
3277                 mem = &eq->dma_mem;
3278                 if (mem->va)
3279                         pci_free_consistent(phba->pcidev, num_eq_pages
3280                                             * PAGE_SIZE,
3281                                             mem->va, mem->dma);
3282         }
3283         return ret;
3284 }
3285
3286 static int beiscsi_create_cqs(struct beiscsi_hba *phba,
3287                              struct hwi_context_memory *phwi_context)
3288 {
3289         unsigned int i, num_cq_pages;
3290         int ret = 0;
3291         struct be_queue_info *cq, *eq;
3292         struct be_dma_mem *mem;
3293         struct be_eq_obj *pbe_eq;
3294         void *cq_vaddress;
3295         dma_addr_t paddr;
3296
3297         num_cq_pages = PAGES_REQUIRED(phba->params.num_cq_entries * \
3298                                       sizeof(struct sol_cqe));
3299
3300         for (i = 0; i < phba->num_cpus; i++) {
3301                 cq = &phwi_context->be_cq[i];
3302                 eq = &phwi_context->be_eq[i].q;
3303                 pbe_eq = &phwi_context->be_eq[i];
3304                 pbe_eq->cq = cq;
3305                 pbe_eq->phba = phba;
3306                 mem = &cq->dma_mem;
3307                 cq_vaddress = pci_alloc_consistent(phba->pcidev,
3308                                                      num_cq_pages * PAGE_SIZE,
3309                                                      &paddr);
3310                 if (!cq_vaddress)
3311                         goto create_cq_error;
3312                 ret = be_fill_queue(cq, phba->params.num_cq_entries,
3313                                     sizeof(struct sol_cqe), cq_vaddress);
3314                 if (ret) {
3315                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3316                                     "BM_%d : be_fill_queue Failed "
3317                                     "for ISCSI CQ\n");
3318                         goto create_cq_error;
3319                 }
3320
3321                 mem->dma = paddr;
3322                 ret = beiscsi_cmd_cq_create(&phba->ctrl, cq, eq, false,
3323                                             false, 0);
3324                 if (ret) {
3325                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3326                                     "BM_%d : beiscsi_cmd_eq_create"
3327                                     "Failed for ISCSI CQ\n");
3328                         goto create_cq_error;
3329                 }
3330                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3331                             "BM_%d : iscsi cq_id is %d for eq_id %d\n"
3332                             "iSCSI CQ CREATED\n", cq->id, eq->id);
3333         }
3334         return 0;
3335
3336 create_cq_error:
3337         for (i = 0; i < phba->num_cpus; i++) {
3338                 cq = &phwi_context->be_cq[i];
3339                 mem = &cq->dma_mem;
3340                 if (mem->va)
3341                         pci_free_consistent(phba->pcidev, num_cq_pages
3342                                             * PAGE_SIZE,
3343                                             mem->va, mem->dma);
3344         }
3345         return ret;
3346
3347 }
3348
3349 static int
3350 beiscsi_create_def_hdr(struct beiscsi_hba *phba,
3351                        struct hwi_context_memory *phwi_context,
3352                        struct hwi_controller *phwi_ctrlr,
3353                        unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3354 {
3355         unsigned int idx;
3356         int ret;
3357         struct be_queue_info *dq, *cq;
3358         struct be_dma_mem *mem;
3359         struct be_mem_descriptor *mem_descr;
3360         void *dq_vaddress;
3361
3362         idx = 0;
3363         dq = &phwi_context->be_def_hdrq[ulp_num];
3364         cq = &phwi_context->be_cq[0];
3365         mem = &dq->dma_mem;
3366         mem_descr = phba->init_mem;
3367         mem_descr += HWI_MEM_ASYNC_HEADER_RING_ULP0 +
3368                     (ulp_num * MEM_DESCR_OFFSET);
3369         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3370         ret = be_fill_queue(dq, mem_descr->mem_array[0].size /
3371                             sizeof(struct phys_addr),
3372                             sizeof(struct phys_addr), dq_vaddress);
3373         if (ret) {
3374                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3375                             "BM_%d : be_fill_queue Failed for DEF PDU HDR on ULP : %d\n",
3376                             ulp_num);
3377
3378                 return ret;
3379         }
3380         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3381                                   bus_address.u.a64.address;
3382         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dq,
3383                                               def_pdu_ring_sz,
3384                                               phba->params.defpdu_hdr_sz,
3385                                               BEISCSI_DEFQ_HDR, ulp_num);
3386         if (ret) {
3387                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3388                             "BM_%d : be_cmd_create_default_pdu_queue Failed DEFHDR on ULP : %d\n",
3389                             ulp_num);
3390
3391                 return ret;
3392         }
3393
3394         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3395                     "BM_%d : iscsi hdr def pdu id for ULP : %d is %d\n",
3396                     ulp_num,
3397                     phwi_context->be_def_hdrq[ulp_num].id);
3398         hwi_post_async_buffers(phba, BEISCSI_DEFQ_HDR, ulp_num);
3399         return 0;
3400 }
3401
3402 static int
3403 beiscsi_create_def_data(struct beiscsi_hba *phba,
3404                         struct hwi_context_memory *phwi_context,
3405                         struct hwi_controller *phwi_ctrlr,
3406                         unsigned int def_pdu_ring_sz, uint8_t ulp_num)
3407 {
3408         unsigned int idx;
3409         int ret;
3410         struct be_queue_info *dataq, *cq;
3411         struct be_dma_mem *mem;
3412         struct be_mem_descriptor *mem_descr;
3413         void *dq_vaddress;
3414
3415         idx = 0;
3416         dataq = &phwi_context->be_def_dataq[ulp_num];
3417         cq = &phwi_context->be_cq[0];
3418         mem = &dataq->dma_mem;
3419         mem_descr = phba->init_mem;
3420         mem_descr += HWI_MEM_ASYNC_DATA_RING_ULP0 +
3421                     (ulp_num * MEM_DESCR_OFFSET);
3422         dq_vaddress = mem_descr->mem_array[idx].virtual_address;
3423         ret = be_fill_queue(dataq, mem_descr->mem_array[0].size /
3424                             sizeof(struct phys_addr),
3425                             sizeof(struct phys_addr), dq_vaddress);
3426         if (ret) {
3427                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3428                             "BM_%d : be_fill_queue Failed for DEF PDU "
3429                             "DATA on ULP : %d\n",
3430                             ulp_num);
3431
3432                 return ret;
3433         }
3434         mem->dma = (unsigned long)mem_descr->mem_array[idx].
3435                                   bus_address.u.a64.address;
3436         ret = be_cmd_create_default_pdu_queue(&phba->ctrl, cq, dataq,
3437                                               def_pdu_ring_sz,
3438                                               phba->params.defpdu_data_sz,
3439                                               BEISCSI_DEFQ_DATA, ulp_num);
3440         if (ret) {
3441                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3442                             "BM_%d be_cmd_create_default_pdu_queue"
3443                             " Failed for DEF PDU DATA on ULP : %d\n",
3444                             ulp_num);
3445                 return ret;
3446         }
3447
3448         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3449                     "BM_%d : iscsi def data id on ULP : %d is  %d\n",
3450                     ulp_num,
3451                     phwi_context->be_def_dataq[ulp_num].id);
3452
3453         hwi_post_async_buffers(phba, BEISCSI_DEFQ_DATA, ulp_num);
3454         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3455                     "BM_%d : DEFAULT PDU DATA RING CREATED"
3456                     "on ULP : %d\n", ulp_num);
3457
3458         return 0;
3459 }
3460
3461
3462 static int
3463 beiscsi_post_template_hdr(struct beiscsi_hba *phba)
3464 {
3465         struct be_mem_descriptor *mem_descr;
3466         struct mem_array *pm_arr;
3467         struct be_dma_mem sgl;
3468         int status, ulp_num;
3469
3470         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3471                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3472                         mem_descr = (struct be_mem_descriptor *)phba->init_mem;
3473                         mem_descr += HWI_MEM_TEMPLATE_HDR_ULP0 +
3474                                     (ulp_num * MEM_DESCR_OFFSET);
3475                         pm_arr = mem_descr->mem_array;
3476
3477                         hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3478                         status = be_cmd_iscsi_post_template_hdr(
3479                                  &phba->ctrl, &sgl);
3480
3481                         if (status != 0) {
3482                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3483                                             "BM_%d : Post Template HDR Failed for"
3484                                             "ULP_%d\n", ulp_num);
3485                                 return status;
3486                         }
3487
3488                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3489                                     "BM_%d : Template HDR Pages Posted for"
3490                                     "ULP_%d\n", ulp_num);
3491                 }
3492         }
3493         return 0;
3494 }
3495
3496 static int
3497 beiscsi_post_pages(struct beiscsi_hba *phba)
3498 {
3499         struct be_mem_descriptor *mem_descr;
3500         struct mem_array *pm_arr;
3501         unsigned int page_offset, i;
3502         struct be_dma_mem sgl;
3503         int status, ulp_num = 0;
3504
3505         mem_descr = phba->init_mem;
3506         mem_descr += HWI_MEM_SGE;
3507         pm_arr = mem_descr->mem_array;
3508
3509         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3510                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
3511                         break;
3512
3513         page_offset = (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io *
3514                         phba->fw_config.iscsi_icd_start[ulp_num]) / PAGE_SIZE;
3515         for (i = 0; i < mem_descr->num_elements; i++) {
3516                 hwi_build_be_sgl_arr(phba, pm_arr, &sgl);
3517                 status = be_cmd_iscsi_post_sgl_pages(&phba->ctrl, &sgl,
3518                                                 page_offset,
3519                                                 (pm_arr->size / PAGE_SIZE));
3520                 page_offset += pm_arr->size / PAGE_SIZE;
3521                 if (status != 0) {
3522                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3523                                     "BM_%d : post sgl failed.\n");
3524                         return status;
3525                 }
3526                 pm_arr++;
3527         }
3528         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3529                     "BM_%d : POSTED PAGES\n");
3530         return 0;
3531 }
3532
3533 static void be_queue_free(struct beiscsi_hba *phba, struct be_queue_info *q)
3534 {
3535         struct be_dma_mem *mem = &q->dma_mem;
3536         if (mem->va) {
3537                 pci_free_consistent(phba->pcidev, mem->size,
3538                         mem->va, mem->dma);
3539                 mem->va = NULL;
3540         }
3541 }
3542
3543 static int be_queue_alloc(struct beiscsi_hba *phba, struct be_queue_info *q,
3544                 u16 len, u16 entry_size)
3545 {
3546         struct be_dma_mem *mem = &q->dma_mem;
3547
3548         memset(q, 0, sizeof(*q));
3549         q->len = len;
3550         q->entry_size = entry_size;
3551         mem->size = len * entry_size;
3552         mem->va = pci_zalloc_consistent(phba->pcidev, mem->size, &mem->dma);
3553         if (!mem->va)
3554                 return -ENOMEM;
3555         return 0;
3556 }
3557
3558 static int
3559 beiscsi_create_wrb_rings(struct beiscsi_hba *phba,
3560                          struct hwi_context_memory *phwi_context,
3561                          struct hwi_controller *phwi_ctrlr)
3562 {
3563         unsigned int wrb_mem_index, offset, size, num_wrb_rings;
3564         u64 pa_addr_lo;
3565         unsigned int idx, num, i, ulp_num;
3566         struct mem_array *pwrb_arr;
3567         void *wrb_vaddr;
3568         struct be_dma_mem sgl;
3569         struct be_mem_descriptor *mem_descr;
3570         struct hwi_wrb_context *pwrb_context;
3571         int status;
3572         uint8_t ulp_count = 0, ulp_base_num = 0;
3573         uint16_t cid_count_ulp[BEISCSI_ULP_COUNT] = { 0 };
3574
3575         idx = 0;
3576         mem_descr = phba->init_mem;
3577         mem_descr += HWI_MEM_WRB;
3578         pwrb_arr = kmalloc(sizeof(*pwrb_arr) * phba->params.cxns_per_ctrl,
3579                            GFP_KERNEL);
3580         if (!pwrb_arr) {
3581                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3582                             "BM_%d : Memory alloc failed in create wrb ring.\n");
3583                 return -ENOMEM;
3584         }
3585         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3586         pa_addr_lo = mem_descr->mem_array[idx].bus_address.u.a64.address;
3587         num_wrb_rings = mem_descr->mem_array[idx].size /
3588                 (phba->params.wrbs_per_cxn * sizeof(struct iscsi_wrb));
3589
3590         for (num = 0; num < phba->params.cxns_per_ctrl; num++) {
3591                 if (num_wrb_rings) {
3592                         pwrb_arr[num].virtual_address = wrb_vaddr;
3593                         pwrb_arr[num].bus_address.u.a64.address = pa_addr_lo;
3594                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3595                                             sizeof(struct iscsi_wrb);
3596                         wrb_vaddr += pwrb_arr[num].size;
3597                         pa_addr_lo += pwrb_arr[num].size;
3598                         num_wrb_rings--;
3599                 } else {
3600                         idx++;
3601                         wrb_vaddr = mem_descr->mem_array[idx].virtual_address;
3602                         pa_addr_lo = mem_descr->mem_array[idx].\
3603                                         bus_address.u.a64.address;
3604                         num_wrb_rings = mem_descr->mem_array[idx].size /
3605                                         (phba->params.wrbs_per_cxn *
3606                                         sizeof(struct iscsi_wrb));
3607                         pwrb_arr[num].virtual_address = wrb_vaddr;
3608                         pwrb_arr[num].bus_address.u.a64.address\
3609                                                 = pa_addr_lo;
3610                         pwrb_arr[num].size = phba->params.wrbs_per_cxn *
3611                                                  sizeof(struct iscsi_wrb);
3612                         wrb_vaddr += pwrb_arr[num].size;
3613                         pa_addr_lo   += pwrb_arr[num].size;
3614                         num_wrb_rings--;
3615                 }
3616         }
3617
3618         /* Get the ULP Count */
3619         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
3620                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3621                         ulp_count++;
3622                         ulp_base_num = ulp_num;
3623                         cid_count_ulp[ulp_num] =
3624                                 BEISCSI_GET_CID_COUNT(phba, ulp_num);
3625                 }
3626
3627         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3628                 wrb_mem_index = 0;
3629                 offset = 0;
3630                 size = 0;
3631
3632                 if (ulp_count > 1) {
3633                         ulp_base_num = (ulp_base_num + 1) % BEISCSI_ULP_COUNT;
3634
3635                         if (!cid_count_ulp[ulp_base_num])
3636                                 ulp_base_num = (ulp_base_num + 1) %
3637                                                 BEISCSI_ULP_COUNT;
3638
3639                         cid_count_ulp[ulp_base_num]--;
3640                 }
3641
3642
3643                 hwi_build_be_sgl_by_offset(phba, &pwrb_arr[i], &sgl);
3644                 status = be_cmd_wrbq_create(&phba->ctrl, &sgl,
3645                                             &phwi_context->be_wrbq[i],
3646                                             &phwi_ctrlr->wrb_context[i],
3647                                             ulp_base_num);
3648                 if (status != 0) {
3649                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3650                                     "BM_%d : wrbq create failed.");
3651                         kfree(pwrb_arr);
3652                         return status;
3653                 }
3654                 pwrb_context = &phwi_ctrlr->wrb_context[i];
3655                 BE_SET_CID_TO_CRI(i, pwrb_context->cid);
3656         }
3657         kfree(pwrb_arr);
3658         return 0;
3659 }
3660
3661 static void free_wrb_handles(struct beiscsi_hba *phba)
3662 {
3663         unsigned int index;
3664         struct hwi_controller *phwi_ctrlr;
3665         struct hwi_wrb_context *pwrb_context;
3666
3667         phwi_ctrlr = phba->phwi_ctrlr;
3668         for (index = 0; index < phba->params.cxns_per_ctrl; index++) {
3669                 pwrb_context = &phwi_ctrlr->wrb_context[index];
3670                 kfree(pwrb_context->pwrb_handle_base);
3671                 kfree(pwrb_context->pwrb_handle_basestd);
3672         }
3673 }
3674
3675 static void be_mcc_queues_destroy(struct beiscsi_hba *phba)
3676 {
3677         struct be_queue_info *q;
3678         struct be_ctrl_info *ctrl = &phba->ctrl;
3679
3680         q = &phba->ctrl.mcc_obj.q;
3681         if (q->created)
3682                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_MCCQ);
3683         be_queue_free(phba, q);
3684
3685         q = &phba->ctrl.mcc_obj.cq;
3686         if (q->created)
3687                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3688         be_queue_free(phba, q);
3689 }
3690
3691 static void hwi_cleanup(struct beiscsi_hba *phba)
3692 {
3693         struct be_queue_info *q;
3694         struct be_ctrl_info *ctrl = &phba->ctrl;
3695         struct hwi_controller *phwi_ctrlr;
3696         struct hwi_context_memory *phwi_context;
3697         struct hwi_async_pdu_context *pasync_ctx;
3698         int i, eq_for_mcc, ulp_num;
3699
3700         phwi_ctrlr = phba->phwi_ctrlr;
3701         phwi_context = phwi_ctrlr->phwi_ctxt;
3702
3703         be_cmd_iscsi_remove_template_hdr(ctrl);
3704
3705         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
3706                 q = &phwi_context->be_wrbq[i];
3707                 if (q->created)
3708                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_WRBQ);
3709         }
3710         kfree(phwi_context->be_wrbq);
3711         free_wrb_handles(phba);
3712
3713         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3714                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3715
3716                         q = &phwi_context->be_def_hdrq[ulp_num];
3717                         if (q->created)
3718                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3719
3720                         q = &phwi_context->be_def_dataq[ulp_num];
3721                         if (q->created)
3722                                 beiscsi_cmd_q_destroy(ctrl, q, QTYPE_DPDUQ);
3723
3724                         pasync_ctx = phwi_ctrlr->phwi_ctxt->pasync_ctx[ulp_num];
3725                 }
3726         }
3727
3728         beiscsi_cmd_q_destroy(ctrl, NULL, QTYPE_SGL);
3729
3730         for (i = 0; i < (phba->num_cpus); i++) {
3731                 q = &phwi_context->be_cq[i];
3732                 if (q->created)
3733                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_CQ);
3734         }
3735
3736         be_mcc_queues_destroy(phba);
3737         if (phba->msix_enabled)
3738                 eq_for_mcc = 1;
3739         else
3740                 eq_for_mcc = 0;
3741         for (i = 0; i < (phba->num_cpus + eq_for_mcc); i++) {
3742                 q = &phwi_context->be_eq[i].q;
3743                 if (q->created)
3744                         beiscsi_cmd_q_destroy(ctrl, q, QTYPE_EQ);
3745         }
3746         be_cmd_fw_uninit(ctrl);
3747 }
3748
3749 static int be_mcc_queues_create(struct beiscsi_hba *phba,
3750                                 struct hwi_context_memory *phwi_context)
3751 {
3752         struct be_queue_info *q, *cq;
3753         struct be_ctrl_info *ctrl = &phba->ctrl;
3754
3755         /* Alloc MCC compl queue */
3756         cq = &phba->ctrl.mcc_obj.cq;
3757         if (be_queue_alloc(phba, cq, MCC_CQ_LEN,
3758                         sizeof(struct be_mcc_compl)))
3759                 goto err;
3760         /* Ask BE to create MCC compl queue; */
3761         if (phba->msix_enabled) {
3762                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq
3763                                          [phba->num_cpus].q, false, true, 0))
3764                 goto mcc_cq_free;
3765         } else {
3766                 if (beiscsi_cmd_cq_create(ctrl, cq, &phwi_context->be_eq[0].q,
3767                                           false, true, 0))
3768                 goto mcc_cq_free;
3769         }
3770
3771         /* Alloc MCC queue */
3772         q = &phba->ctrl.mcc_obj.q;
3773         if (be_queue_alloc(phba, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
3774                 goto mcc_cq_destroy;
3775
3776         /* Ask BE to create MCC queue */
3777         if (beiscsi_cmd_mccq_create(phba, q, cq))
3778                 goto mcc_q_free;
3779
3780         return 0;
3781
3782 mcc_q_free:
3783         be_queue_free(phba, q);
3784 mcc_cq_destroy:
3785         beiscsi_cmd_q_destroy(ctrl, cq, QTYPE_CQ);
3786 mcc_cq_free:
3787         be_queue_free(phba, cq);
3788 err:
3789         return -ENOMEM;
3790 }
3791
3792 /**
3793  * find_num_cpus()- Get the CPU online count
3794  * @phba: ptr to priv structure
3795  *
3796  * CPU count is used for creating EQ.
3797  **/
3798 static void find_num_cpus(struct beiscsi_hba *phba)
3799 {
3800         int  num_cpus = 0;
3801
3802         num_cpus = num_online_cpus();
3803
3804         switch (phba->generation) {
3805         case BE_GEN2:
3806         case BE_GEN3:
3807                 phba->num_cpus = (num_cpus > BEISCSI_MAX_NUM_CPUS) ?
3808                                   BEISCSI_MAX_NUM_CPUS : num_cpus;
3809                 break;
3810         case BE_GEN4:
3811                 /*
3812                  * If eqid_count == 1 fall back to
3813                  * INTX mechanism
3814                  **/
3815                 if (phba->fw_config.eqid_count == 1) {
3816                         enable_msix = 0;
3817                         phba->num_cpus = 1;
3818                         return;
3819                 }
3820
3821                 phba->num_cpus =
3822                         (num_cpus > (phba->fw_config.eqid_count - 1)) ?
3823                         (phba->fw_config.eqid_count - 1) : num_cpus;
3824                 break;
3825         default:
3826                 phba->num_cpus = 1;
3827         }
3828 }
3829
3830 static int hwi_init_port(struct beiscsi_hba *phba)
3831 {
3832         struct hwi_controller *phwi_ctrlr;
3833         struct hwi_context_memory *phwi_context;
3834         unsigned int def_pdu_ring_sz;
3835         struct be_ctrl_info *ctrl = &phba->ctrl;
3836         int status, ulp_num;
3837
3838         phwi_ctrlr = phba->phwi_ctrlr;
3839         phwi_context = phwi_ctrlr->phwi_ctxt;
3840         phwi_context->max_eqd = 128;
3841         phwi_context->min_eqd = 0;
3842         phwi_context->cur_eqd = 0;
3843         be_cmd_fw_initialize(&phba->ctrl);
3844
3845         status = beiscsi_create_eqs(phba, phwi_context);
3846         if (status != 0) {
3847                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3848                             "BM_%d : EQ not created\n");
3849                 goto error;
3850         }
3851
3852         status = be_mcc_queues_create(phba, phwi_context);
3853         if (status != 0)
3854                 goto error;
3855
3856         status = mgmt_check_supported_fw(ctrl, phba);
3857         if (status != 0) {
3858                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3859                             "BM_%d : Unsupported fw version\n");
3860                 goto error;
3861         }
3862
3863         status = beiscsi_create_cqs(phba, phwi_context);
3864         if (status != 0) {
3865                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3866                             "BM_%d : CQ not created\n");
3867                 goto error;
3868         }
3869
3870         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3871                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3872
3873                         def_pdu_ring_sz =
3874                                 BEISCSI_GET_CID_COUNT(phba, ulp_num) *
3875                                 sizeof(struct phys_addr);
3876
3877                         status = beiscsi_create_def_hdr(phba, phwi_context,
3878                                                         phwi_ctrlr,
3879                                                         def_pdu_ring_sz,
3880                                                         ulp_num);
3881                         if (status != 0) {
3882                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3883                                             "BM_%d : Default Header not created for ULP : %d\n",
3884                                             ulp_num);
3885                                 goto error;
3886                         }
3887
3888                         status = beiscsi_create_def_data(phba, phwi_context,
3889                                                          phwi_ctrlr,
3890                                                          def_pdu_ring_sz,
3891                                                          ulp_num);
3892                         if (status != 0) {
3893                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3894                                             "BM_%d : Default Data not created for ULP : %d\n",
3895                                             ulp_num);
3896                                 goto error;
3897                         }
3898                 }
3899         }
3900
3901         status = beiscsi_post_pages(phba);
3902         if (status != 0) {
3903                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3904                             "BM_%d : Post SGL Pages Failed\n");
3905                 goto error;
3906         }
3907
3908         status = beiscsi_post_template_hdr(phba);
3909         if (status != 0) {
3910                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3911                             "BM_%d : Template HDR Posting for CXN Failed\n");
3912         }
3913
3914         status = beiscsi_create_wrb_rings(phba, phwi_context, phwi_ctrlr);
3915         if (status != 0) {
3916                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3917                             "BM_%d : WRB Rings not created\n");
3918                 goto error;
3919         }
3920
3921         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
3922                 uint16_t async_arr_idx = 0;
3923
3924                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported)) {
3925                         uint16_t cri = 0;
3926                         struct hwi_async_pdu_context *pasync_ctx;
3927
3928                         pasync_ctx = HWI_GET_ASYNC_PDU_CTX(
3929                                      phwi_ctrlr, ulp_num);
3930                         for (cri = 0; cri <
3931                              phba->params.cxns_per_ctrl; cri++) {
3932                                 if (ulp_num == BEISCSI_GET_ULP_FROM_CRI
3933                                                (phwi_ctrlr, cri))
3934                                         pasync_ctx->cid_to_async_cri_map[
3935                                         phwi_ctrlr->wrb_context[cri].cid] =
3936                                         async_arr_idx++;
3937                         }
3938                 }
3939         }
3940
3941         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3942                     "BM_%d : hwi_init_port success\n");
3943         return 0;
3944
3945 error:
3946         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3947                     "BM_%d : hwi_init_port failed");
3948         hwi_cleanup(phba);
3949         return status;
3950 }
3951
3952 static int hwi_init_controller(struct beiscsi_hba *phba)
3953 {
3954         struct hwi_controller *phwi_ctrlr;
3955
3956         phwi_ctrlr = phba->phwi_ctrlr;
3957         if (1 == phba->init_mem[HWI_MEM_ADDN_CONTEXT].num_elements) {
3958                 phwi_ctrlr->phwi_ctxt = (struct hwi_context_memory *)phba->
3959                     init_mem[HWI_MEM_ADDN_CONTEXT].mem_array[0].virtual_address;
3960                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
3961                             "BM_%d :  phwi_ctrlr->phwi_ctxt=%p\n",
3962                             phwi_ctrlr->phwi_ctxt);
3963         } else {
3964                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3965                             "BM_%d : HWI_MEM_ADDN_CONTEXT is more "
3966                             "than one element.Failing to load\n");
3967                 return -ENOMEM;
3968         }
3969
3970         iscsi_init_global_templates(phba);
3971         if (beiscsi_init_wrb_handle(phba))
3972                 return -ENOMEM;
3973
3974         if (hwi_init_async_pdu_ctx(phba)) {
3975                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3976                             "BM_%d : hwi_init_async_pdu_ctx failed\n");
3977                 return -ENOMEM;
3978         }
3979
3980         if (hwi_init_port(phba) != 0) {
3981                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
3982                             "BM_%d : hwi_init_controller failed\n");
3983
3984                 return -ENOMEM;
3985         }
3986         return 0;
3987 }
3988
3989 static void beiscsi_free_mem(struct beiscsi_hba *phba)
3990 {
3991         struct be_mem_descriptor *mem_descr;
3992         int i, j;
3993
3994         mem_descr = phba->init_mem;
3995         i = 0;
3996         j = 0;
3997         for (i = 0; i < SE_MEM_MAX; i++) {
3998                 for (j = mem_descr->num_elements; j > 0; j--) {
3999                         pci_free_consistent(phba->pcidev,
4000                           mem_descr->mem_array[j - 1].size,
4001                           mem_descr->mem_array[j - 1].virtual_address,
4002                           (unsigned long)mem_descr->mem_array[j - 1].
4003                           bus_address.u.a64.address);
4004                 }
4005
4006                 kfree(mem_descr->mem_array);
4007                 mem_descr++;
4008         }
4009         kfree(phba->init_mem);
4010         kfree(phba->phwi_ctrlr->wrb_context);
4011         kfree(phba->phwi_ctrlr);
4012 }
4013
4014 static int beiscsi_init_controller(struct beiscsi_hba *phba)
4015 {
4016         int ret = -ENOMEM;
4017
4018         ret = beiscsi_get_memory(phba);
4019         if (ret < 0) {
4020                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4021                             "BM_%d : beiscsi_dev_probe -"
4022                             "Failed in beiscsi_alloc_memory\n");
4023                 return ret;
4024         }
4025
4026         ret = hwi_init_controller(phba);
4027         if (ret)
4028                 goto free_init;
4029         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4030                     "BM_%d : Return success from beiscsi_init_controller");
4031
4032         return 0;
4033
4034 free_init:
4035         beiscsi_free_mem(phba);
4036         return ret;
4037 }
4038
4039 static int beiscsi_init_sgl_handle(struct beiscsi_hba *phba)
4040 {
4041         struct be_mem_descriptor *mem_descr_sglh, *mem_descr_sg;
4042         struct sgl_handle *psgl_handle;
4043         struct iscsi_sge *pfrag;
4044         unsigned int arr_index, i, idx;
4045         unsigned int ulp_icd_start, ulp_num = 0;
4046
4047         phba->io_sgl_hndl_avbl = 0;
4048         phba->eh_sgl_hndl_avbl = 0;
4049
4050         mem_descr_sglh = phba->init_mem;
4051         mem_descr_sglh += HWI_MEM_SGLH;
4052         if (1 == mem_descr_sglh->num_elements) {
4053                 phba->io_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4054                                                  phba->params.ios_per_ctrl,
4055                                                  GFP_KERNEL);
4056                 if (!phba->io_sgl_hndl_base) {
4057                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4058                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
4059                         return -ENOMEM;
4060                 }
4061                 phba->eh_sgl_hndl_base = kzalloc(sizeof(struct sgl_handle *) *
4062                                                  (phba->params.icds_per_ctrl -
4063                                                  phba->params.ios_per_ctrl),
4064                                                  GFP_KERNEL);
4065                 if (!phba->eh_sgl_hndl_base) {
4066                         kfree(phba->io_sgl_hndl_base);
4067                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4068                                     "BM_%d : Mem Alloc Failed. Failing to load\n");
4069                         return -ENOMEM;
4070                 }
4071         } else {
4072                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4073                             "BM_%d : HWI_MEM_SGLH is more than one element."
4074                             "Failing to load\n");
4075                 return -ENOMEM;
4076         }
4077
4078         arr_index = 0;
4079         idx = 0;
4080         while (idx < mem_descr_sglh->num_elements) {
4081                 psgl_handle = mem_descr_sglh->mem_array[idx].virtual_address;
4082
4083                 for (i = 0; i < (mem_descr_sglh->mem_array[idx].size /
4084                       sizeof(struct sgl_handle)); i++) {
4085                         if (arr_index < phba->params.ios_per_ctrl) {
4086                                 phba->io_sgl_hndl_base[arr_index] = psgl_handle;
4087                                 phba->io_sgl_hndl_avbl++;
4088                                 arr_index++;
4089                         } else {
4090                                 phba->eh_sgl_hndl_base[arr_index -
4091                                         phba->params.ios_per_ctrl] =
4092                                                                 psgl_handle;
4093                                 arr_index++;
4094                                 phba->eh_sgl_hndl_avbl++;
4095                         }
4096                         psgl_handle++;
4097                 }
4098                 idx++;
4099         }
4100         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4101                     "BM_%d : phba->io_sgl_hndl_avbl=%d"
4102                     "phba->eh_sgl_hndl_avbl=%d\n",
4103                     phba->io_sgl_hndl_avbl,
4104                     phba->eh_sgl_hndl_avbl);
4105
4106         mem_descr_sg = phba->init_mem;
4107         mem_descr_sg += HWI_MEM_SGE;
4108         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4109                     "\n BM_%d : mem_descr_sg->num_elements=%d\n",
4110                     mem_descr_sg->num_elements);
4111
4112         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++)
4113                 if (test_bit(ulp_num, &phba->fw_config.ulp_supported))
4114                         break;
4115
4116         ulp_icd_start = phba->fw_config.iscsi_icd_start[ulp_num];
4117
4118         arr_index = 0;
4119         idx = 0;
4120         while (idx < mem_descr_sg->num_elements) {
4121                 pfrag = mem_descr_sg->mem_array[idx].virtual_address;
4122
4123                 for (i = 0;
4124                      i < (mem_descr_sg->mem_array[idx].size) /
4125                      (sizeof(struct iscsi_sge) * phba->params.num_sge_per_io);
4126                      i++) {
4127                         if (arr_index < phba->params.ios_per_ctrl)
4128                                 psgl_handle = phba->io_sgl_hndl_base[arr_index];
4129                         else
4130                                 psgl_handle = phba->eh_sgl_hndl_base[arr_index -
4131                                                 phba->params.ios_per_ctrl];
4132                         psgl_handle->pfrag = pfrag;
4133                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_hi, pfrag, 0);
4134                         AMAP_SET_BITS(struct amap_iscsi_sge, addr_lo, pfrag, 0);
4135                         pfrag += phba->params.num_sge_per_io;
4136                         psgl_handle->sgl_index = ulp_icd_start + arr_index++;
4137                 }
4138                 idx++;
4139         }
4140         phba->io_sgl_free_index = 0;
4141         phba->io_sgl_alloc_index = 0;
4142         phba->eh_sgl_free_index = 0;
4143         phba->eh_sgl_alloc_index = 0;
4144         return 0;
4145 }
4146
4147 static int hba_setup_cid_tbls(struct beiscsi_hba *phba)
4148 {
4149         int ret;
4150         uint16_t i, ulp_num;
4151         struct ulp_cid_info *ptr_cid_info = NULL;
4152
4153         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4154                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4155                         ptr_cid_info = kzalloc(sizeof(struct ulp_cid_info),
4156                                                GFP_KERNEL);
4157
4158                         if (!ptr_cid_info) {
4159                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4160                                             "BM_%d : Failed to allocate memory"
4161                                             "for ULP_CID_INFO for ULP : %d\n",
4162                                             ulp_num);
4163                                 ret = -ENOMEM;
4164                                 goto free_memory;
4165
4166                         }
4167
4168                         /* Allocate memory for CID array */
4169                         ptr_cid_info->cid_array = kzalloc(sizeof(void *) *
4170                                                   BEISCSI_GET_CID_COUNT(phba,
4171                                                   ulp_num), GFP_KERNEL);
4172                         if (!ptr_cid_info->cid_array) {
4173                                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4174                                             "BM_%d : Failed to allocate memory"
4175                                             "for CID_ARRAY for ULP : %d\n",
4176                                             ulp_num);
4177                                 kfree(ptr_cid_info);
4178                                 ptr_cid_info = NULL;
4179                                 ret = -ENOMEM;
4180
4181                                 goto free_memory;
4182                         }
4183                         ptr_cid_info->avlbl_cids = BEISCSI_GET_CID_COUNT(
4184                                                    phba, ulp_num);
4185
4186                         /* Save the cid_info_array ptr */
4187                         phba->cid_array_info[ulp_num] = ptr_cid_info;
4188                 }
4189         }
4190         phba->ep_array = kzalloc(sizeof(struct iscsi_endpoint *) *
4191                                  phba->params.cxns_per_ctrl, GFP_KERNEL);
4192         if (!phba->ep_array) {
4193                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4194                             "BM_%d : Failed to allocate memory in "
4195                             "hba_setup_cid_tbls\n");
4196                 ret = -ENOMEM;
4197
4198                 goto free_memory;
4199         }
4200
4201         phba->conn_table = kzalloc(sizeof(struct beiscsi_conn *) *
4202                                    phba->params.cxns_per_ctrl, GFP_KERNEL);
4203         if (!phba->conn_table) {
4204                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4205                             "BM_%d : Failed to allocate memory in"
4206                             "hba_setup_cid_tbls\n");
4207
4208                 kfree(phba->ep_array);
4209                 phba->ep_array = NULL;
4210                 ret = -ENOMEM;
4211
4212                 goto free_memory;
4213         }
4214
4215         for (i = 0; i < phba->params.cxns_per_ctrl; i++) {
4216                 ulp_num = phba->phwi_ctrlr->wrb_context[i].ulp_num;
4217
4218                 ptr_cid_info = phba->cid_array_info[ulp_num];
4219                 ptr_cid_info->cid_array[ptr_cid_info->cid_alloc++] =
4220                         phba->phwi_ctrlr->wrb_context[i].cid;
4221
4222         }
4223
4224         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4225                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4226                         ptr_cid_info = phba->cid_array_info[ulp_num];
4227
4228                         ptr_cid_info->cid_alloc = 0;
4229                         ptr_cid_info->cid_free = 0;
4230                 }
4231         }
4232         return 0;
4233
4234 free_memory:
4235         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4236                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4237                         ptr_cid_info = phba->cid_array_info[ulp_num];
4238
4239                         if (ptr_cid_info) {
4240                                 kfree(ptr_cid_info->cid_array);
4241                                 kfree(ptr_cid_info);
4242                                 phba->cid_array_info[ulp_num] = NULL;
4243                         }
4244                 }
4245         }
4246
4247         return ret;
4248 }
4249
4250 static void hwi_enable_intr(struct beiscsi_hba *phba)
4251 {
4252         struct be_ctrl_info *ctrl = &phba->ctrl;
4253         struct hwi_controller *phwi_ctrlr;
4254         struct hwi_context_memory *phwi_context;
4255         struct be_queue_info *eq;
4256         u8 __iomem *addr;
4257         u32 reg, i;
4258         u32 enabled;
4259
4260         phwi_ctrlr = phba->phwi_ctrlr;
4261         phwi_context = phwi_ctrlr->phwi_ctxt;
4262
4263         addr = (u8 __iomem *) ((u8 __iomem *) ctrl->pcicfg +
4264                         PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET);
4265         reg = ioread32(addr);
4266
4267         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4268         if (!enabled) {
4269                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4270                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4271                             "BM_%d : reg =x%08x addr=%p\n", reg, addr);
4272                 iowrite32(reg, addr);
4273         }
4274
4275         if (!phba->msix_enabled) {
4276                 eq = &phwi_context->be_eq[0].q;
4277                 beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4278                             "BM_%d : eq->id=%d\n", eq->id);
4279
4280                 hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4281         } else {
4282                 for (i = 0; i <= phba->num_cpus; i++) {
4283                         eq = &phwi_context->be_eq[i].q;
4284                         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
4285                                     "BM_%d : eq->id=%d\n", eq->id);
4286                         hwi_ring_eq_db(phba, eq->id, 0, 0, 1, 1);
4287                 }
4288         }
4289 }
4290
4291 static void hwi_disable_intr(struct beiscsi_hba *phba)
4292 {
4293         struct be_ctrl_info *ctrl = &phba->ctrl;
4294
4295         u8 __iomem *addr = ctrl->pcicfg + PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET;
4296         u32 reg = ioread32(addr);
4297
4298         u32 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4299         if (enabled) {
4300                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
4301                 iowrite32(reg, addr);
4302         } else
4303                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
4304                             "BM_%d : In hwi_disable_intr, Already Disabled\n");
4305 }
4306
4307 /**
4308  * beiscsi_get_boot_info()- Get the boot session info
4309  * @phba: The device priv structure instance
4310  *
4311  * Get the boot target info and store in driver priv structure
4312  *
4313  * return values
4314  *      Success: 0
4315  *      Failure: Non-Zero Value
4316  **/
4317 static int beiscsi_get_boot_info(struct beiscsi_hba *phba)
4318 {
4319         struct be_cmd_get_session_resp *session_resp;
4320         struct be_dma_mem nonemb_cmd;
4321         unsigned int tag;
4322         unsigned int s_handle;
4323         int ret = -ENOMEM;
4324
4325         /* Get the session handle of the boot target */
4326         ret = be_mgmt_get_boot_shandle(phba, &s_handle);
4327         if (ret) {
4328                 beiscsi_log(phba, KERN_ERR,
4329                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4330                             "BM_%d : No boot session\n");
4331                 return ret;
4332         }
4333         nonemb_cmd.va = pci_zalloc_consistent(phba->ctrl.pdev,
4334                                               sizeof(*session_resp),
4335                                               &nonemb_cmd.dma);
4336         if (nonemb_cmd.va == NULL) {
4337                 beiscsi_log(phba, KERN_ERR,
4338                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4339                             "BM_%d : Failed to allocate memory for"
4340                             "beiscsi_get_session_info\n");
4341
4342                 return -ENOMEM;
4343         }
4344
4345         tag = mgmt_get_session_info(phba, s_handle,
4346                                     &nonemb_cmd);
4347         if (!tag) {
4348                 beiscsi_log(phba, KERN_ERR,
4349                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4350                             "BM_%d : beiscsi_get_session_info"
4351                             " Failed\n");
4352
4353                 goto boot_freemem;
4354         }
4355
4356         ret = beiscsi_mccq_compl(phba, tag, NULL, &nonemb_cmd);
4357         if (ret) {
4358                 beiscsi_log(phba, KERN_ERR,
4359                             BEISCSI_LOG_INIT | BEISCSI_LOG_CONFIG,
4360                             "BM_%d : beiscsi_get_session_info Failed");
4361
4362                 if (ret != -EBUSY)
4363                         goto boot_freemem;
4364                 else
4365                         return ret;
4366         }
4367
4368         session_resp = nonemb_cmd.va ;
4369
4370         memcpy(&phba->boot_sess, &session_resp->session_info,
4371                sizeof(struct mgmt_session_info));
4372         ret = 0;
4373
4374 boot_freemem:
4375         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
4376                     nonemb_cmd.va, nonemb_cmd.dma);
4377         return ret;
4378 }
4379
4380 static void beiscsi_boot_release(void *data)
4381 {
4382         struct beiscsi_hba *phba = data;
4383
4384         scsi_host_put(phba->shost);
4385 }
4386
4387 static int beiscsi_setup_boot_info(struct beiscsi_hba *phba)
4388 {
4389         struct iscsi_boot_kobj *boot_kobj;
4390
4391         /* it has been created previously */
4392         if (phba->boot_kset)
4393                 return 0;
4394
4395         /* get boot info using mgmt cmd */
4396         if (beiscsi_get_boot_info(phba))
4397                 /* Try to see if we can carry on without this */
4398                 return 0;
4399
4400         phba->boot_kset = iscsi_boot_create_host_kset(phba->shost->host_no);
4401         if (!phba->boot_kset)
4402                 return -ENOMEM;
4403
4404         /* get a ref because the show function will ref the phba */
4405         if (!scsi_host_get(phba->shost))
4406                 goto free_kset;
4407         boot_kobj = iscsi_boot_create_target(phba->boot_kset, 0, phba,
4408                                              beiscsi_show_boot_tgt_info,
4409                                              beiscsi_tgt_get_attr_visibility,
4410                                              beiscsi_boot_release);
4411         if (!boot_kobj)
4412                 goto put_shost;
4413
4414         if (!scsi_host_get(phba->shost))
4415                 goto free_kset;
4416         boot_kobj = iscsi_boot_create_initiator(phba->boot_kset, 0, phba,
4417                                                 beiscsi_show_boot_ini_info,
4418                                                 beiscsi_ini_get_attr_visibility,
4419                                                 beiscsi_boot_release);
4420         if (!boot_kobj)
4421                 goto put_shost;
4422
4423         if (!scsi_host_get(phba->shost))
4424                 goto free_kset;
4425         boot_kobj = iscsi_boot_create_ethernet(phba->boot_kset, 0, phba,
4426                                                beiscsi_show_boot_eth_info,
4427                                                beiscsi_eth_get_attr_visibility,
4428                                                beiscsi_boot_release);
4429         if (!boot_kobj)
4430                 goto put_shost;
4431         return 0;
4432
4433 put_shost:
4434         scsi_host_put(phba->shost);
4435 free_kset:
4436         iscsi_boot_destroy_kset(phba->boot_kset);
4437         return -ENOMEM;
4438 }
4439
4440 static int beiscsi_init_port(struct beiscsi_hba *phba)
4441 {
4442         int ret;
4443
4444         ret = beiscsi_init_controller(phba);
4445         if (ret < 0) {
4446                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4447                             "BM_%d : beiscsi_dev_probe - Failed in"
4448                             "beiscsi_init_controller\n");
4449                 return ret;
4450         }
4451         ret = beiscsi_init_sgl_handle(phba);
4452         if (ret < 0) {
4453                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4454                             "BM_%d : beiscsi_dev_probe - Failed in"
4455                             "beiscsi_init_sgl_handle\n");
4456                 goto do_cleanup_ctrlr;
4457         }
4458
4459         if (hba_setup_cid_tbls(phba)) {
4460                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
4461                             "BM_%d : Failed in hba_setup_cid_tbls\n");
4462                 kfree(phba->io_sgl_hndl_base);
4463                 kfree(phba->eh_sgl_hndl_base);
4464                 goto do_cleanup_ctrlr;
4465         }
4466
4467         return ret;
4468
4469 do_cleanup_ctrlr:
4470         hwi_cleanup(phba);
4471         return ret;
4472 }
4473
4474 static void hwi_purge_eq(struct beiscsi_hba *phba)
4475 {
4476         struct hwi_controller *phwi_ctrlr;
4477         struct hwi_context_memory *phwi_context;
4478         struct be_queue_info *eq;
4479         struct be_eq_entry *eqe = NULL;
4480         int i, eq_msix;
4481         unsigned int num_processed;
4482
4483         phwi_ctrlr = phba->phwi_ctrlr;
4484         phwi_context = phwi_ctrlr->phwi_ctxt;
4485         if (phba->msix_enabled)
4486                 eq_msix = 1;
4487         else
4488                 eq_msix = 0;
4489
4490         for (i = 0; i < (phba->num_cpus + eq_msix); i++) {
4491                 eq = &phwi_context->be_eq[i].q;
4492                 eqe = queue_tail_node(eq);
4493                 num_processed = 0;
4494                 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
4495                                         & EQE_VALID_MASK) {
4496                         AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
4497                         queue_tail_inc(eq);
4498                         eqe = queue_tail_node(eq);
4499                         num_processed++;
4500                 }
4501
4502                 if (num_processed)
4503                         hwi_ring_eq_db(phba, eq->id, 1, num_processed, 1, 1);
4504         }
4505 }
4506
4507 static void beiscsi_clean_port(struct beiscsi_hba *phba)
4508 {
4509         int mgmt_status, ulp_num;
4510         struct ulp_cid_info *ptr_cid_info = NULL;
4511
4512         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4513                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4514                         mgmt_status = mgmt_epfw_cleanup(phba, ulp_num);
4515                         if (mgmt_status)
4516                                 beiscsi_log(phba, KERN_WARNING,
4517                                             BEISCSI_LOG_INIT,
4518                                             "BM_%d : mgmt_epfw_cleanup FAILED"
4519                                             " for ULP_%d\n", ulp_num);
4520                 }
4521         }
4522
4523         hwi_purge_eq(phba);
4524         hwi_cleanup(phba);
4525         kfree(phba->io_sgl_hndl_base);
4526         kfree(phba->eh_sgl_hndl_base);
4527         kfree(phba->ep_array);
4528         kfree(phba->conn_table);
4529
4530         for (ulp_num = 0; ulp_num < BEISCSI_ULP_COUNT; ulp_num++) {
4531                 if (test_bit(ulp_num, (void *)&phba->fw_config.ulp_supported)) {
4532                         ptr_cid_info = phba->cid_array_info[ulp_num];
4533
4534                         if (ptr_cid_info) {
4535                                 kfree(ptr_cid_info->cid_array);
4536                                 kfree(ptr_cid_info);
4537                                 phba->cid_array_info[ulp_num] = NULL;
4538                         }
4539                 }
4540         }
4541
4542 }
4543
4544 /**
4545  * beiscsi_free_mgmt_task_handles()- Free driver CXN resources
4546  * @beiscsi_conn: ptr to the conn to be cleaned up
4547  * @task: ptr to iscsi_task resource to be freed.
4548  *
4549  * Free driver mgmt resources binded to CXN.
4550  **/
4551 void
4552 beiscsi_free_mgmt_task_handles(struct beiscsi_conn *beiscsi_conn,
4553                                 struct iscsi_task *task)
4554 {
4555         struct beiscsi_io_task *io_task;
4556         struct beiscsi_hba *phba = beiscsi_conn->phba;
4557         struct hwi_wrb_context *pwrb_context;
4558         struct hwi_controller *phwi_ctrlr;
4559         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4560                                 beiscsi_conn->beiscsi_conn_cid);
4561
4562         phwi_ctrlr = phba->phwi_ctrlr;
4563         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4564
4565         io_task = task->dd_data;
4566
4567         if (io_task->pwrb_handle) {
4568                 memset(io_task->pwrb_handle->pwrb, 0,
4569                        sizeof(struct iscsi_wrb));
4570                 free_wrb_handle(phba, pwrb_context,
4571                                 io_task->pwrb_handle);
4572                 io_task->pwrb_handle = NULL;
4573         }
4574
4575         if (io_task->psgl_handle) {
4576                 spin_lock_bh(&phba->mgmt_sgl_lock);
4577                 free_mgmt_sgl_handle(phba,
4578                                      io_task->psgl_handle);
4579                 io_task->psgl_handle = NULL;
4580                 spin_unlock_bh(&phba->mgmt_sgl_lock);
4581         }
4582
4583         if (io_task->mtask_addr)
4584                 pci_unmap_single(phba->pcidev,
4585                                  io_task->mtask_addr,
4586                                  io_task->mtask_data_count,
4587                                  PCI_DMA_TODEVICE);
4588 }
4589
4590 /**
4591  * beiscsi_cleanup_task()- Free driver resources of the task
4592  * @task: ptr to the iscsi task
4593  *
4594  **/
4595 static void beiscsi_cleanup_task(struct iscsi_task *task)
4596 {
4597         struct beiscsi_io_task *io_task = task->dd_data;
4598         struct iscsi_conn *conn = task->conn;
4599         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4600         struct beiscsi_hba *phba = beiscsi_conn->phba;
4601         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4602         struct hwi_wrb_context *pwrb_context;
4603         struct hwi_controller *phwi_ctrlr;
4604         uint16_t cri_index = BE_GET_CRI_FROM_CID(
4605                              beiscsi_conn->beiscsi_conn_cid);
4606
4607         phwi_ctrlr = phba->phwi_ctrlr;
4608         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4609
4610         if (io_task->cmd_bhs) {
4611                 pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4612                               io_task->bhs_pa.u.a64.address);
4613                 io_task->cmd_bhs = NULL;
4614         }
4615
4616         if (task->sc) {
4617                 if (io_task->pwrb_handle) {
4618                         free_wrb_handle(phba, pwrb_context,
4619                                         io_task->pwrb_handle);
4620                         io_task->pwrb_handle = NULL;
4621                 }
4622
4623                 if (io_task->psgl_handle) {
4624                         spin_lock(&phba->io_sgl_lock);
4625                         free_io_sgl_handle(phba, io_task->psgl_handle);
4626                         spin_unlock(&phba->io_sgl_lock);
4627                         io_task->psgl_handle = NULL;
4628                 }
4629
4630                 if (io_task->scsi_cmnd) {
4631                         scsi_dma_unmap(io_task->scsi_cmnd);
4632                         io_task->scsi_cmnd = NULL;
4633                 }
4634         } else {
4635                 if (!beiscsi_conn->login_in_progress)
4636                         beiscsi_free_mgmt_task_handles(beiscsi_conn, task);
4637         }
4638 }
4639
4640 void
4641 beiscsi_offload_connection(struct beiscsi_conn *beiscsi_conn,
4642                            struct beiscsi_offload_params *params)
4643 {
4644         struct wrb_handle *pwrb_handle;
4645         struct beiscsi_hba *phba = beiscsi_conn->phba;
4646         struct iscsi_task *task = beiscsi_conn->task;
4647         struct iscsi_session *session = task->conn->session;
4648         u32 doorbell = 0;
4649
4650         /*
4651          * We can always use 0 here because it is reserved by libiscsi for
4652          * login/startup related tasks.
4653          */
4654         beiscsi_conn->login_in_progress = 0;
4655         spin_lock_bh(&session->back_lock);
4656         beiscsi_cleanup_task(task);
4657         spin_unlock_bh(&session->back_lock);
4658
4659         pwrb_handle = alloc_wrb_handle(phba, beiscsi_conn->beiscsi_conn_cid);
4660
4661         /* Check for the adapter family */
4662         if (is_chip_be2_be3r(phba))
4663                 beiscsi_offload_cxn_v0(params, pwrb_handle,
4664                                        phba->init_mem);
4665         else
4666                 beiscsi_offload_cxn_v2(params, pwrb_handle);
4667
4668         be_dws_le_to_cpu(pwrb_handle->pwrb,
4669                          sizeof(struct iscsi_target_context_update_wrb));
4670
4671         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4672         doorbell |= (pwrb_handle->wrb_index & DB_DEF_PDU_WRB_INDEX_MASK)
4673                              << DB_DEF_PDU_WRB_INDEX_SHIFT;
4674         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4675         iowrite32(doorbell, phba->db_va +
4676                   beiscsi_conn->doorbell_offset);
4677 }
4678
4679 static void beiscsi_parse_pdu(struct iscsi_conn *conn, itt_t itt,
4680                               int *index, int *age)
4681 {
4682         *index = (int)itt;
4683         if (age)
4684                 *age = conn->session->age;
4685 }
4686
4687 /**
4688  * beiscsi_alloc_pdu - allocates pdu and related resources
4689  * @task: libiscsi task
4690  * @opcode: opcode of pdu for task
4691  *
4692  * This is called with the session lock held. It will allocate
4693  * the wrb and sgl if needed for the command. And it will prep
4694  * the pdu's itt. beiscsi_parse_pdu will later translate
4695  * the pdu itt to the libiscsi task itt.
4696  */
4697 static int beiscsi_alloc_pdu(struct iscsi_task *task, uint8_t opcode)
4698 {
4699         struct beiscsi_io_task *io_task = task->dd_data;
4700         struct iscsi_conn *conn = task->conn;
4701         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4702         struct beiscsi_hba *phba = beiscsi_conn->phba;
4703         struct hwi_wrb_context *pwrb_context;
4704         struct hwi_controller *phwi_ctrlr;
4705         itt_t itt;
4706         uint16_t cri_index = 0;
4707         struct beiscsi_session *beiscsi_sess = beiscsi_conn->beiscsi_sess;
4708         dma_addr_t paddr;
4709
4710         io_task->cmd_bhs = pci_pool_alloc(beiscsi_sess->bhs_pool,
4711                                           GFP_ATOMIC, &paddr);
4712         if (!io_task->cmd_bhs)
4713                 return -ENOMEM;
4714         io_task->bhs_pa.u.a64.address = paddr;
4715         io_task->libiscsi_itt = (itt_t)task->itt;
4716         io_task->conn = beiscsi_conn;
4717
4718         task->hdr = (struct iscsi_hdr *)&io_task->cmd_bhs->iscsi_hdr;
4719         task->hdr_max = sizeof(struct be_cmd_bhs);
4720         io_task->psgl_handle = NULL;
4721         io_task->pwrb_handle = NULL;
4722
4723         if (task->sc) {
4724                 spin_lock(&phba->io_sgl_lock);
4725                 io_task->psgl_handle = alloc_io_sgl_handle(phba);
4726                 spin_unlock(&phba->io_sgl_lock);
4727                 if (!io_task->psgl_handle) {
4728                         beiscsi_log(phba, KERN_ERR,
4729                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4730                                     "BM_%d : Alloc of IO_SGL_ICD Failed"
4731                                     "for the CID : %d\n",
4732                                     beiscsi_conn->beiscsi_conn_cid);
4733                         goto free_hndls;
4734                 }
4735                 io_task->pwrb_handle = alloc_wrb_handle(phba,
4736                                         beiscsi_conn->beiscsi_conn_cid);
4737                 if (!io_task->pwrb_handle) {
4738                         beiscsi_log(phba, KERN_ERR,
4739                                     BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4740                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4741                                     "for the CID : %d\n",
4742                                     beiscsi_conn->beiscsi_conn_cid);
4743                         goto free_io_hndls;
4744                 }
4745         } else {
4746                 io_task->scsi_cmnd = NULL;
4747                 if ((opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_LOGIN) {
4748                         beiscsi_conn->task = task;
4749                         if (!beiscsi_conn->login_in_progress) {
4750                                 spin_lock(&phba->mgmt_sgl_lock);
4751                                 io_task->psgl_handle = (struct sgl_handle *)
4752                                                 alloc_mgmt_sgl_handle(phba);
4753                                 spin_unlock(&phba->mgmt_sgl_lock);
4754                                 if (!io_task->psgl_handle) {
4755                                         beiscsi_log(phba, KERN_ERR,
4756                                                     BEISCSI_LOG_IO |
4757                                                     BEISCSI_LOG_CONFIG,
4758                                                     "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4759                                                     "for the CID : %d\n",
4760                                                     beiscsi_conn->
4761                                                     beiscsi_conn_cid);
4762                                         goto free_hndls;
4763                                 }
4764
4765                                 beiscsi_conn->login_in_progress = 1;
4766                                 beiscsi_conn->plogin_sgl_handle =
4767                                                         io_task->psgl_handle;
4768                                 io_task->pwrb_handle =
4769                                         alloc_wrb_handle(phba,
4770                                         beiscsi_conn->beiscsi_conn_cid);
4771                                 if (!io_task->pwrb_handle) {
4772                                         beiscsi_log(phba, KERN_ERR,
4773                                                     BEISCSI_LOG_IO |
4774                                                     BEISCSI_LOG_CONFIG,
4775                                                     "BM_%d : Alloc of WRB_HANDLE Failed"
4776                                                     "for the CID : %d\n",
4777                                                     beiscsi_conn->
4778                                                     beiscsi_conn_cid);
4779                                         goto free_mgmt_hndls;
4780                                 }
4781                                 beiscsi_conn->plogin_wrb_handle =
4782                                                         io_task->pwrb_handle;
4783
4784                         } else {
4785                                 io_task->psgl_handle =
4786                                                 beiscsi_conn->plogin_sgl_handle;
4787                                 io_task->pwrb_handle =
4788                                                 beiscsi_conn->plogin_wrb_handle;
4789                         }
4790                 } else {
4791                         spin_lock(&phba->mgmt_sgl_lock);
4792                         io_task->psgl_handle = alloc_mgmt_sgl_handle(phba);
4793                         spin_unlock(&phba->mgmt_sgl_lock);
4794                         if (!io_task->psgl_handle) {
4795                                 beiscsi_log(phba, KERN_ERR,
4796                                             BEISCSI_LOG_IO |
4797                                             BEISCSI_LOG_CONFIG,
4798                                             "BM_%d : Alloc of MGMT_SGL_ICD Failed"
4799                                             "for the CID : %d\n",
4800                                             beiscsi_conn->
4801                                             beiscsi_conn_cid);
4802                                 goto free_hndls;
4803                         }
4804                         io_task->pwrb_handle =
4805                                         alloc_wrb_handle(phba,
4806                                         beiscsi_conn->beiscsi_conn_cid);
4807                         if (!io_task->pwrb_handle) {
4808                                 beiscsi_log(phba, KERN_ERR,
4809                                             BEISCSI_LOG_IO | BEISCSI_LOG_CONFIG,
4810                                             "BM_%d : Alloc of WRB_HANDLE Failed"
4811                                             "for the CID : %d\n",
4812                                             beiscsi_conn->beiscsi_conn_cid);
4813                                 goto free_mgmt_hndls;
4814                         }
4815
4816                 }
4817         }
4818         itt = (itt_t) cpu_to_be32(((unsigned int)io_task->pwrb_handle->
4819                                  wrb_index << 16) | (unsigned int)
4820                                 (io_task->psgl_handle->sgl_index));
4821         io_task->pwrb_handle->pio_handle = task;
4822
4823         io_task->cmd_bhs->iscsi_hdr.itt = itt;
4824         return 0;
4825
4826 free_io_hndls:
4827         spin_lock(&phba->io_sgl_lock);
4828         free_io_sgl_handle(phba, io_task->psgl_handle);
4829         spin_unlock(&phba->io_sgl_lock);
4830         goto free_hndls;
4831 free_mgmt_hndls:
4832         spin_lock(&phba->mgmt_sgl_lock);
4833         free_mgmt_sgl_handle(phba, io_task->psgl_handle);
4834         io_task->psgl_handle = NULL;
4835         spin_unlock(&phba->mgmt_sgl_lock);
4836 free_hndls:
4837         phwi_ctrlr = phba->phwi_ctrlr;
4838         cri_index = BE_GET_CRI_FROM_CID(
4839         beiscsi_conn->beiscsi_conn_cid);
4840         pwrb_context = &phwi_ctrlr->wrb_context[cri_index];
4841         if (io_task->pwrb_handle)
4842                 free_wrb_handle(phba, pwrb_context, io_task->pwrb_handle);
4843         io_task->pwrb_handle = NULL;
4844         pci_pool_free(beiscsi_sess->bhs_pool, io_task->cmd_bhs,
4845                       io_task->bhs_pa.u.a64.address);
4846         io_task->cmd_bhs = NULL;
4847         return -ENOMEM;
4848 }
4849 int beiscsi_iotask_v2(struct iscsi_task *task, struct scatterlist *sg,
4850                        unsigned int num_sg, unsigned int xferlen,
4851                        unsigned int writedir)
4852 {
4853
4854         struct beiscsi_io_task *io_task = task->dd_data;
4855         struct iscsi_conn *conn = task->conn;
4856         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4857         struct beiscsi_hba *phba = beiscsi_conn->phba;
4858         struct iscsi_wrb *pwrb = NULL;
4859         unsigned int doorbell = 0;
4860
4861         pwrb = io_task->pwrb_handle->pwrb;
4862
4863         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4864         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4865
4866         if (writedir) {
4867                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4868                               INI_WR_CMD);
4869                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 1);
4870         } else {
4871                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, type, pwrb,
4872                               INI_RD_CMD);
4873                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, dsp, pwrb, 0);
4874         }
4875
4876         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb_v2,
4877                                           type, pwrb);
4878
4879         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, lun, pwrb,
4880                       cpu_to_be16(*(unsigned short *)
4881                       &io_task->cmd_bhs->iscsi_hdr.lun));
4882         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb, xferlen);
4883         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4884                       io_task->pwrb_handle->wrb_index);
4885         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4886                       be32_to_cpu(task->cmdsn));
4887         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4888                       io_task->psgl_handle->sgl_index);
4889
4890         hwi_write_sgl_v2(pwrb, sg, num_sg, io_task);
4891         AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4892                       io_task->pwrb_handle->nxt_wrb_index);
4893
4894         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4895
4896         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4897         doorbell |= (io_task->pwrb_handle->wrb_index &
4898                      DB_DEF_PDU_WRB_INDEX_MASK) <<
4899                      DB_DEF_PDU_WRB_INDEX_SHIFT;
4900         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4901         iowrite32(doorbell, phba->db_va +
4902                   beiscsi_conn->doorbell_offset);
4903         return 0;
4904 }
4905
4906 static int beiscsi_iotask(struct iscsi_task *task, struct scatterlist *sg,
4907                           unsigned int num_sg, unsigned int xferlen,
4908                           unsigned int writedir)
4909 {
4910
4911         struct beiscsi_io_task *io_task = task->dd_data;
4912         struct iscsi_conn *conn = task->conn;
4913         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4914         struct beiscsi_hba *phba = beiscsi_conn->phba;
4915         struct iscsi_wrb *pwrb = NULL;
4916         unsigned int doorbell = 0;
4917
4918         pwrb = io_task->pwrb_handle->pwrb;
4919         io_task->cmd_bhs->iscsi_hdr.exp_statsn = 0;
4920         io_task->bhs_len = sizeof(struct be_cmd_bhs);
4921
4922         if (writedir) {
4923                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4924                               INI_WR_CMD);
4925                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 1);
4926         } else {
4927                 AMAP_SET_BITS(struct amap_iscsi_wrb, type, pwrb,
4928                               INI_RD_CMD);
4929                 AMAP_SET_BITS(struct amap_iscsi_wrb, dsp, pwrb, 0);
4930         }
4931
4932         io_task->wrb_type = AMAP_GET_BITS(struct amap_iscsi_wrb,
4933                                           type, pwrb);
4934
4935         AMAP_SET_BITS(struct amap_iscsi_wrb, lun, pwrb,
4936                       cpu_to_be16(*(unsigned short *)
4937                                   &io_task->cmd_bhs->iscsi_hdr.lun));
4938         AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb, xferlen);
4939         AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4940                       io_task->pwrb_handle->wrb_index);
4941         AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4942                       be32_to_cpu(task->cmdsn));
4943         AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4944                       io_task->psgl_handle->sgl_index);
4945
4946         hwi_write_sgl(pwrb, sg, num_sg, io_task);
4947
4948         AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4949                       io_task->pwrb_handle->nxt_wrb_index);
4950         be_dws_le_to_cpu(pwrb, sizeof(struct iscsi_wrb));
4951
4952         doorbell |= beiscsi_conn->beiscsi_conn_cid & DB_WRB_POST_CID_MASK;
4953         doorbell |= (io_task->pwrb_handle->wrb_index &
4954                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
4955         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
4956
4957         iowrite32(doorbell, phba->db_va +
4958                   beiscsi_conn->doorbell_offset);
4959         return 0;
4960 }
4961
4962 static int beiscsi_mtask(struct iscsi_task *task)
4963 {
4964         struct beiscsi_io_task *io_task = task->dd_data;
4965         struct iscsi_conn *conn = task->conn;
4966         struct beiscsi_conn *beiscsi_conn = conn->dd_data;
4967         struct beiscsi_hba *phba = beiscsi_conn->phba;
4968         struct iscsi_wrb *pwrb = NULL;
4969         unsigned int doorbell = 0;
4970         unsigned int cid;
4971         unsigned int pwrb_typeoffset = 0;
4972
4973         cid = beiscsi_conn->beiscsi_conn_cid;
4974         pwrb = io_task->pwrb_handle->pwrb;
4975         memset(pwrb, 0, sizeof(*pwrb));
4976
4977         if (is_chip_be2_be3r(phba)) {
4978                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb,
4979                               be32_to_cpu(task->cmdsn));
4980                 AMAP_SET_BITS(struct amap_iscsi_wrb, wrb_idx, pwrb,
4981                               io_task->pwrb_handle->wrb_index);
4982                 AMAP_SET_BITS(struct amap_iscsi_wrb, sgl_icd_idx, pwrb,
4983                               io_task->psgl_handle->sgl_index);
4984                 AMAP_SET_BITS(struct amap_iscsi_wrb, r2t_exp_dtl, pwrb,
4985                               task->data_count);
4986                 AMAP_SET_BITS(struct amap_iscsi_wrb, ptr2nextwrb, pwrb,
4987                               io_task->pwrb_handle->nxt_wrb_index);
4988                 pwrb_typeoffset = BE_WRB_TYPE_OFFSET;
4989         } else {
4990                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, cmdsn_itt, pwrb,
4991                               be32_to_cpu(task->cmdsn));
4992                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, wrb_idx, pwrb,
4993                               io_task->pwrb_handle->wrb_index);
4994                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, sgl_idx, pwrb,
4995                               io_task->psgl_handle->sgl_index);
4996                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, r2t_exp_dtl, pwrb,
4997                               task->data_count);
4998                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2, ptr2nextwrb, pwrb,
4999                               io_task->pwrb_handle->nxt_wrb_index);
5000                 pwrb_typeoffset = SKH_WRB_TYPE_OFFSET;
5001         }
5002
5003
5004         switch (task->hdr->opcode & ISCSI_OPCODE_MASK) {
5005         case ISCSI_OP_LOGIN:
5006                 AMAP_SET_BITS(struct amap_iscsi_wrb, cmdsn_itt, pwrb, 1);
5007                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
5008                 hwi_write_buffer(pwrb, task);
5009                 break;
5010         case ISCSI_OP_NOOP_OUT:
5011                 if (task->hdr->ttt != ISCSI_RESERVED_TAG) {
5012                         ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
5013                         if (is_chip_be2_be3r(phba))
5014                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
5015                                               dmsg, pwrb, 1);
5016                         else
5017                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
5018                                               dmsg, pwrb, 1);
5019                 } else {
5020                         ADAPTER_SET_WRB_TYPE(pwrb, INI_RD_CMD, pwrb_typeoffset);
5021                         if (is_chip_be2_be3r(phba))
5022                                 AMAP_SET_BITS(struct amap_iscsi_wrb,
5023                                               dmsg, pwrb, 0);
5024                         else
5025                                 AMAP_SET_BITS(struct amap_iscsi_wrb_v2,
5026                                               dmsg, pwrb, 0);
5027                 }
5028                 hwi_write_buffer(pwrb, task);
5029                 break;
5030         case ISCSI_OP_TEXT:
5031                 ADAPTER_SET_WRB_TYPE(pwrb, TGT_DM_CMD, pwrb_typeoffset);
5032                 hwi_write_buffer(pwrb, task);
5033                 break;
5034         case ISCSI_OP_SCSI_TMFUNC:
5035                 ADAPTER_SET_WRB_TYPE(pwrb, INI_TMF_CMD, pwrb_typeoffset);
5036                 hwi_write_buffer(pwrb, task);
5037                 break;
5038         case ISCSI_OP_LOGOUT:
5039                 ADAPTER_SET_WRB_TYPE(pwrb, HWH_TYPE_LOGOUT, pwrb_typeoffset);
5040                 hwi_write_buffer(pwrb, task);
5041                 break;
5042
5043         default:
5044                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5045                             "BM_%d : opcode =%d Not supported\n",
5046                             task->hdr->opcode & ISCSI_OPCODE_MASK);
5047
5048                 return -EINVAL;
5049         }
5050
5051         /* Set the task type */
5052         io_task->wrb_type = (is_chip_be2_be3r(phba)) ?
5053                 AMAP_GET_BITS(struct amap_iscsi_wrb, type, pwrb) :
5054                 AMAP_GET_BITS(struct amap_iscsi_wrb_v2, type, pwrb);
5055
5056         doorbell |= cid & DB_WRB_POST_CID_MASK;
5057         doorbell |= (io_task->pwrb_handle->wrb_index &
5058                      DB_DEF_PDU_WRB_INDEX_MASK) << DB_DEF_PDU_WRB_INDEX_SHIFT;
5059         doorbell |= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT;
5060         iowrite32(doorbell, phba->db_va +
5061                   beiscsi_conn->doorbell_offset);
5062         return 0;
5063 }
5064
5065 static int beiscsi_task_xmit(struct iscsi_task *task)
5066 {
5067         struct beiscsi_io_task *io_task = task->dd_data;
5068         struct scsi_cmnd *sc = task->sc;
5069         struct beiscsi_hba *phba = NULL;
5070         struct scatterlist *sg;
5071         int num_sg;
5072         unsigned int  writedir = 0, xferlen = 0;
5073
5074         phba = ((struct beiscsi_conn *)task->conn->dd_data)->phba;
5075
5076         if (!sc)
5077                 return beiscsi_mtask(task);
5078
5079         io_task->scsi_cmnd = sc;
5080         num_sg = scsi_dma_map(sc);
5081         if (num_sg < 0) {
5082                 struct iscsi_conn *conn = task->conn;
5083                 struct beiscsi_hba *phba = NULL;
5084
5085                 phba = ((struct beiscsi_conn *)conn->dd_data)->phba;
5086                 beiscsi_log(phba, KERN_ERR,
5087                             BEISCSI_LOG_IO | BEISCSI_LOG_ISCSI,
5088                             "BM_%d : scsi_dma_map Failed "
5089                             "Driver_ITT : 0x%x ITT : 0x%x Xferlen : 0x%x\n",
5090                             be32_to_cpu(io_task->cmd_bhs->iscsi_hdr.itt),
5091                             io_task->libiscsi_itt, scsi_bufflen(sc));
5092
5093                 return num_sg;
5094         }
5095         xferlen = scsi_bufflen(sc);
5096         sg = scsi_sglist(sc);
5097         if (sc->sc_data_direction == DMA_TO_DEVICE)
5098                 writedir = 1;
5099          else
5100                 writedir = 0;
5101
5102          return phba->iotask_fn(task, sg, num_sg, xferlen, writedir);
5103 }
5104
5105 /**
5106  * beiscsi_bsg_request - handle bsg request from ISCSI transport
5107  * @job: job to handle
5108  */
5109 static int beiscsi_bsg_request(struct bsg_job *job)
5110 {
5111         struct Scsi_Host *shost;
5112         struct beiscsi_hba *phba;
5113         struct iscsi_bsg_request *bsg_req = job->request;
5114         int rc = -EINVAL;
5115         unsigned int tag;
5116         struct be_dma_mem nonemb_cmd;
5117         struct be_cmd_resp_hdr *resp;
5118         struct iscsi_bsg_reply *bsg_reply = job->reply;
5119         unsigned short status, extd_status;
5120
5121         shost = iscsi_job_to_shost(job);
5122         phba = iscsi_host_priv(shost);
5123
5124         switch (bsg_req->msgcode) {
5125         case ISCSI_BSG_HST_VENDOR:
5126                 nonemb_cmd.va = pci_alloc_consistent(phba->ctrl.pdev,
5127                                         job->request_payload.payload_len,
5128                                         &nonemb_cmd.dma);
5129                 if (nonemb_cmd.va == NULL) {
5130                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5131                                     "BM_%d : Failed to allocate memory for "
5132                                     "beiscsi_bsg_request\n");
5133                         return -ENOMEM;
5134                 }
5135                 tag = mgmt_vendor_specific_fw_cmd(&phba->ctrl, phba, job,
5136                                                   &nonemb_cmd);
5137                 if (!tag) {
5138                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5139                                     "BM_%d : MBX Tag Allocation Failed\n");
5140
5141                         pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5142                                             nonemb_cmd.va, nonemb_cmd.dma);
5143                         return -EAGAIN;
5144                 }
5145
5146                 rc = wait_event_interruptible_timeout(
5147                                         phba->ctrl.mcc_wait[tag],
5148                                         phba->ctrl.mcc_numtag[tag],
5149                                         msecs_to_jiffies(
5150                                         BEISCSI_HOST_MBX_TIMEOUT));
5151                 extd_status = (phba->ctrl.mcc_numtag[tag] & 0x0000FF00) >> 8;
5152                 status = phba->ctrl.mcc_numtag[tag] & 0x000000FF;
5153                 free_mcc_tag(&phba->ctrl, tag);
5154                 resp = (struct be_cmd_resp_hdr *)nonemb_cmd.va;
5155                 sg_copy_from_buffer(job->reply_payload.sg_list,
5156                                     job->reply_payload.sg_cnt,
5157                                     nonemb_cmd.va, (resp->response_length
5158                                     + sizeof(*resp)));
5159                 bsg_reply->reply_payload_rcv_len = resp->response_length;
5160                 bsg_reply->result = status;
5161                 bsg_job_done(job, bsg_reply->result,
5162                              bsg_reply->reply_payload_rcv_len);
5163                 pci_free_consistent(phba->ctrl.pdev, nonemb_cmd.size,
5164                                     nonemb_cmd.va, nonemb_cmd.dma);
5165                 if (status || extd_status) {
5166                         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5167                                     "BM_%d : MBX Cmd Failed"
5168                                     " status = %d extd_status = %d\n",
5169                                     status, extd_status);
5170
5171                         return -EIO;
5172                 } else {
5173                         rc = 0;
5174                 }
5175                 break;
5176
5177         default:
5178                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_CONFIG,
5179                                 "BM_%d : Unsupported bsg command: 0x%x\n",
5180                                 bsg_req->msgcode);
5181                 break;
5182         }
5183
5184         return rc;
5185 }
5186
5187 void beiscsi_hba_attrs_init(struct beiscsi_hba *phba)
5188 {
5189         /* Set the logging parameter */
5190         beiscsi_log_enable_init(phba, beiscsi_log_enable);
5191 }
5192
5193 /*
5194  * beiscsi_quiesce()- Cleanup Driver resources
5195  * @phba: Instance Priv structure
5196  * @unload_state:i Clean or EEH unload state
5197  *
5198  * Free the OS and HW resources held by the driver
5199  **/
5200 static void beiscsi_quiesce(struct beiscsi_hba *phba,
5201                 uint32_t unload_state)
5202 {
5203         struct hwi_controller *phwi_ctrlr;
5204         struct hwi_context_memory *phwi_context;
5205         struct be_eq_obj *pbe_eq;
5206         unsigned int i, msix_vec;
5207
5208         phwi_ctrlr = phba->phwi_ctrlr;
5209         phwi_context = phwi_ctrlr->phwi_ctxt;
5210         hwi_disable_intr(phba);
5211         if (phba->msix_enabled) {
5212                 for (i = 0; i <= phba->num_cpus; i++) {
5213                         msix_vec = phba->msix_entries[i].vector;
5214                         synchronize_irq(msix_vec);
5215                         free_irq(msix_vec, &phwi_context->be_eq[i]);
5216                         kfree(phba->msi_name[i]);
5217                 }
5218         } else
5219                 if (phba->pcidev->irq) {
5220                         synchronize_irq(phba->pcidev->irq);
5221                         free_irq(phba->pcidev->irq, phba);
5222                 }
5223         pci_disable_msix(phba->pcidev);
5224         cancel_delayed_work_sync(&phba->beiscsi_hw_check_task);
5225
5226         for (i = 0; i < phba->num_cpus; i++) {
5227                 pbe_eq = &phwi_context->be_eq[i];
5228                 blk_iopoll_disable(&pbe_eq->iopoll);
5229         }
5230
5231         if (unload_state == BEISCSI_CLEAN_UNLOAD) {
5232                 destroy_workqueue(phba->wq);
5233                 beiscsi_clean_port(phba);
5234                 beiscsi_free_mem(phba);
5235
5236                 beiscsi_unmap_pci_function(phba);
5237                 pci_free_consistent(phba->pcidev,
5238                                     phba->ctrl.mbox_mem_alloced.size,
5239                                     phba->ctrl.mbox_mem_alloced.va,
5240                                     phba->ctrl.mbox_mem_alloced.dma);
5241         } else {
5242                 hwi_purge_eq(phba);
5243                 hwi_cleanup(phba);
5244         }
5245
5246 }
5247
5248 static void beiscsi_remove(struct pci_dev *pcidev)
5249 {
5250
5251         struct beiscsi_hba *phba = NULL;
5252
5253         phba = pci_get_drvdata(pcidev);
5254         if (!phba) {
5255                 dev_err(&pcidev->dev, "beiscsi_remove called with no phba\n");
5256                 return;
5257         }
5258
5259         beiscsi_destroy_def_ifaces(phba);
5260         beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
5261         iscsi_boot_destroy_kset(phba->boot_kset);
5262         iscsi_host_remove(phba->shost);
5263         pci_dev_put(phba->pcidev);
5264         iscsi_host_free(phba->shost);
5265         pci_disable_pcie_error_reporting(pcidev);
5266         pci_set_drvdata(pcidev, NULL);
5267         pci_disable_device(pcidev);
5268 }
5269
5270 static void beiscsi_shutdown(struct pci_dev *pcidev)
5271 {
5272
5273         struct beiscsi_hba *phba = NULL;
5274
5275         phba = (struct beiscsi_hba *)pci_get_drvdata(pcidev);
5276         if (!phba) {
5277                 dev_err(&pcidev->dev, "beiscsi_shutdown called with no phba\n");
5278                 return;
5279         }
5280
5281         phba->state = BE_ADAPTER_STATE_SHUTDOWN;
5282         iscsi_host_for_each_session(phba->shost, be2iscsi_fail_session);
5283         beiscsi_quiesce(phba, BEISCSI_CLEAN_UNLOAD);
5284         pci_disable_device(pcidev);
5285 }
5286
5287 static void beiscsi_msix_enable(struct beiscsi_hba *phba)
5288 {
5289         int i, status;
5290
5291         for (i = 0; i <= phba->num_cpus; i++)
5292                 phba->msix_entries[i].entry = i;
5293
5294         status = pci_enable_msix_range(phba->pcidev, phba->msix_entries,
5295                                        phba->num_cpus + 1, phba->num_cpus + 1);
5296         if (status > 0)
5297                 phba->msix_enabled = true;
5298
5299         return;
5300 }
5301
5302 static void be_eqd_update(struct beiscsi_hba *phba)
5303 {
5304         struct be_set_eqd set_eqd[MAX_CPUS];
5305         struct be_aic_obj *aic;
5306         struct be_eq_obj *pbe_eq;
5307         struct hwi_controller *phwi_ctrlr;
5308         struct hwi_context_memory *phwi_context;
5309         int eqd, i, num = 0;
5310         ulong now;
5311         u32 pps, delta;
5312         unsigned int tag;
5313
5314         phwi_ctrlr = phba->phwi_ctrlr;
5315         phwi_context = phwi_ctrlr->phwi_ctxt;
5316
5317         for (i = 0; i <= phba->num_cpus; i++) {
5318                 aic = &phba->aic_obj[i];
5319                 pbe_eq = &phwi_context->be_eq[i];
5320                 now = jiffies;
5321                 if (!aic->jiffs || time_before(now, aic->jiffs) ||
5322                     pbe_eq->cq_count < aic->eq_prev) {
5323                         aic->jiffs = now;
5324                         aic->eq_prev = pbe_eq->cq_count;
5325                         continue;
5326                 }
5327                 delta = jiffies_to_msecs(now - aic->jiffs);
5328                 pps = (((u32)(pbe_eq->cq_count - aic->eq_prev) * 1000) / delta);
5329                 eqd = (pps / 1500) << 2;
5330
5331                 if (eqd < 8)
5332                         eqd = 0;
5333                 eqd = min_t(u32, eqd, phwi_context->max_eqd);
5334                 eqd = max_t(u32, eqd, phwi_context->min_eqd);
5335
5336                 aic->jiffs = now;
5337                 aic->eq_prev = pbe_eq->cq_count;
5338
5339                 if (eqd != aic->prev_eqd) {
5340                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
5341                         set_eqd[num].eq_id = pbe_eq->q.id;
5342                         aic->prev_eqd = eqd;
5343                         num++;
5344                 }
5345         }
5346         if (num) {
5347                 tag = be_cmd_modify_eq_delay(phba, set_eqd, num);
5348                 if (tag)
5349                         beiscsi_mccq_compl(phba, tag, NULL, NULL);
5350         }
5351 }
5352
5353 static void be_check_boot_session(struct beiscsi_hba *phba)
5354 {
5355         if (beiscsi_setup_boot_info(phba))
5356                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5357                             "BM_%d : Could not set up "
5358                             "iSCSI boot info on async event.\n");
5359 }
5360
5361 /*
5362  * beiscsi_hw_health_check()- Check adapter health
5363  * @work: work item to check HW health
5364  *
5365  * Check if adapter in an unrecoverable state or not.
5366  **/
5367 static void
5368 beiscsi_hw_health_check(struct work_struct *work)
5369 {
5370         struct beiscsi_hba *phba =
5371                 container_of(work, struct beiscsi_hba,
5372                              beiscsi_hw_check_task.work);
5373
5374         be_eqd_update(phba);
5375
5376         if (phba->state & BE_ADAPTER_CHECK_BOOT) {
5377                 phba->state &= ~BE_ADAPTER_CHECK_BOOT;
5378                 be_check_boot_session(phba);
5379         }
5380
5381         beiscsi_ue_detect(phba);
5382
5383         schedule_delayed_work(&phba->beiscsi_hw_check_task,
5384                               msecs_to_jiffies(1000));
5385 }
5386
5387
5388 static pci_ers_result_t beiscsi_eeh_err_detected(struct pci_dev *pdev,
5389                 pci_channel_state_t state)
5390 {
5391         struct beiscsi_hba *phba = NULL;
5392
5393         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5394         phba->state |= BE_ADAPTER_PCI_ERR;
5395
5396         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5397                     "BM_%d : EEH error detected\n");
5398
5399         beiscsi_quiesce(phba, BEISCSI_EEH_UNLOAD);
5400
5401         if (state == pci_channel_io_perm_failure) {
5402                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5403                             "BM_%d : EEH : State PERM Failure");
5404                 return PCI_ERS_RESULT_DISCONNECT;
5405         }
5406
5407         pci_disable_device(pdev);
5408
5409         /* The error could cause the FW to trigger a flash debug dump.
5410          * Resetting the card while flash dump is in progress
5411          * can cause it not to recover; wait for it to finish.
5412          * Wait only for first function as it is needed only once per
5413          * adapter.
5414          **/
5415         if (pdev->devfn == 0)
5416                 ssleep(30);
5417
5418         return PCI_ERS_RESULT_NEED_RESET;
5419 }
5420
5421 static pci_ers_result_t beiscsi_eeh_reset(struct pci_dev *pdev)
5422 {
5423         struct beiscsi_hba *phba = NULL;
5424         int status = 0;
5425
5426         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5427
5428         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5429                     "BM_%d : EEH Reset\n");
5430
5431         status = pci_enable_device(pdev);
5432         if (status)
5433                 return PCI_ERS_RESULT_DISCONNECT;
5434
5435         pci_set_master(pdev);
5436         pci_set_power_state(pdev, PCI_D0);
5437         pci_restore_state(pdev);
5438
5439         /* Wait for the CHIP Reset to complete */
5440         status = be_chk_reset_complete(phba);
5441         if (!status) {
5442                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5443                             "BM_%d : EEH Reset Completed\n");
5444         } else {
5445                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5446                             "BM_%d : EEH Reset Completion Failure\n");
5447                 return PCI_ERS_RESULT_DISCONNECT;
5448         }
5449
5450         pci_cleanup_aer_uncorrect_error_status(pdev);
5451         return PCI_ERS_RESULT_RECOVERED;
5452 }
5453
5454 static void beiscsi_eeh_resume(struct pci_dev *pdev)
5455 {
5456         int ret = 0, i;
5457         struct be_eq_obj *pbe_eq;
5458         struct beiscsi_hba *phba = NULL;
5459         struct hwi_controller *phwi_ctrlr;
5460         struct hwi_context_memory *phwi_context;
5461
5462         phba = (struct beiscsi_hba *)pci_get_drvdata(pdev);
5463         pci_save_state(pdev);
5464
5465         if (enable_msix)
5466                 find_num_cpus(phba);
5467         else
5468                 phba->num_cpus = 1;
5469
5470         if (enable_msix) {
5471                 beiscsi_msix_enable(phba);
5472                 if (!phba->msix_enabled)
5473                         phba->num_cpus = 1;
5474         }
5475
5476         ret = beiscsi_cmd_reset_function(phba);
5477         if (ret) {
5478                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5479                             "BM_%d : Reset Failed\n");
5480                 goto ret_err;
5481         }
5482
5483         ret = be_chk_reset_complete(phba);
5484         if (ret) {
5485                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5486                             "BM_%d : Failed to get out of reset.\n");
5487                 goto ret_err;
5488         }
5489
5490         beiscsi_get_params(phba);
5491         phba->shost->max_id = phba->params.cxns_per_ctrl;
5492         phba->shost->can_queue = phba->params.ios_per_ctrl;
5493         ret = hwi_init_controller(phba);
5494
5495         for (i = 0; i < MAX_MCC_CMD; i++) {
5496                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5497                 phba->ctrl.mcc_tag[i] = i + 1;
5498                 phba->ctrl.mcc_numtag[i + 1] = 0;
5499                 phba->ctrl.mcc_tag_available++;
5500         }
5501
5502         phwi_ctrlr = phba->phwi_ctrlr;
5503         phwi_context = phwi_ctrlr->phwi_ctxt;
5504
5505         for (i = 0; i < phba->num_cpus; i++) {
5506                 pbe_eq = &phwi_context->be_eq[i];
5507                 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5508                                 be_iopoll);
5509                 blk_iopoll_enable(&pbe_eq->iopoll);
5510         }
5511
5512         i = (phba->msix_enabled) ? i : 0;
5513         /* Work item for MCC handling */
5514         pbe_eq = &phwi_context->be_eq[i];
5515         INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5516
5517         ret = beiscsi_init_irqs(phba);
5518         if (ret < 0) {
5519                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5520                             "BM_%d : beiscsi_eeh_resume - "
5521                             "Failed to beiscsi_init_irqs\n");
5522                 goto ret_err;
5523         }
5524
5525         hwi_enable_intr(phba);
5526         phba->state &= ~BE_ADAPTER_PCI_ERR;
5527
5528         return;
5529 ret_err:
5530         beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5531                     "BM_%d : AER EEH Resume Failed\n");
5532 }
5533
5534 static int beiscsi_dev_probe(struct pci_dev *pcidev,
5535                              const struct pci_device_id *id)
5536 {
5537         struct beiscsi_hba *phba = NULL;
5538         struct hwi_controller *phwi_ctrlr;
5539         struct hwi_context_memory *phwi_context;
5540         struct be_eq_obj *pbe_eq;
5541         int ret = 0, i;
5542
5543         ret = beiscsi_enable_pci(pcidev);
5544         if (ret < 0) {
5545                 dev_err(&pcidev->dev,
5546                         "beiscsi_dev_probe - Failed to enable pci device\n");
5547                 return ret;
5548         }
5549
5550         phba = beiscsi_hba_alloc(pcidev);
5551         if (!phba) {
5552                 dev_err(&pcidev->dev,
5553                         "beiscsi_dev_probe - Failed in beiscsi_hba_alloc\n");
5554                 goto disable_pci;
5555         }
5556
5557         /* Enable EEH reporting */
5558         ret = pci_enable_pcie_error_reporting(pcidev);
5559         if (ret)
5560                 beiscsi_log(phba, KERN_WARNING, BEISCSI_LOG_INIT,
5561                             "BM_%d : PCIe Error Reporting "
5562                             "Enabling Failed\n");
5563
5564         pci_save_state(pcidev);
5565
5566         /* Initialize Driver configuration Paramters */
5567         beiscsi_hba_attrs_init(phba);
5568
5569         phba->fw_timeout = false;
5570         phba->mac_addr_set = false;
5571
5572
5573         switch (pcidev->device) {
5574         case BE_DEVICE_ID1:
5575         case OC_DEVICE_ID1:
5576         case OC_DEVICE_ID2:
5577                 phba->generation = BE_GEN2;
5578                 phba->iotask_fn = beiscsi_iotask;
5579                 break;
5580         case BE_DEVICE_ID2:
5581         case OC_DEVICE_ID3:
5582                 phba->generation = BE_GEN3;
5583                 phba->iotask_fn = beiscsi_iotask;
5584                 break;
5585         case OC_SKH_ID1:
5586                 phba->generation = BE_GEN4;
5587                 phba->iotask_fn = beiscsi_iotask_v2;
5588                 break;
5589         default:
5590                 phba->generation = 0;
5591         }
5592
5593         ret = be_ctrl_init(phba, pcidev);
5594         if (ret) {
5595                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5596                             "BM_%d : beiscsi_dev_probe-"
5597                             "Failed in be_ctrl_init\n");
5598                 goto hba_free;
5599         }
5600
5601         ret = beiscsi_cmd_reset_function(phba);
5602         if (ret) {
5603                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5604                             "BM_%d : Reset Failed\n");
5605                 goto hba_free;
5606         }
5607         ret = be_chk_reset_complete(phba);
5608         if (ret) {
5609                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5610                             "BM_%d : Failed to get out of reset.\n");
5611                 goto hba_free;
5612         }
5613
5614         spin_lock_init(&phba->io_sgl_lock);
5615         spin_lock_init(&phba->mgmt_sgl_lock);
5616         spin_lock_init(&phba->isr_lock);
5617         spin_lock_init(&phba->async_pdu_lock);
5618         ret = mgmt_get_fw_config(&phba->ctrl, phba);
5619         if (ret != 0) {
5620                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5621                             "BM_%d : Error getting fw config\n");
5622                 goto free_port;
5623         }
5624
5625         if (enable_msix)
5626                 find_num_cpus(phba);
5627         else
5628                 phba->num_cpus = 1;
5629
5630         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5631                     "BM_%d : num_cpus = %d\n",
5632                     phba->num_cpus);
5633
5634         if (enable_msix) {
5635                 beiscsi_msix_enable(phba);
5636                 if (!phba->msix_enabled)
5637                         phba->num_cpus = 1;
5638         }
5639
5640         phba->shost->max_id = phba->params.cxns_per_ctrl;
5641         beiscsi_get_params(phba);
5642         phba->shost->can_queue = phba->params.ios_per_ctrl;
5643         ret = beiscsi_init_port(phba);
5644         if (ret < 0) {
5645                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5646                             "BM_%d : beiscsi_dev_probe-"
5647                             "Failed in beiscsi_init_port\n");
5648                 goto free_port;
5649         }
5650
5651         for (i = 0; i < MAX_MCC_CMD; i++) {
5652                 init_waitqueue_head(&phba->ctrl.mcc_wait[i + 1]);
5653                 phba->ctrl.mcc_tag[i] = i + 1;
5654                 phba->ctrl.mcc_numtag[i + 1] = 0;
5655                 phba->ctrl.mcc_tag_available++;
5656                 memset(&phba->ctrl.ptag_state[i].tag_mem_state, 0,
5657                        sizeof(struct be_dma_mem));
5658         }
5659
5660         phba->ctrl.mcc_alloc_index = phba->ctrl.mcc_free_index = 0;
5661
5662         snprintf(phba->wq_name, sizeof(phba->wq_name), "beiscsi_%02x_wq",
5663                  phba->shost->host_no);
5664         phba->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 1, phba->wq_name);
5665         if (!phba->wq) {
5666                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5667                             "BM_%d : beiscsi_dev_probe-"
5668                             "Failed to allocate work queue\n");
5669                 goto free_twq;
5670         }
5671
5672         INIT_DELAYED_WORK(&phba->beiscsi_hw_check_task,
5673                           beiscsi_hw_health_check);
5674
5675         phwi_ctrlr = phba->phwi_ctrlr;
5676         phwi_context = phwi_ctrlr->phwi_ctxt;
5677
5678         for (i = 0; i < phba->num_cpus; i++) {
5679                 pbe_eq = &phwi_context->be_eq[i];
5680                 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5681                                 be_iopoll);
5682                 blk_iopoll_enable(&pbe_eq->iopoll);
5683         }
5684
5685         i = (phba->msix_enabled) ? i : 0;
5686         /* Work item for MCC handling */
5687         pbe_eq = &phwi_context->be_eq[i];
5688         INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5689
5690         ret = beiscsi_init_irqs(phba);
5691         if (ret < 0) {
5692                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5693                             "BM_%d : beiscsi_dev_probe-"
5694                             "Failed to beiscsi_init_irqs\n");
5695                 goto free_blkenbld;
5696         }
5697         hwi_enable_intr(phba);
5698
5699         if (iscsi_host_add(phba->shost, &phba->pcidev->dev))
5700                 goto free_blkenbld;
5701
5702         if (beiscsi_setup_boot_info(phba))
5703                 /*
5704                  * log error but continue, because we may not be using
5705                  * iscsi boot.
5706                  */
5707                 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
5708                             "BM_%d : Could not set up "
5709                             "iSCSI boot info.\n");
5710
5711         beiscsi_create_def_ifaces(phba);
5712         schedule_delayed_work(&phba->beiscsi_hw_check_task,
5713                               msecs_to_jiffies(1000));
5714
5715         beiscsi_log(phba, KERN_INFO, BEISCSI_LOG_INIT,
5716                     "\n\n\n BM_%d : SUCCESS - DRIVER LOADED\n\n\n");
5717         return 0;
5718
5719 free_blkenbld:
5720         destroy_workqueue(phba->wq);
5721         for (i = 0; i < phba->num_cpus; i++) {
5722                 pbe_eq = &phwi_context->be_eq[i];
5723                 blk_iopoll_disable(&pbe_eq->iopoll);
5724         }
5725 free_twq:
5726         beiscsi_clean_port(phba);
5727         beiscsi_free_mem(phba);
5728 free_port:
5729         pci_free_consistent(phba->pcidev,
5730                             phba->ctrl.mbox_mem_alloced.size,
5731                             phba->ctrl.mbox_mem_alloced.va,
5732                            phba->ctrl.mbox_mem_alloced.dma);
5733         beiscsi_unmap_pci_function(phba);
5734 hba_free:
5735         if (phba->msix_enabled)
5736                 pci_disable_msix(phba->pcidev);
5737         pci_dev_put(phba->pcidev);
5738         iscsi_host_free(phba->shost);
5739         pci_set_drvdata(pcidev, NULL);
5740 disable_pci:
5741         pci_disable_device(pcidev);
5742         return ret;
5743 }
5744
5745 static struct pci_error_handlers beiscsi_eeh_handlers = {
5746         .error_detected = beiscsi_eeh_err_detected,
5747         .slot_reset = beiscsi_eeh_reset,
5748         .resume = beiscsi_eeh_resume,
5749 };
5750
5751 struct iscsi_transport beiscsi_iscsi_transport = {
5752         .owner = THIS_MODULE,
5753         .name = DRV_NAME,
5754         .caps = CAP_RECOVERY_L0 | CAP_HDRDGST | CAP_TEXT_NEGO |
5755                 CAP_MULTI_R2T | CAP_DATADGST | CAP_DATA_PATH_OFFLOAD,
5756         .create_session = beiscsi_session_create,
5757         .destroy_session = beiscsi_session_destroy,
5758         .create_conn = beiscsi_conn_create,
5759         .bind_conn = beiscsi_conn_bind,
5760         .destroy_conn = iscsi_conn_teardown,
5761         .attr_is_visible = be2iscsi_attr_is_visible,
5762         .set_iface_param = be2iscsi_iface_set_param,
5763         .get_iface_param = be2iscsi_iface_get_param,
5764         .set_param = beiscsi_set_param,
5765         .get_conn_param = iscsi_conn_get_param,
5766         .get_session_param = iscsi_session_get_param,
5767         .get_host_param = beiscsi_get_host_param,
5768         .start_conn = beiscsi_conn_start,
5769         .stop_conn = iscsi_conn_stop,
5770         .send_pdu = iscsi_conn_send_pdu,
5771         .xmit_task = beiscsi_task_xmit,
5772         .cleanup_task = beiscsi_cleanup_task,
5773         .alloc_pdu = beiscsi_alloc_pdu,
5774         .parse_pdu_itt = beiscsi_parse_pdu,
5775         .get_stats = beiscsi_conn_get_stats,
5776         .get_ep_param = beiscsi_ep_get_param,
5777         .ep_connect = beiscsi_ep_connect,
5778         .ep_poll = beiscsi_ep_poll,
5779         .ep_disconnect = beiscsi_ep_disconnect,
5780         .session_recovery_timedout = iscsi_session_recovery_timedout,
5781         .bsg_request = beiscsi_bsg_request,
5782 };
5783
5784 static struct pci_driver beiscsi_pci_driver = {
5785         .name = DRV_NAME,
5786         .probe = beiscsi_dev_probe,
5787         .remove = beiscsi_remove,
5788         .shutdown = beiscsi_shutdown,
5789         .id_table = beiscsi_pci_id_table,
5790         .err_handler = &beiscsi_eeh_handlers
5791 };
5792
5793
5794 static int __init beiscsi_module_init(void)
5795 {
5796         int ret;
5797
5798         beiscsi_scsi_transport =
5799                         iscsi_register_transport(&beiscsi_iscsi_transport);
5800         if (!beiscsi_scsi_transport) {
5801                 printk(KERN_ERR
5802                        "beiscsi_module_init - Unable to  register beiscsi transport.\n");
5803                 return -ENOMEM;
5804         }
5805         printk(KERN_INFO "In beiscsi_module_init, tt=%p\n",
5806                &beiscsi_iscsi_transport);
5807
5808         ret = pci_register_driver(&beiscsi_pci_driver);
5809         if (ret) {
5810                 printk(KERN_ERR
5811                        "beiscsi_module_init - Unable to  register beiscsi pci driver.\n");
5812                 goto unregister_iscsi_transport;
5813         }
5814         return 0;
5815
5816 unregister_iscsi_transport:
5817         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5818         return ret;
5819 }
5820
5821 static void __exit beiscsi_module_exit(void)
5822 {
5823         pci_unregister_driver(&beiscsi_pci_driver);
5824         iscsi_unregister_transport(&beiscsi_iscsi_transport);
5825 }
5826
5827 module_init(beiscsi_module_init);
5828 module_exit(beiscsi_module_exit);