kernel/drivers/scsi/cxgbi/cxgb4i/cxgb4i.c

   1 /*
   2  * cxgb4i.c: Chelsio T4 iSCSI driver.
   3  *
   4  * Copyright (c) 2010 Chelsio Communications, Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation.
   9  *
  10  * Written by:  Karen Xie (kxie@chelsio.com)
  11  *              Rakesh Ranjan (rranjan@chelsio.com)
  12  */
  13
  14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
  15
  16 #include <linux/module.h>
  17 #include <linux/moduleparam.h>
  18 #include <scsi/scsi_host.h>
  19 #include <net/tcp.h>
  20 #include <net/dst.h>
  21 #include <linux/netdevice.h>
  22 #include <net/addrconf.h>
  23
  24 #include "t4_regs.h"
  25 #include "t4_msg.h"
  26 #include "cxgb4.h"
  27 #include "cxgb4_uld.h"
  28 #include "t4fw_api.h"
  29 #include "l2t.h"
  30 #include "cxgb4i.h"
  31 #include "clip_tbl.h"
  32
  33 static unsigned int dbg_level;
  34
  35 #include "../libcxgbi.h"
  36
  37 #define DRV_MODULE_NAME         "cxgb4i"
  38 #define DRV_MODULE_DESC         "Chelsio T4/T5 iSCSI Driver"
  39 #define DRV_MODULE_VERSION      "0.9.4"
  40
  41 static char version[] =
  42         DRV_MODULE_DESC " " DRV_MODULE_NAME
  43         " v" DRV_MODULE_VERSION "\n";
  44
  45 MODULE_AUTHOR("Chelsio Communications, Inc.");
  46 MODULE_DESCRIPTION(DRV_MODULE_DESC);
  47 MODULE_VERSION(DRV_MODULE_VERSION);
  48 MODULE_LICENSE("GPL");
  49
  50 module_param(dbg_level, uint, 0644);
  51 MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)");
  52
  53 static int cxgb4i_rcv_win = 256 * 1024;
  54 module_param(cxgb4i_rcv_win, int, 0644);
  55 MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes");
  56
  57 static int cxgb4i_snd_win = 128 * 1024;
  58 module_param(cxgb4i_snd_win, int, 0644);
  59 MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes");
  60
  61 static int cxgb4i_rx_credit_thres = 10 * 1024;
  62 module_param(cxgb4i_rx_credit_thres, int, 0644);
  63 MODULE_PARM_DESC(cxgb4i_rx_credit_thres,
  64                 "RX credits return threshold in bytes (default=10KB)");
  65
  66 static unsigned int cxgb4i_max_connect = (8 * 1024);
  67 module_param(cxgb4i_max_connect, uint, 0644);
  68 MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections");
  69
  70 static unsigned short cxgb4i_sport_base = 20000;
  71 module_param(cxgb4i_sport_base, ushort, 0644);
  72 MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)");
  73
  74 typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *);
  75
  76 static void *t4_uld_add(const struct cxgb4_lld_info *);
  77 static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *);
  78 static int t4_uld_state_change(void *, enum cxgb4_state state);
  79 static inline int send_tx_flowc_wr(struct cxgbi_sock *);
  80
  81 static const struct cxgb4_uld_info cxgb4i_uld_info = {
  82         .name = DRV_MODULE_NAME,
  83         .add = t4_uld_add,
  84         .rx_handler = t4_uld_rx_handler,
  85         .state_change = t4_uld_state_change,
  86 };
  87
  88 static struct scsi_host_template cxgb4i_host_template = {
  89         .module         = THIS_MODULE,
  90         .name           = DRV_MODULE_NAME,
  91         .proc_name      = DRV_MODULE_NAME,
  92         .can_queue      = CXGB4I_SCSI_HOST_QDEPTH,
  93         .queuecommand   = iscsi_queuecommand,
  94         .change_queue_depth = scsi_change_queue_depth,
  95         .sg_tablesize   = SG_ALL,
  96         .max_sectors    = 0xFFFF,
  97         .cmd_per_lun    = ISCSI_DEF_CMD_PER_LUN,
  98         .eh_abort_handler = iscsi_eh_abort,
  99         .eh_device_reset_handler = iscsi_eh_device_reset,
 100         .eh_target_reset_handler = iscsi_eh_recover_target,
 101         .target_alloc   = iscsi_target_alloc,
 102         .use_clustering = DISABLE_CLUSTERING,
 103         .this_id        = -1,
 104         .track_queue_depth = 1,
 105 };
 106
 107 static struct iscsi_transport cxgb4i_iscsi_transport = {
 108         .owner          = THIS_MODULE,
 109         .name           = DRV_MODULE_NAME,
 110         .caps           = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST |
 111                                 CAP_DATADGST | CAP_DIGEST_OFFLOAD |
 112                                 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
 113         .attr_is_visible        = cxgbi_attr_is_visible,
 114         .get_host_param = cxgbi_get_host_param,
 115         .set_host_param = cxgbi_set_host_param,
 116         /* session management */
 117         .create_session = cxgbi_create_session,
 118         .destroy_session        = cxgbi_destroy_session,
 119         .get_session_param = iscsi_session_get_param,
 120         /* connection management */
 121         .create_conn    = cxgbi_create_conn,
 122         .bind_conn              = cxgbi_bind_conn,
 123         .destroy_conn   = iscsi_tcp_conn_teardown,
 124         .start_conn             = iscsi_conn_start,
 125         .stop_conn              = iscsi_conn_stop,
 126         .get_conn_param = iscsi_conn_get_param,
 127         .set_param      = cxgbi_set_conn_param,
 128         .get_stats      = cxgbi_get_conn_stats,
 129         /* pdu xmit req from user space */
 130         .send_pdu       = iscsi_conn_send_pdu,
 131         /* task */
 132         .init_task      = iscsi_tcp_task_init,
 133         .xmit_task      = iscsi_tcp_task_xmit,
 134         .cleanup_task   = cxgbi_cleanup_task,
 135         /* pdu */
 136         .alloc_pdu      = cxgbi_conn_alloc_pdu,
 137         .init_pdu       = cxgbi_conn_init_pdu,
 138         .xmit_pdu       = cxgbi_conn_xmit_pdu,
 139         .parse_pdu_itt  = cxgbi_parse_pdu_itt,
 140         /* TCP connect/disconnect */
 141         .get_ep_param   = cxgbi_get_ep_param,
 142         .ep_connect     = cxgbi_ep_connect,
 143         .ep_poll        = cxgbi_ep_poll,
 144         .ep_disconnect  = cxgbi_ep_disconnect,
 145         /* Error recovery timeout call */
 146         .session_recovery_timedout = iscsi_session_recovery_timedout,
 147 };
 148
 149 static struct scsi_transport_template *cxgb4i_stt;
 150
 151 /*
 152  * CPL (Chelsio Protocol Language) defines a message passing interface between
 153  * the host driver and Chelsio asic.
 154  * The section below implments CPLs that related to iscsi tcp connection
 155  * open/close/abort and data send/receive.
 156  */
 157
 158 #define DIV_ROUND_UP(n, d)      (((n) + (d) - 1) / (d))
 159 #define RCV_BUFSIZ_MASK         0x3FFU
 160 #define MAX_IMM_TX_PKT_LEN      128
 161
 162 static int push_tx_frames(struct cxgbi_sock *, int);
 163
 164 /*
 165  * is_ofld_imm - check whether a packet can be sent as immediate data
 166  * @skb: the packet
 167  *
 168  * Returns true if a packet can be sent as an offload WR with immediate
 169  * data.  We currently use the same limit as for Ethernet packets.
 170  */
 171 static inline bool is_ofld_imm(const struct sk_buff *skb)
 172 {
 173         int len = skb->len;
 174
 175         if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
 176                 len += sizeof(struct fw_ofld_tx_data_wr);
 177
 178         return len <= MAX_IMM_TX_PKT_LEN;
 179 }
 180
 181 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
 182                                 struct l2t_entry *e)
 183 {
 184         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 185         int t4 = is_t4(lldi->adapter_type);
 186         int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
 187         unsigned long long opt0;
 188         unsigned int opt2;
 189         unsigned int qid_atid = ((unsigned int)csk->atid) |
 190                                  (((unsigned int)csk->rss_qid) << 14);
 191
 192         opt0 = KEEP_ALIVE_F |
 193                 WND_SCALE_V(wscale) |
 194                 MSS_IDX_V(csk->mss_idx) |
 195                 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
 196                 TX_CHAN_V(csk->tx_chan) |
 197                 SMAC_SEL_V(csk->smac_idx) |
 198                 ULP_MODE_V(ULP_MODE_ISCSI) |
 199                 RCV_BUFSIZ_V(cxgb4i_rcv_win >> 10);
 200         opt2 = RX_CHANNEL_V(0) |
 201                 RSS_QUEUE_VALID_F |
 202                 (RX_FC_DISABLE_F) |
 203                 RSS_QUEUE_V(csk->rss_qid);
 204
 205         if (is_t4(lldi->adapter_type)) {
 206                 struct cpl_act_open_req *req =
 207                                 (struct cpl_act_open_req *)skb->head;
 208
 209                 INIT_TP_WR(req, 0);
 210                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 211                                         qid_atid));
 212                 req->local_port = csk->saddr.sin_port;
 213                 req->peer_port = csk->daddr.sin_port;
 214                 req->local_ip = csk->saddr.sin_addr.s_addr;
 215                 req->peer_ip = csk->daddr.sin_addr.s_addr;
 216                 req->opt0 = cpu_to_be64(opt0);
 217                 req->params = cpu_to_be32(cxgb4_select_ntuple(
 218                                         csk->cdev->ports[csk->port_id],
 219                                         csk->l2t));
 220                 opt2 |= RX_FC_VALID_F;
 221                 req->opt2 = cpu_to_be32(opt2);
 222
 223                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 224                         "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
 225                         csk, &req->local_ip, ntohs(req->local_port),
 226                         &req->peer_ip, ntohs(req->peer_port),
 227                         csk->atid, csk->rss_qid);
 228         } else {
 229                 struct cpl_t5_act_open_req *req =
 230                                 (struct cpl_t5_act_open_req *)skb->head;
 231
 232                 INIT_TP_WR(req, 0);
 233                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
 234                                         qid_atid));
 235                 req->local_port = csk->saddr.sin_port;
 236                 req->peer_port = csk->daddr.sin_port;
 237                 req->local_ip = csk->saddr.sin_addr.s_addr;
 238                 req->peer_ip = csk->daddr.sin_addr.s_addr;
 239                 req->opt0 = cpu_to_be64(opt0);
 240                 req->params = cpu_to_be64(FILTER_TUPLE_V(
 241                                 cxgb4_select_ntuple(
 242                                         csk->cdev->ports[csk->port_id],
 243                                         csk->l2t)));
 244                 opt2 |= 1 << 31;
 245                 req->opt2 = cpu_to_be32(opt2);
 246
 247                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 248                         "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
 249                         csk, &req->local_ip, ntohs(req->local_port),
 250                         &req->peer_ip, ntohs(req->peer_port),
 251                         csk->atid, csk->rss_qid);
 252         }
 253
 254         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 255
 256         pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n",
 257                        (&csk->saddr), (&csk->daddr), t4 ? 4 : 5, csk,
 258                        csk->state, csk->flags, csk->atid, csk->rss_qid);
 259
 260         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 261 }
 262
 263 #if IS_ENABLED(CONFIG_IPV6)
 264 static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb,
 265                                struct l2t_entry *e)
 266 {
 267         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 268         int t4 = is_t4(lldi->adapter_type);
 269         int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
 270         unsigned long long opt0;
 271         unsigned int opt2;
 272         unsigned int qid_atid = ((unsigned int)csk->atid) |
 273                                  (((unsigned int)csk->rss_qid) << 14);
 274
 275         opt0 = KEEP_ALIVE_F |
 276                 WND_SCALE_V(wscale) |
 277                 MSS_IDX_V(csk->mss_idx) |
 278                 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) |
 279                 TX_CHAN_V(csk->tx_chan) |
 280                 SMAC_SEL_V(csk->smac_idx) |
 281                 ULP_MODE_V(ULP_MODE_ISCSI) |
 282                 RCV_BUFSIZ_V(cxgb4i_rcv_win >> 10);
 283
 284         opt2 = RX_CHANNEL_V(0) |
 285                 RSS_QUEUE_VALID_F |
 286                 RX_FC_DISABLE_F |
 287                 RSS_QUEUE_V(csk->rss_qid);
 288
 289         if (t4) {
 290                 struct cpl_act_open_req6 *req =
 291                             (struct cpl_act_open_req6 *)skb->head;
 292
 293                 INIT_TP_WR(req, 0);
 294                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 295                                                             qid_atid));
 296                 req->local_port = csk->saddr6.sin6_port;
 297                 req->peer_port = csk->daddr6.sin6_port;
 298
 299                 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
 300                 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
 301                                                                     8);
 302                 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
 303                 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
 304                                                                     8);
 305
 306                 req->opt0 = cpu_to_be64(opt0);
 307
 308                 opt2 |= RX_FC_VALID_F;
 309                 req->opt2 = cpu_to_be32(opt2);
 310
 311                 req->params = cpu_to_be32(cxgb4_select_ntuple(
 312                                           csk->cdev->ports[csk->port_id],
 313                                           csk->l2t));
 314         } else {
 315                 struct cpl_t5_act_open_req6 *req =
 316                                 (struct cpl_t5_act_open_req6 *)skb->head;
 317
 318                 INIT_TP_WR(req, 0);
 319                 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6,
 320                                                             qid_atid));
 321                 req->local_port = csk->saddr6.sin6_port;
 322                 req->peer_port = csk->daddr6.sin6_port;
 323                 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr);
 324                 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr +
 325                                                                         8);
 326                 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr);
 327                 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr +
 328                                                                         8);
 329                 req->opt0 = cpu_to_be64(opt0);
 330
 331                 opt2 |= T5_OPT_2_VALID_F;
 332                 req->opt2 = cpu_to_be32(opt2);
 333
 334                 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple(
 335                                           csk->cdev->ports[csk->port_id],
 336                                           csk->l2t)));
 337         }
 338
 339         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 340
 341         pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n",
 342                 t4 ? 4 : 5, csk, csk->state, csk->flags, csk->atid,
 343                 &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port),
 344                 &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port),
 345                 csk->rss_qid);
 346
 347         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 348 }
 349 #endif
 350
 351 static void send_close_req(struct cxgbi_sock *csk)
 352 {
 353         struct sk_buff *skb = csk->cpl_close;
 354         struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
 355         unsigned int tid = csk->tid;
 356
 357         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 358                 "csk 0x%p,%u,0x%lx, tid %u.\n",
 359                 csk, csk->state, csk->flags, csk->tid);
 360         csk->cpl_close = NULL;
 361         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 362         INIT_TP_WR(req, tid);
 363         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 364         req->rsvd = 0;
 365
 366         cxgbi_sock_skb_entail(csk, skb);
 367         if (csk->state >= CTP_ESTABLISHED)
 368                 push_tx_frames(csk, 1);
 369 }
 370
 371 static void abort_arp_failure(void *handle, struct sk_buff *skb)
 372 {
 373         struct cxgbi_sock *csk = (struct cxgbi_sock *)handle;
 374         struct cpl_abort_req *req;
 375
 376         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 377                 "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
 378                 csk, csk->state, csk->flags, csk->tid);
 379         req = (struct cpl_abort_req *)skb->data;
 380         req->cmd = CPL_ABORT_NO_RST;
 381         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 382 }
 383
 384 static void send_abort_req(struct cxgbi_sock *csk)
 385 {
 386         struct cpl_abort_req *req;
 387         struct sk_buff *skb = csk->cpl_abort_req;
 388
 389         if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev)
 390                 return;
 391
 392         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 393                 send_tx_flowc_wr(csk);
 394                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 395         }
 396
 397         cxgbi_sock_set_state(csk, CTP_ABORTING);
 398         cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
 399         cxgbi_sock_purge_write_queue(csk);
 400
 401         csk->cpl_abort_req = NULL;
 402         req = (struct cpl_abort_req *)skb->head;
 403         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 404         req->cmd = CPL_ABORT_SEND_RST;
 405         t4_set_arp_err_handler(skb, csk, abort_arp_failure);
 406         INIT_TP_WR(req, csk->tid);
 407         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
 408         req->rsvd0 = htonl(csk->snd_nxt);
 409         req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
 410
 411         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 412                 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
 413                 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
 414                 req->rsvd1);
 415
 416         cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 417 }
 418
 419 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
 420 {
 421         struct sk_buff *skb = csk->cpl_abort_rpl;
 422         struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
 423
 424         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 425                 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
 426                 csk, csk->state, csk->flags, csk->tid, rst_status);
 427
 428         csk->cpl_abort_rpl = NULL;
 429         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 430         INIT_TP_WR(rpl, csk->tid);
 431         OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
 432         rpl->cmd = rst_status;
 433         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 434 }
 435
 436 /*
 437  * CPL connection rx data ack: host ->
 438  * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
 439  * credits sent.
 440  */
 441 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
 442 {
 443         struct sk_buff *skb;
 444         struct cpl_rx_data_ack *req;
 445
 446         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
 447                 "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
 448                 csk, csk->state, csk->flags, csk->tid, credits);
 449
 450         skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
 451         if (!skb) {
 452                 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
 453                 return 0;
 454         }
 455         req = (struct cpl_rx_data_ack *)skb->head;
 456
 457         set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
 458         INIT_TP_WR(req, csk->tid);
 459         OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
 460                                       csk->tid));
 461         req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits)
 462                                        | RX_FORCE_ACK_F);
 463         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 464         return credits;
 465 }
 466
 467 /*
 468  * sgl_len - calculates the size of an SGL of the given capacity
 469  * @n: the number of SGL entries
 470  * Calculates the number of flits needed for a scatter/gather list that
 471  * can hold the given number of entries.
 472  */
 473 static inline unsigned int sgl_len(unsigned int n)
 474 {
 475         n--;
 476         return (3 * n) / 2 + (n & 1) + 2;
 477 }
 478
 479 /*
 480  * calc_tx_flits_ofld - calculate # of flits for an offload packet
 481  * @skb: the packet
 482  *
 483  * Returns the number of flits needed for the given offload packet.
 484  * These packets are already fully constructed and no additional headers
 485  * will be added.
 486  */
 487 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
 488 {
 489         unsigned int flits, cnt;
 490
 491         if (is_ofld_imm(skb))
 492                 return DIV_ROUND_UP(skb->len, 8);
 493         flits = skb_transport_offset(skb) / 8;
 494         cnt = skb_shinfo(skb)->nr_frags;
 495         if (skb_tail_pointer(skb) != skb_transport_header(skb))
 496                 cnt++;
 497         return flits + sgl_len(cnt);
 498 }
 499
 500 #define FLOWC_WR_NPARAMS_MIN    9
 501 static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp)
 502 {
 503         int nparams, flowclen16, flowclen;
 504
 505         nparams = FLOWC_WR_NPARAMS_MIN;
 506         flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]);
 507         flowclen16 = DIV_ROUND_UP(flowclen, 16);
 508         flowclen = flowclen16 * 16;
 509         /*
 510          * Return the number of 16-byte credits used by the FlowC request.
 511          * Pass back the nparams and actual FlowC length if requested.
 512          */
 513         if (nparamsp)
 514                 *nparamsp = nparams;
 515         if (flowclenp)
 516                 *flowclenp = flowclen;
 517
 518         return flowclen16;
 519 }
 520
 521 static inline int send_tx_flowc_wr(struct cxgbi_sock *csk)
 522 {
 523         struct sk_buff *skb;
 524         struct fw_flowc_wr *flowc;
 525         int nparams, flowclen16, flowclen;
 526
 527         flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen);
 528         skb = alloc_wr(flowclen, 0, GFP_ATOMIC);
 529         flowc = (struct fw_flowc_wr *)skb->head;
 530         flowc->op_to_nparams =
 531                 htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams));
 532         flowc->flowid_len16 =
 533                 htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid));
 534         flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
 535         flowc->mnemval[0].val = htonl(csk->cdev->pfvf);
 536         flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
 537         flowc->mnemval[1].val = htonl(csk->tx_chan);
 538         flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
 539         flowc->mnemval[2].val = htonl(csk->tx_chan);
 540         flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
 541         flowc->mnemval[3].val = htonl(csk->rss_qid);
 542         flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
 543         flowc->mnemval[4].val = htonl(csk->snd_nxt);
 544         flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
 545         flowc->mnemval[5].val = htonl(csk->rcv_nxt);
 546         flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
 547         flowc->mnemval[6].val = htonl(cxgb4i_snd_win);
 548         flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
 549         flowc->mnemval[7].val = htonl(csk->advmss);
 550         flowc->mnemval[8].mnemonic = 0;
 551         flowc->mnemval[8].val = 0;
 552         flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX;
 553         flowc->mnemval[8].val = 16384;
 554
 555         set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 556
 557         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 558                 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
 559                 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid,
 560                 csk->snd_nxt, csk->rcv_nxt, cxgb4i_snd_win,
 561                 csk->advmss);
 562
 563         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
 564
 565         return flowclen16;
 566 }
 567
 568 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
 569                                    int dlen, int len, u32 credits, int compl)
 570 {
 571         struct fw_ofld_tx_data_wr *req;
 572         unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3;
 573         unsigned int wr_ulp_mode = 0, val;
 574         bool imm = is_ofld_imm(skb);
 575
 576         req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
 577
 578         if (imm) {
 579                 req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
 580                                         FW_WR_COMPL_F |
 581                                         FW_WR_IMMDLEN_V(dlen));
 582                 req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) |
 583                                                 FW_WR_LEN16_V(credits));
 584         } else {
 585                 req->op_to_immdlen =
 586                         cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) |
 587                                         FW_WR_COMPL_F |
 588                                         FW_WR_IMMDLEN_V(0));
 589                 req->flowid_len16 =
 590                         cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
 591                                         FW_WR_LEN16_V(credits));
 592         }
 593         if (submode)
 594                 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) |
 595                                 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
 596         val = skb_peek(&csk->write_queue) ? 0 : 1;
 597         req->tunnel_to_proxy = htonl(wr_ulp_mode |
 598                                      FW_OFLD_TX_DATA_WR_SHOVE_V(val));
 599         req->plen = htonl(len);
 600         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))
 601                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 602 }
 603
 604 static void arp_failure_skb_discard(void *handle, struct sk_buff *skb)
 605 {
 606         kfree_skb(skb);
 607 }
 608
 609 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
 610 {
 611         int total_size = 0;
 612         struct sk_buff *skb;
 613
 614         if (unlikely(csk->state < CTP_ESTABLISHED ||
 615                 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
 616                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK |
 617                          1 << CXGBI_DBG_PDU_TX,
 618                         "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
 619                         csk, csk->state, csk->flags, csk->tid);
 620                 return 0;
 621         }
 622
 623         while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
 624                 int dlen = skb->len;
 625                 int len = skb->len;
 626                 unsigned int credits_needed;
 627                 int flowclen16 = 0;
 628
 629                 skb_reset_transport_header(skb);
 630                 if (is_ofld_imm(skb))
 631                         credits_needed = DIV_ROUND_UP(dlen, 16);
 632                 else
 633                         credits_needed = DIV_ROUND_UP(
 634                                                 8 * calc_tx_flits_ofld(skb),
 635                                                 16);
 636
 637                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR)))
 638                         credits_needed += DIV_ROUND_UP(
 639                                         sizeof(struct fw_ofld_tx_data_wr),
 640                                         16);
 641
 642                 /*
 643                  * Assumes the initial credits is large enough to support
 644                  * fw_flowc_wr plus largest possible first payload
 645                  */
 646                 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 647                         flowclen16 = send_tx_flowc_wr(csk);
 648                         csk->wr_cred -= flowclen16;
 649                         csk->wr_una_cred += flowclen16;
 650                         cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 651                 }
 652
 653                 if (csk->wr_cred < credits_needed) {
 654                         log_debug(1 << CXGBI_DBG_PDU_TX,
 655                                 "csk 0x%p, skb %u/%u, wr %d < %u.\n",
 656                                 csk, skb->len, skb->data_len,
 657                                 credits_needed, csk->wr_cred);
 658                         break;
 659                 }
 660                 __skb_unlink(skb, &csk->write_queue);
 661                 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
 662                 skb->csum = credits_needed + flowclen16;
 663                 csk->wr_cred -= credits_needed;
 664                 csk->wr_una_cred += credits_needed;
 665                 cxgbi_sock_enqueue_wr(csk, skb);
 666
 667                 log_debug(1 << CXGBI_DBG_PDU_TX,
 668                         "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
 669                         csk, skb->len, skb->data_len, credits_needed,
 670                         csk->wr_cred, csk->wr_una_cred);
 671
 672                 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
 673                         len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
 674                         make_tx_data_wr(csk, skb, dlen, len, credits_needed,
 675                                         req_completion);
 676                         csk->snd_nxt += len;
 677                         cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
 678                 }
 679                 total_size += skb->truesize;
 680                 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);
 681
 682                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
 683                         "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
 684                         csk, csk->state, csk->flags, csk->tid, skb, len);
 685
 686                 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
 687         }
 688         return total_size;
 689 }
 690
 691 static inline void free_atid(struct cxgbi_sock *csk)
 692 {
 693         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 694
 695         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
 696                 cxgb4_free_atid(lldi->tids, csk->atid);
 697                 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
 698                 cxgbi_sock_put(csk);
 699         }
 700 }
 701
 702 static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb)
 703 {
 704         struct cxgbi_sock *csk;
 705         struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data;
 706         unsigned short tcp_opt = ntohs(req->tcp_opt);
 707         unsigned int tid = GET_TID(req);
 708         unsigned int atid = TID_TID_G(ntohl(req->tos_atid));
 709         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 710         struct tid_info *t = lldi->tids;
 711         u32 rcv_isn = be32_to_cpu(req->rcv_isn);
 712
 713         csk = lookup_atid(t, atid);
 714         if (unlikely(!csk)) {
 715                 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev);
 716                 goto rel_skb;
 717         }
 718
 719         if (csk->atid != atid) {
 720                 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
 721                         atid, csk, csk->state, csk->flags, csk->tid, csk->atid);
 722                 goto rel_skb;
 723         }
 724
 725         pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
 726                        (&csk->saddr), (&csk->daddr),
 727                        atid, tid, csk, csk->state, csk->flags, rcv_isn);
 728
 729         module_put(THIS_MODULE);
 730
 731         cxgbi_sock_get(csk);
 732         csk->tid = tid;
 733         cxgb4_insert_tid(lldi->tids, csk, tid);
 734         cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
 735
 736         free_atid(csk);
 737
 738         spin_lock_bh(&csk->lock);
 739         if (unlikely(csk->state != CTP_ACTIVE_OPEN))
 740                 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
 741                         csk, csk->state, csk->flags, csk->tid);
 742
 743         if (csk->retry_timer.function) {
 744                 del_timer(&csk->retry_timer);
 745                 csk->retry_timer.function = NULL;
 746         }
 747
 748         csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
 749         /*
 750          * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
 751          * pass through opt0.
 752          */
 753         if (cxgb4i_rcv_win > (RCV_BUFSIZ_MASK << 10))
 754                 csk->rcv_wup -= cxgb4i_rcv_win - (RCV_BUFSIZ_MASK << 10);
 755
 756         csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40;
 757         if (TCPOPT_TSTAMP_G(tcp_opt))
 758                 csk->advmss -= 12;
 759         if (csk->advmss < 128)
 760                 csk->advmss = 128;
 761
 762         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 763                 "csk 0x%p, mss_idx %u, advmss %u.\n",
 764                         csk, TCPOPT_MSS_G(tcp_opt), csk->advmss);
 765
 766         cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
 767
 768         if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
 769                 send_abort_req(csk);
 770         else {
 771                 if (skb_queue_len(&csk->write_queue))
 772                         push_tx_frames(csk, 0);
 773                 cxgbi_conn_tx_open(csk);
 774         }
 775         spin_unlock_bh(&csk->lock);
 776
 777 rel_skb:
 778         __kfree_skb(skb);
 779 }
 780
 781 static int act_open_rpl_status_to_errno(int status)
 782 {
 783         switch (status) {
 784         case CPL_ERR_CONN_RESET:
 785                 return -ECONNREFUSED;
 786         case CPL_ERR_ARP_MISS:
 787                 return -EHOSTUNREACH;
 788         case CPL_ERR_CONN_TIMEDOUT:
 789                 return -ETIMEDOUT;
 790         case CPL_ERR_TCAM_FULL:
 791                 return -ENOMEM;
 792         case CPL_ERR_CONN_EXIST:
 793                 return -EADDRINUSE;
 794         default:
 795                 return -EIO;
 796         }
 797 }
 798
 799 static void csk_act_open_retry_timer(unsigned long data)
 800 {
 801         struct sk_buff *skb = NULL;
 802         struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
 803         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
 804         void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *,
 805                                    struct l2t_entry *);
 806         int t4 = is_t4(lldi->adapter_type), size, size6;
 807
 808         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
 809                 "csk 0x%p,%u,0x%lx,%u.\n",
 810                 csk, csk->state, csk->flags, csk->tid);
 811
 812         cxgbi_sock_get(csk);
 813         spin_lock_bh(&csk->lock);
 814
 815         if (t4) {
 816                 size = sizeof(struct cpl_act_open_req);
 817                 size6 = sizeof(struct cpl_act_open_req6);
 818         } else {
 819                 size = sizeof(struct cpl_t5_act_open_req);
 820                 size6 = sizeof(struct cpl_t5_act_open_req6);
 821         }
 822
 823         if (csk->csk_family == AF_INET) {
 824                 send_act_open_func = send_act_open_req;
 825                 skb = alloc_wr(size, 0, GFP_ATOMIC);
 826 #if IS_ENABLED(CONFIG_IPV6)
 827         } else {
 828                 send_act_open_func = send_act_open_req6;
 829                 skb = alloc_wr(size6, 0, GFP_ATOMIC);
 830 #endif
 831         }
 832
 833         if (!skb)
 834                 cxgbi_sock_fail_act_open(csk, -ENOMEM);
 835         else {
 836                 skb->sk = (struct sock *)csk;
 837                 t4_set_arp_err_handler(skb, csk,
 838                                        cxgbi_sock_act_open_req_arp_failure);
 839                 send_act_open_func(csk, skb, csk->l2t);
 840         }
 841
 842         spin_unlock_bh(&csk->lock);
 843         cxgbi_sock_put(csk);
 844
 845 }
 846
 847 static inline bool is_neg_adv(unsigned int status)
 848 {
 849         return status == CPL_ERR_RTX_NEG_ADVICE ||
 850                 status == CPL_ERR_KEEPALV_NEG_ADVICE ||
 851                 status == CPL_ERR_PERSIST_NEG_ADVICE;
 852 }
 853
 854 static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
 855 {
 856         struct cxgbi_sock *csk;
 857         struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data;
 858         unsigned int tid = GET_TID(rpl);
 859         unsigned int atid =
 860                 TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status)));
 861         unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status));
 862         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 863         struct tid_info *t = lldi->tids;
 864
 865         csk = lookup_atid(t, atid);
 866         if (unlikely(!csk)) {
 867                 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid);
 868                 goto rel_skb;
 869         }
 870
 871         pr_info_ipaddr("tid %u/%u, status %u.\n"
 872                        "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr),
 873                        atid, tid, status, csk, csk->state, csk->flags);
 874
 875         if (is_neg_adv(status))
 876                 goto rel_skb;
 877
 878         module_put(THIS_MODULE);
 879
 880         if (status && status != CPL_ERR_TCAM_FULL &&
 881             status != CPL_ERR_CONN_EXIST &&
 882             status != CPL_ERR_ARP_MISS)
 883                 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl));
 884
 885         cxgbi_sock_get(csk);
 886         spin_lock_bh(&csk->lock);
 887
 888         if (status == CPL_ERR_CONN_EXIST &&
 889             csk->retry_timer.function != csk_act_open_retry_timer) {
 890                 csk->retry_timer.function = csk_act_open_retry_timer;
 891                 mod_timer(&csk->retry_timer, jiffies + HZ / 2);
 892         } else
 893                 cxgbi_sock_fail_act_open(csk,
 894                                         act_open_rpl_status_to_errno(status));
 895
 896         spin_unlock_bh(&csk->lock);
 897         cxgbi_sock_put(csk);
 898 rel_skb:
 899         __kfree_skb(skb);
 900 }
 901
 902 static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb)
 903 {
 904         struct cxgbi_sock *csk;
 905         struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data;
 906         unsigned int tid = GET_TID(req);
 907         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 908         struct tid_info *t = lldi->tids;
 909
 910         csk = lookup_tid(t, tid);
 911         if (unlikely(!csk)) {
 912                 pr_err("can't find connection for tid %u.\n", tid);
 913                 goto rel_skb;
 914         }
 915         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
 916                        (&csk->saddr), (&csk->daddr),
 917                        csk, csk->state, csk->flags, csk->tid);
 918         cxgbi_sock_rcv_peer_close(csk);
 919 rel_skb:
 920         __kfree_skb(skb);
 921 }
 922
 923 static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
 924 {
 925         struct cxgbi_sock *csk;
 926         struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data;
 927         unsigned int tid = GET_TID(rpl);
 928         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 929         struct tid_info *t = lldi->tids;
 930
 931         csk = lookup_tid(t, tid);
 932         if (unlikely(!csk)) {
 933                 pr_err("can't find connection for tid %u.\n", tid);
 934                 goto rel_skb;
 935         }
 936         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n",
 937                        (&csk->saddr), (&csk->daddr),
 938                        csk, csk->state, csk->flags, csk->tid);
 939         cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
 940 rel_skb:
 941         __kfree_skb(skb);
 942 }
 943
 944 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
 945                                                                 int *need_rst)
 946 {
 947         switch (abort_reason) {
 948         case CPL_ERR_BAD_SYN: /* fall through */
 949         case CPL_ERR_CONN_RESET:
 950                 return csk->state > CTP_ESTABLISHED ?
 951                         -EPIPE : -ECONNRESET;
 952         case CPL_ERR_XMIT_TIMEDOUT:
 953         case CPL_ERR_PERSIST_TIMEDOUT:
 954         case CPL_ERR_FINWAIT2_TIMEDOUT:
 955         case CPL_ERR_KEEPALIVE_TIMEDOUT:
 956                 return -ETIMEDOUT;
 957         default:
 958                 return -EIO;
 959         }
 960 }
 961
 962 static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
 963 {
 964         struct cxgbi_sock *csk;
 965         struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data;
 966         unsigned int tid = GET_TID(req);
 967         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
 968         struct tid_info *t = lldi->tids;
 969         int rst_status = CPL_ABORT_NO_RST;
 970
 971         csk = lookup_tid(t, tid);
 972         if (unlikely(!csk)) {
 973                 pr_err("can't find connection for tid %u.\n", tid);
 974                 goto rel_skb;
 975         }
 976
 977         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
 978                        (&csk->saddr), (&csk->daddr),
 979                        csk, csk->state, csk->flags, csk->tid, req->status);
 980
 981         if (is_neg_adv(req->status))
 982                 goto rel_skb;
 983
 984         cxgbi_sock_get(csk);
 985         spin_lock_bh(&csk->lock);
 986
 987         cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
 988
 989         if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
 990                 send_tx_flowc_wr(csk);
 991                 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
 992         }
 993
 994         cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
 995         cxgbi_sock_set_state(csk, CTP_ABORTING);
 996
 997         send_abort_rpl(csk, rst_status);
 998
 999         if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
1000                 csk->err = abort_status_to_errno(csk, req->status, &rst_status);
1001                 cxgbi_sock_closed(csk);
1002         }
1003
1004         spin_unlock_bh(&csk->lock);
1005         cxgbi_sock_put(csk);
1006 rel_skb:
1007         __kfree_skb(skb);
1008 }
1009
1010 static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
1011 {
1012         struct cxgbi_sock *csk;
1013         struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data;
1014         unsigned int tid = GET_TID(rpl);
1015         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1016         struct tid_info *t = lldi->tids;
1017
1018         csk = lookup_tid(t, tid);
1019         if (!csk)
1020                 goto rel_skb;
1021
1022         if (csk)
1023                 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n",
1024                                (&csk->saddr), (&csk->daddr), csk,
1025                                csk->state, csk->flags, csk->tid, rpl->status);
1026
1027         if (rpl->status == CPL_ERR_ABORT_FAILED)
1028                 goto rel_skb;
1029
1030         cxgbi_sock_rcv_abort_rpl(csk);
1031 rel_skb:
1032         __kfree_skb(skb);
1033 }
1034
1035 static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb)
1036 {
1037         struct cxgbi_sock *csk;
1038         struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data;
1039         unsigned int tid = GET_TID(cpl);
1040         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1041         struct tid_info *t = lldi->tids;
1042
1043         csk = lookup_tid(t, tid);
1044         if (!csk) {
1045                 pr_err("can't find connection for tid %u.\n", tid);
1046         } else {
1047                 /* not expecting this, reset the connection. */
1048                 pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid);
1049                 spin_lock_bh(&csk->lock);
1050                 send_abort_req(csk);
1051                 spin_unlock_bh(&csk->lock);
1052         }
1053         __kfree_skb(skb);
1054 }
1055
1056 static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb)
1057 {
1058         struct cxgbi_sock *csk;
1059         struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data;
1060         unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp);
1061         unsigned int tid = GET_TID(cpl);
1062         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1063         struct tid_info *t = lldi->tids;
1064
1065         csk = lookup_tid(t, tid);
1066         if (unlikely(!csk)) {
1067                 pr_err("can't find conn. for tid %u.\n", tid);
1068                 goto rel_skb;
1069         }
1070
1071         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1072                 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
1073                 csk, csk->state, csk->flags, csk->tid, skb, skb->len,
1074                 pdu_len_ddp);
1075
1076         spin_lock_bh(&csk->lock);
1077
1078         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
1079                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1080                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1081                         csk, csk->state, csk->flags, csk->tid);
1082                 if (csk->state != CTP_ABORTING)
1083                         goto abort_conn;
1084                 else
1085                         goto discard;
1086         }
1087
1088         cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq);
1089         cxgbi_skcb_flags(skb) = 0;
1090
1091         skb_reset_transport_header(skb);
1092         __skb_pull(skb, sizeof(*cpl));
1093         __pskb_trim(skb, ntohs(cpl->len));
1094
1095         if (!csk->skb_ulp_lhdr) {
1096                 unsigned char *bhs;
1097                 unsigned int hlen, dlen, plen;
1098
1099                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1100                         "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
1101                         csk, csk->state, csk->flags, csk->tid, skb);
1102                 csk->skb_ulp_lhdr = skb;
1103                 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR);
1104
1105                 if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) {
1106                         pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
1107                                 csk->tid, cxgbi_skcb_tcp_seq(skb),
1108                                 csk->rcv_nxt);
1109                         goto abort_conn;
1110                 }
1111
1112                 bhs = skb->data;
1113                 hlen = ntohs(cpl->len);
1114                 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF;
1115
1116                 plen = ISCSI_PDU_LEN_G(pdu_len_ddp);
1117                 if (is_t4(lldi->adapter_type))
1118                         plen -= 40;
1119
1120                 if ((hlen + dlen) != plen) {
1121                         pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
1122                                 "mismatch %u != %u + %u, seq 0x%x.\n",
1123                                 csk->tid, plen, hlen, dlen,
1124                                 cxgbi_skcb_tcp_seq(skb));
1125                         goto abort_conn;
1126                 }
1127
1128                 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3);
1129                 if (dlen)
1130                         cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len;
1131                 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb);
1132
1133                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1134                         "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
1135                         csk, skb, *bhs, hlen, dlen,
1136                         ntohl(*((unsigned int *)(bhs + 16))),
1137                         ntohl(*((unsigned int *)(bhs + 24))));
1138
1139         } else {
1140                 struct sk_buff *lskb = csk->skb_ulp_lhdr;
1141
1142                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA);
1143                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1144                         "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
1145                         csk, csk->state, csk->flags, skb, lskb);
1146         }
1147
1148         __skb_queue_tail(&csk->receive_queue, skb);
1149         spin_unlock_bh(&csk->lock);
1150         return;
1151
1152 abort_conn:
1153         send_abort_req(csk);
1154 discard:
1155         spin_unlock_bh(&csk->lock);
1156 rel_skb:
1157         __kfree_skb(skb);
1158 }
1159
1160 static void do_rx_data_ddp(struct cxgbi_device *cdev,
1161                                   struct sk_buff *skb)
1162 {
1163         struct cxgbi_sock *csk;
1164         struct sk_buff *lskb;
1165         struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data;
1166         unsigned int tid = GET_TID(rpl);
1167         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1168         struct tid_info *t = lldi->tids;
1169         unsigned int status = ntohl(rpl->ddpvld);
1170
1171         csk = lookup_tid(t, tid);
1172         if (unlikely(!csk)) {
1173                 pr_err("can't find connection for tid %u.\n", tid);
1174                 goto rel_skb;
1175         }
1176
1177         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
1178                 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
1179                 csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr);
1180
1181         spin_lock_bh(&csk->lock);
1182
1183         if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
1184                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1185                         "csk 0x%p,%u,0x%lx,%u, bad state.\n",
1186                         csk, csk->state, csk->flags, csk->tid);
1187                 if (csk->state != CTP_ABORTING)
1188                         goto abort_conn;
1189                 else
1190                         goto discard;
1191         }
1192
1193         if (!csk->skb_ulp_lhdr) {
1194                 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid);
1195                 goto abort_conn;
1196         }
1197
1198         lskb = csk->skb_ulp_lhdr;
1199         csk->skb_ulp_lhdr = NULL;
1200
1201         cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc);
1202
1203         if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb))
1204                 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
1205                         csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb));
1206
1207         if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
1208                 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
1209                         csk, lskb, status, cxgbi_skcb_flags(lskb));
1210                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR);
1211         }
1212         if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
1213                 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
1214                         csk, lskb, status, cxgbi_skcb_flags(lskb));
1215                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR);
1216         }
1217         if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
1218                 log_debug(1 << CXGBI_DBG_PDU_RX,
1219                         "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
1220                         csk, lskb, status);
1221                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR);
1222         }
1223         if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) &&
1224                 !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) {
1225                 log_debug(1 << CXGBI_DBG_PDU_RX,
1226                         "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
1227                         csk, lskb, status);
1228                 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD);
1229         }
1230         log_debug(1 << CXGBI_DBG_PDU_RX,
1231                 "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
1232                 csk, lskb, cxgbi_skcb_flags(lskb));
1233
1234         cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS);
1235         cxgbi_conn_pdu_ready(csk);
1236         spin_unlock_bh(&csk->lock);
1237         goto rel_skb;
1238
1239 abort_conn:
1240         send_abort_req(csk);
1241 discard:
1242         spin_unlock_bh(&csk->lock);
1243 rel_skb:
1244         __kfree_skb(skb);
1245 }
1246
1247 static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb)
1248 {
1249         struct cxgbi_sock *csk;
1250         struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data;
1251         unsigned int tid = GET_TID(rpl);
1252         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1253         struct tid_info *t = lldi->tids;
1254
1255         csk = lookup_tid(t, tid);
1256         if (unlikely(!csk))
1257                 pr_err("can't find connection for tid %u.\n", tid);
1258         else {
1259                 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1260                         "csk 0x%p,%u,0x%lx,%u.\n",
1261                         csk, csk->state, csk->flags, csk->tid);
1262                 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una),
1263                                         rpl->seq_vld);
1264         }
1265         __kfree_skb(skb);
1266 }
1267
1268 static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
1269 {
1270         struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
1271         unsigned int tid = GET_TID(rpl);
1272         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1273         struct tid_info *t = lldi->tids;
1274         struct cxgbi_sock *csk;
1275
1276         csk = lookup_tid(t, tid);
1277         if (!csk)
1278                 pr_err("can't find conn. for tid %u.\n", tid);
1279
1280         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1281                 "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
1282                 csk, csk->state, csk->flags, csk->tid, rpl->status);
1283
1284         if (rpl->status != CPL_ERR_NONE)
1285                 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
1286                         csk, tid, rpl->status);
1287
1288         __kfree_skb(skb);
1289 }
1290
1291 static int alloc_cpls(struct cxgbi_sock *csk)
1292 {
1293         csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req),
1294                                         0, GFP_KERNEL);
1295         if (!csk->cpl_close)
1296                 return -ENOMEM;
1297
1298         csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req),
1299                                         0, GFP_KERNEL);
1300         if (!csk->cpl_abort_req)
1301                 goto free_cpls;
1302
1303         csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl),
1304                                         0, GFP_KERNEL);
1305         if (!csk->cpl_abort_rpl)
1306                 goto free_cpls;
1307         return 0;
1308
1309 free_cpls:
1310         cxgbi_sock_free_cpl_skbs(csk);
1311         return -ENOMEM;
1312 }
1313
1314 static inline void l2t_put(struct cxgbi_sock *csk)
1315 {
1316         if (csk->l2t) {
1317                 cxgb4_l2t_release(csk->l2t);
1318                 csk->l2t = NULL;
1319                 cxgbi_sock_put(csk);
1320         }
1321 }
1322
1323 static void release_offload_resources(struct cxgbi_sock *csk)
1324 {
1325         struct cxgb4_lld_info *lldi;
1326 #if IS_ENABLED(CONFIG_IPV6)
1327         struct net_device *ndev = csk->cdev->ports[csk->port_id];
1328 #endif
1329
1330         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1331                 "csk 0x%p,%u,0x%lx,%u.\n",
1332                 csk, csk->state, csk->flags, csk->tid);
1333
1334         cxgbi_sock_free_cpl_skbs(csk);
1335         if (csk->wr_cred != csk->wr_max_cred) {
1336                 cxgbi_sock_purge_wr_queue(csk);
1337                 cxgbi_sock_reset_wr_list(csk);
1338         }
1339
1340         l2t_put(csk);
1341 #if IS_ENABLED(CONFIG_IPV6)
1342         if (csk->csk_family == AF_INET6)
1343                 cxgb4_clip_release(ndev,
1344                                    (const u32 *)&csk->saddr6.sin6_addr, 1);
1345 #endif
1346
1347         if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
1348                 free_atid(csk);
1349         else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
1350                 lldi = cxgbi_cdev_priv(csk->cdev);
1351                 cxgb4_remove_tid(lldi->tids, 0, csk->tid);
1352                 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
1353                 cxgbi_sock_put(csk);
1354         }
1355         csk->dst = NULL;
1356         csk->cdev = NULL;
1357 }
1358
1359 static int init_act_open(struct cxgbi_sock *csk)
1360 {
1361         struct cxgbi_device *cdev = csk->cdev;
1362         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1363         struct net_device *ndev = cdev->ports[csk->port_id];
1364         struct sk_buff *skb = NULL;
1365         struct neighbour *n = NULL;
1366         void *daddr;
1367         unsigned int step;
1368         unsigned int size, size6;
1369         int t4 = is_t4(lldi->adapter_type);
1370
1371         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1372                 "csk 0x%p,%u,0x%lx,%u.\n",
1373                 csk, csk->state, csk->flags, csk->tid);
1374
1375         if (csk->csk_family == AF_INET)
1376                 daddr = &csk->daddr.sin_addr.s_addr;
1377 #if IS_ENABLED(CONFIG_IPV6)
1378         else if (csk->csk_family == AF_INET6)
1379                 daddr = &csk->daddr6.sin6_addr;
1380 #endif
1381         else {
1382                 pr_err("address family 0x%x not supported\n", csk->csk_family);
1383                 goto rel_resource;
1384         }
1385
1386         n = dst_neigh_lookup(csk->dst, daddr);
1387
1388         if (!n) {
1389                 pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name);
1390                 goto rel_resource;
1391         }
1392
1393         csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
1394         if (csk->atid < 0) {
1395                 pr_err("%s, NO atid available.\n", ndev->name);
1396                 return -EINVAL;
1397         }
1398         cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
1399         cxgbi_sock_get(csk);
1400
1401         csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0);
1402         if (!csk->l2t) {
1403                 pr_err("%s, cannot alloc l2t.\n", ndev->name);
1404                 goto rel_resource_without_clip;
1405         }
1406         cxgbi_sock_get(csk);
1407
1408 #if IS_ENABLED(CONFIG_IPV6)
1409         if (csk->csk_family == AF_INET6)
1410                 cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1);
1411 #endif
1412
1413         if (t4) {
1414                 size = sizeof(struct cpl_act_open_req);
1415                 size6 = sizeof(struct cpl_act_open_req6);
1416         } else {
1417                 size = sizeof(struct cpl_t5_act_open_req);
1418                 size6 = sizeof(struct cpl_t5_act_open_req6);
1419         }
1420
1421         if (csk->csk_family == AF_INET)
1422                 skb = alloc_wr(size, 0, GFP_NOIO);
1423 #if IS_ENABLED(CONFIG_IPV6)
1424         else
1425                 skb = alloc_wr(size6, 0, GFP_NOIO);
1426 #endif
1427
1428         if (!skb)
1429                 goto rel_resource;
1430         skb->sk = (struct sock *)csk;
1431         t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure);
1432
1433         if (!csk->mtu)
1434                 csk->mtu = dst_mtu(csk->dst);
1435         cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx);
1436         csk->tx_chan = cxgb4_port_chan(ndev);
1437         /* SMT two entries per row */
1438         csk->smac_idx = ((cxgb4_port_viid(ndev) & 0x7F)) << 1;
1439         step = lldi->ntxq / lldi->nchan;
1440         csk->txq_idx = cxgb4_port_idx(ndev) * step;
1441         step = lldi->nrxq / lldi->nchan;
1442         csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step];
1443         csk->wr_cred = lldi->wr_cred -
1444                        DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16);
1445         csk->wr_max_cred = csk->wr_cred;
1446         csk->wr_una_cred = 0;
1447         cxgbi_sock_reset_wr_list(csk);
1448         csk->err = 0;
1449
1450         pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n",
1451                        (&csk->saddr), (&csk->daddr), csk, csk->state,
1452                        csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid,
1453                        csk->mtu, csk->mss_idx, csk->smac_idx);
1454
1455         /* must wait for either a act_open_rpl or act_open_establish */
1456         try_module_get(THIS_MODULE);
1457         cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1458         if (csk->csk_family == AF_INET)
1459                 send_act_open_req(csk, skb, csk->l2t);
1460 #if IS_ENABLED(CONFIG_IPV6)
1461         else
1462                 send_act_open_req6(csk, skb, csk->l2t);
1463 #endif
1464         neigh_release(n);
1465
1466         return 0;
1467
1468 rel_resource:
1469 #if IS_ENABLED(CONFIG_IPV6)
1470         if (csk->csk_family == AF_INET6)
1471                 cxgb4_clip_release(ndev,
1472                                    (const u32 *)&csk->saddr6.sin6_addr, 1);
1473 #endif
1474 rel_resource_without_clip:
1475         if (n)
1476                 neigh_release(n);
1477         if (skb)
1478                 __kfree_skb(skb);
1479         return -EINVAL;
1480 }
1481
1482 cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = {
1483         [CPL_ACT_ESTABLISH] = do_act_establish,
1484         [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1485         [CPL_PEER_CLOSE] = do_peer_close,
1486         [CPL_ABORT_REQ_RSS] = do_abort_req_rss,
1487         [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss,
1488         [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1489         [CPL_FW4_ACK] = do_fw4_ack,
1490         [CPL_ISCSI_HDR] = do_rx_iscsi_hdr,
1491         [CPL_ISCSI_DATA] = do_rx_iscsi_hdr,
1492         [CPL_SET_TCB_RPL] = do_set_tcb_rpl,
1493         [CPL_RX_DATA_DDP] = do_rx_data_ddp,
1494         [CPL_RX_ISCSI_DDP] = do_rx_data_ddp,
1495         [CPL_RX_DATA] = do_rx_data,
1496 };
1497
1498 int cxgb4i_ofld_init(struct cxgbi_device *cdev)
1499 {
1500         int rc;
1501
1502         if (cxgb4i_max_connect > CXGB4I_MAX_CONN)
1503                 cxgb4i_max_connect = CXGB4I_MAX_CONN;
1504
1505         rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base,
1506                                         cxgb4i_max_connect);
1507         if (rc < 0)
1508                 return rc;
1509
1510         cdev->csk_release_offload_resources = release_offload_resources;
1511         cdev->csk_push_tx_frames = push_tx_frames;
1512         cdev->csk_send_abort_req = send_abort_req;
1513         cdev->csk_send_close_req = send_close_req;
1514         cdev->csk_send_rx_credits = send_rx_credits;
1515         cdev->csk_alloc_cpls = alloc_cpls;
1516         cdev->csk_init_act_open = init_act_open;
1517
1518         pr_info("cdev 0x%p, offload up, added.\n", cdev);
1519         return 0;
1520 }
1521
1522 /*
1523  * functions to program the pagepod in h/w
1524  */
1525 #define ULPMEM_IDATA_MAX_NPPODS 4 /* 256/PPOD_SIZE */
1526 static inline void ulp_mem_io_set_hdr(struct cxgb4_lld_info *lldi,
1527                                 struct ulp_mem_io *req,
1528                                 unsigned int wr_len, unsigned int dlen,
1529                                 unsigned int pm_addr)
1530 {
1531         struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
1532
1533         INIT_ULPTX_WR(req, wr_len, 0, 0);
1534         if (is_t4(lldi->adapter_type))
1535                 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
1536                                         (ULP_MEMIO_ORDER_F));
1537         else
1538                 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) |
1539                                         (T5_ULP_MEMIO_IMM_F));
1540         req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5));
1541         req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5));
1542         req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
1543
1544         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
1545         idata->len = htonl(dlen);
1546 }
1547
1548 static int ddp_ppod_write_idata(struct cxgbi_device *cdev, unsigned int port_id,
1549                                 struct cxgbi_pagepod_hdr *hdr, unsigned int idx,
1550                                 unsigned int npods,
1551                                 struct cxgbi_gather_list *gl,
1552                                 unsigned int gl_pidx)
1553 {
1554         struct cxgbi_ddp_info *ddp = cdev->ddp;
1555         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1556         struct sk_buff *skb;
1557         struct ulp_mem_io *req;
1558         struct ulptx_idata *idata;
1559         struct cxgbi_pagepod *ppod;
1560         unsigned int pm_addr = idx * PPOD_SIZE + ddp->llimit;
1561         unsigned int dlen = PPOD_SIZE * npods;
1562         unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
1563                                 sizeof(struct ulptx_idata) + dlen, 16);
1564         unsigned int i;
1565
1566         skb = alloc_wr(wr_len, 0, GFP_ATOMIC);
1567         if (!skb) {
1568                 pr_err("cdev 0x%p, idx %u, npods %u, OOM.\n",
1569                         cdev, idx, npods);
1570                 return -ENOMEM;
1571         }
1572         req = (struct ulp_mem_io *)skb->head;
1573         set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
1574
1575         ulp_mem_io_set_hdr(lldi, req, wr_len, dlen, pm_addr);
1576         idata = (struct ulptx_idata *)(req + 1);
1577         ppod = (struct cxgbi_pagepod *)(idata + 1);
1578
1579         for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES_MAX) {
1580                 if (!hdr && !gl)
1581                         cxgbi_ddp_ppod_clear(ppod);
1582                 else
1583                         cxgbi_ddp_ppod_set(ppod, hdr, gl, gl_pidx);
1584         }
1585
1586         cxgb4_ofld_send(cdev->ports[port_id], skb);
1587         return 0;
1588 }
1589
1590 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
1591                         unsigned int idx, unsigned int npods,
1592                         struct cxgbi_gather_list *gl)
1593 {
1594         unsigned int i, cnt;
1595         int err = 0;
1596
1597         for (i = 0; i < npods; i += cnt, idx += cnt) {
1598                 cnt = npods - i;
1599                 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
1600                         cnt = ULPMEM_IDATA_MAX_NPPODS;
1601                 err = ddp_ppod_write_idata(csk->cdev, csk->port_id, hdr,
1602                                         idx, cnt, gl, 4 * i);
1603                 if (err < 0)
1604                         break;
1605         }
1606         return err;
1607 }
1608
1609 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
1610                           unsigned int idx, unsigned int npods)
1611 {
1612         unsigned int i, cnt;
1613         int err;
1614
1615         for (i = 0; i < npods; i += cnt, idx += cnt) {
1616                 cnt = npods - i;
1617                 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
1618                         cnt = ULPMEM_IDATA_MAX_NPPODS;
1619                 err = ddp_ppod_write_idata(chba->cdev, chba->port_id, NULL,
1620                                         idx, cnt, NULL, 0);
1621                 if (err < 0)
1622                         break;
1623         }
1624 }
1625
1626 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid,
1627                                 int pg_idx, bool reply)
1628 {
1629         struct sk_buff *skb;
1630         struct cpl_set_tcb_field *req;
1631
1632         if (!pg_idx || pg_idx >= DDP_PGIDX_MAX)
1633                 return 0;
1634
1635         skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1636         if (!skb)
1637                 return -ENOMEM;
1638
1639         /*  set up ulp page size */
1640         req = (struct cpl_set_tcb_field *)skb->head;
1641         INIT_TP_WR(req, csk->tid);
1642         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1643         req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid));
1644         req->word_cookie = htons(0);
1645         req->mask = cpu_to_be64(0x3 << 8);
1646         req->val = cpu_to_be64(pg_idx << 8);
1647         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1648
1649         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1650                 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx);
1651
1652         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1653         return 0;
1654 }
1655
1656 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1657                                  int hcrc, int dcrc, int reply)
1658 {
1659         struct sk_buff *skb;
1660         struct cpl_set_tcb_field *req;
1661
1662         if (!hcrc && !dcrc)
1663                 return 0;
1664
1665         skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1666         if (!skb)
1667                 return -ENOMEM;
1668
1669         csk->hcrc_len = (hcrc ? 4 : 0);
1670         csk->dcrc_len = (dcrc ? 4 : 0);
1671         /*  set up ulp submode */
1672         req = (struct cpl_set_tcb_field *)skb->head;
1673         INIT_TP_WR(req, tid);
1674         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1675         req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid));
1676         req->word_cookie = htons(0);
1677         req->mask = cpu_to_be64(0x3 << 4);
1678         req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
1679                                 (dcrc ? ULP_CRC_DATA : 0)) << 4);
1680         set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1681
1682         log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1683                 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc);
1684
1685         cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1686         return 0;
1687 }
1688
1689 static int cxgb4i_ddp_init(struct cxgbi_device *cdev)
1690 {
1691         struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1692         struct cxgbi_ddp_info *ddp = cdev->ddp;
1693         unsigned int tagmask, pgsz_factor[4];
1694         int err;
1695
1696         if (ddp) {
1697                 kref_get(&ddp->refcnt);
1698                 pr_warn("cdev 0x%p, ddp 0x%p already set up.\n",
1699                         cdev, cdev->ddp);
1700                 return -EALREADY;
1701         }
1702
1703         err = cxgbi_ddp_init(cdev, lldi->vr->iscsi.start,
1704                         lldi->vr->iscsi.start + lldi->vr->iscsi.size - 1,
1705                         lldi->iscsi_iolen, lldi->iscsi_iolen);
1706         if (err < 0)
1707                 return err;
1708
1709         ddp = cdev->ddp;
1710
1711         tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
1712         cxgbi_ddp_page_size_factor(pgsz_factor);
1713         cxgb4_iscsi_init(lldi->ports[0], tagmask, pgsz_factor);
1714
1715         cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1716         cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1717         cdev->csk_ddp_set = ddp_set_map;
1718         cdev->csk_ddp_clear = ddp_clear_map;
1719
1720         pr_info("cxgb4i 0x%p tag: sw %u, rsvd %u,%u, mask 0x%x.\n",
1721                 cdev, cdev->tag_format.sw_bits, cdev->tag_format.rsvd_bits,
1722                 cdev->tag_format.rsvd_shift, cdev->tag_format.rsvd_mask);
1723         pr_info("cxgb4i 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1724                 " %u/%u.\n",
1725                 cdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
1726                 ddp->rsvd_tag_mask, ddp->max_txsz, lldi->iscsi_iolen,
1727                 ddp->max_rxsz, lldi->iscsi_iolen);
1728         pr_info("cxgb4i 0x%p max payload size: %u/%u, %u/%u.\n",
1729                 cdev, cdev->tx_max_size, ddp->max_txsz, cdev->rx_max_size,
1730                 ddp->max_rxsz);
1731         return 0;
1732 }
1733
1734 static void *t4_uld_add(const struct cxgb4_lld_info *lldi)
1735 {
1736         struct cxgbi_device *cdev;
1737         struct port_info *pi;
1738         int i, rc;
1739
1740         cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
1741         if (!cdev) {
1742                 pr_info("t4 device 0x%p, register failed.\n", lldi);
1743                 return NULL;
1744         }
1745         pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
1746                 cdev, lldi->adapter_type, lldi->nports,
1747                 lldi->ports[0]->name, lldi->nchan, lldi->ntxq,
1748                 lldi->nrxq, lldi->wr_cred);
1749         for (i = 0; i < lldi->nrxq; i++)
1750                 log_debug(1 << CXGBI_DBG_DEV,
1751                         "t4 0x%p, rxq id #%d: %u.\n",
1752                         cdev, i, lldi->rxq_ids[i]);
1753
1754         memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi));
1755         cdev->flags = CXGBI_FLAG_DEV_T4;
1756         cdev->pdev = lldi->pdev;
1757         cdev->ports = lldi->ports;
1758         cdev->nports = lldi->nports;
1759         cdev->mtus = lldi->mtus;
1760         cdev->nmtus = NMTUS;
1761         cdev->snd_win = cxgb4i_snd_win;
1762         cdev->rcv_win = cxgb4i_rcv_win;
1763         cdev->rx_credit_thres = cxgb4i_rx_credit_thres;
1764         cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN;
1765         cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr);
1766         cdev->itp = &cxgb4i_iscsi_transport;
1767
1768         cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
1769                         << FW_VIID_PFN_S;
1770         pr_info("cdev 0x%p,%s, pfvf %u.\n",
1771                 cdev, lldi->ports[0]->name, cdev->pfvf);
1772
1773         rc = cxgb4i_ddp_init(cdev);
1774         if (rc) {
1775                 pr_info("t4 0x%p ddp init failed.\n", cdev);
1776                 goto err_out;
1777         }
1778         rc = cxgb4i_ofld_init(cdev);
1779         if (rc) {
1780                 pr_info("t4 0x%p ofld init failed.\n", cdev);
1781                 goto err_out;
1782         }
1783
1784         rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN,
1785                                 &cxgb4i_host_template, cxgb4i_stt);
1786         if (rc)
1787                 goto err_out;
1788
1789         for (i = 0; i < cdev->nports; i++) {
1790                 pi = netdev_priv(lldi->ports[i]);
1791                 cdev->hbas[i]->port_id = pi->port_id;
1792         }
1793         return cdev;
1794
1795 err_out:
1796         cxgbi_device_unregister(cdev);
1797         return ERR_PTR(-ENOMEM);
1798 }
1799
1800 #define RX_PULL_LEN     128
1801 static int t4_uld_rx_handler(void *handle, const __be64 *rsp,
1802                                 const struct pkt_gl *pgl)
1803 {
1804         const struct cpl_act_establish *rpl;
1805         struct sk_buff *skb;
1806         unsigned int opc;
1807         struct cxgbi_device *cdev = handle;
1808
1809         if (pgl == NULL) {
1810                 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8;
1811
1812                 skb = alloc_wr(len, 0, GFP_ATOMIC);
1813                 if (!skb)
1814                         goto nomem;
1815                 skb_copy_to_linear_data(skb, &rsp[1], len);
1816         } else {
1817                 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) {
1818                         pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
1819                                 pgl->va, be64_to_cpu(*rsp),
1820                                 be64_to_cpu(*(u64 *)pgl->va),
1821                                 pgl->tot_len);
1822                         return 0;
1823                 }
1824                 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN);
1825                 if (unlikely(!skb))
1826                         goto nomem;
1827         }
1828
1829         rpl = (struct cpl_act_establish *)skb->data;
1830         opc = rpl->ot.opcode;
1831         log_debug(1 << CXGBI_DBG_TOE,
1832                 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
1833                  cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb);
1834         if (cxgb4i_cplhandlers[opc])
1835                 cxgb4i_cplhandlers[opc](cdev, skb);
1836         else {
1837                 pr_err("No handler for opcode 0x%x.\n", opc);
1838                 __kfree_skb(skb);
1839         }
1840         return 0;
1841 nomem:
1842         log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n");
1843         return 1;
1844 }
1845
1846 static int t4_uld_state_change(void *handle, enum cxgb4_state state)
1847 {
1848         struct cxgbi_device *cdev = handle;
1849
1850         switch (state) {
1851         case CXGB4_STATE_UP:
1852                 pr_info("cdev 0x%p, UP.\n", cdev);
1853                 break;
1854         case CXGB4_STATE_START_RECOVERY:
1855                 pr_info("cdev 0x%p, RECOVERY.\n", cdev);
1856                 /* close all connections */
1857                 break;
1858         case CXGB4_STATE_DOWN:
1859                 pr_info("cdev 0x%p, DOWN.\n", cdev);
1860                 break;
1861         case CXGB4_STATE_DETACH:
1862                 pr_info("cdev 0x%p, DETACH.\n", cdev);
1863                 cxgbi_device_unregister(cdev);
1864                 break;
1865         default:
1866                 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state);
1867                 break;
1868         }
1869         return 0;
1870 }
1871
1872 static int __init cxgb4i_init_module(void)
1873 {
1874         int rc;
1875
1876         printk(KERN_INFO "%s", version);
1877
1878         rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1879         if (rc < 0)
1880                 return rc;
1881         cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info);
1882
1883         return 0;
1884 }
1885
1886 static void __exit cxgb4i_exit_module(void)
1887 {
1888         cxgb4_unregister_uld(CXGB4_ULD_ISCSI);
1889         cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4);
1890         cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1891 }
1892
1893 module_init(cxgb4i_init_module);
1894 module_exit(cxgb4i_exit_module);