kernel/drivers/infiniband/core/verbs.c

   1 /*
   2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
   3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
   5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
   7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   9  *
  10  * This software is available to you under a choice of one of two
  11  * licenses.  You may choose to be licensed under the terms of the GNU
  12  * General Public License (GPL) Version 2, available from the file
  13  * COPYING in the main directory of this source tree, or the
  14  * OpenIB.org BSD license below:
  15  *
  16  *     Redistribution and use in source and binary forms, with or
  17  *     without modification, are permitted provided that the following
  18  *     conditions are met:
  19  *
  20  *      - Redistributions of source code must retain the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer.
  23  *
  24  *      - Redistributions in binary form must reproduce the above
  25  *        copyright notice, this list of conditions and the following
  26  *        disclaimer in the documentation and/or other materials
  27  *        provided with the distribution.
  28  *
  29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  36  * SOFTWARE.
  37  */
  38
  39 #include <linux/errno.h>
  40 #include <linux/err.h>
  41 #include <linux/export.h>
  42 #include <linux/string.h>
  43 #include <linux/slab.h>
  44
  45 #include <rdma/ib_verbs.h>
  46 #include <rdma/ib_cache.h>
  47 #include <rdma/ib_addr.h>
  48
  49 #include "core_priv.h"
  50
  51 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
  52 {
  53         switch (rate) {
  54         case IB_RATE_2_5_GBPS: return  1;
  55         case IB_RATE_5_GBPS:   return  2;
  56         case IB_RATE_10_GBPS:  return  4;
  57         case IB_RATE_20_GBPS:  return  8;
  58         case IB_RATE_30_GBPS:  return 12;
  59         case IB_RATE_40_GBPS:  return 16;
  60         case IB_RATE_60_GBPS:  return 24;
  61         case IB_RATE_80_GBPS:  return 32;
  62         case IB_RATE_120_GBPS: return 48;
  63         default:               return -1;
  64         }
  65 }
  66 EXPORT_SYMBOL(ib_rate_to_mult);
  67
  68 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
  69 {
  70         switch (mult) {
  71         case 1:  return IB_RATE_2_5_GBPS;
  72         case 2:  return IB_RATE_5_GBPS;
  73         case 4:  return IB_RATE_10_GBPS;
  74         case 8:  return IB_RATE_20_GBPS;
  75         case 12: return IB_RATE_30_GBPS;
  76         case 16: return IB_RATE_40_GBPS;
  77         case 24: return IB_RATE_60_GBPS;
  78         case 32: return IB_RATE_80_GBPS;
  79         case 48: return IB_RATE_120_GBPS;
  80         default: return IB_RATE_PORT_CURRENT;
  81         }
  82 }
  83 EXPORT_SYMBOL(mult_to_ib_rate);
  84
  85 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
  86 {
  87         switch (rate) {
  88         case IB_RATE_2_5_GBPS: return 2500;
  89         case IB_RATE_5_GBPS:   return 5000;
  90         case IB_RATE_10_GBPS:  return 10000;
  91         case IB_RATE_20_GBPS:  return 20000;
  92         case IB_RATE_30_GBPS:  return 30000;
  93         case IB_RATE_40_GBPS:  return 40000;
  94         case IB_RATE_60_GBPS:  return 60000;
  95         case IB_RATE_80_GBPS:  return 80000;
  96         case IB_RATE_120_GBPS: return 120000;
  97         case IB_RATE_14_GBPS:  return 14062;
  98         case IB_RATE_56_GBPS:  return 56250;
  99         case IB_RATE_112_GBPS: return 112500;
 100         case IB_RATE_168_GBPS: return 168750;
 101         case IB_RATE_25_GBPS:  return 25781;
 102         case IB_RATE_100_GBPS: return 103125;
 103         case IB_RATE_200_GBPS: return 206250;
 104         case IB_RATE_300_GBPS: return 309375;
 105         default:               return -1;
 106         }
 107 }
 108 EXPORT_SYMBOL(ib_rate_to_mbps);
 109
 110 __attribute_const__ enum rdma_transport_type
 111 rdma_node_get_transport(enum rdma_node_type node_type)
 112 {
 113         switch (node_type) {
 114         case RDMA_NODE_IB_CA:
 115         case RDMA_NODE_IB_SWITCH:
 116         case RDMA_NODE_IB_ROUTER:
 117                 return RDMA_TRANSPORT_IB;
 118         case RDMA_NODE_RNIC:
 119                 return RDMA_TRANSPORT_IWARP;
 120         case RDMA_NODE_USNIC:
 121                 return RDMA_TRANSPORT_USNIC;
 122         case RDMA_NODE_USNIC_UDP:
 123                 return RDMA_TRANSPORT_USNIC_UDP;
 124         default:
 125                 BUG();
 126                 return 0;
 127         }
 128 }
 129 EXPORT_SYMBOL(rdma_node_get_transport);
 130
 131 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
 132 {
 133         if (device->get_link_layer)
 134                 return device->get_link_layer(device, port_num);
 135
 136         switch (rdma_node_get_transport(device->node_type)) {
 137         case RDMA_TRANSPORT_IB:
 138                 return IB_LINK_LAYER_INFINIBAND;
 139         case RDMA_TRANSPORT_IWARP:
 140         case RDMA_TRANSPORT_USNIC:
 141         case RDMA_TRANSPORT_USNIC_UDP:
 142                 return IB_LINK_LAYER_ETHERNET;
 143         default:
 144                 return IB_LINK_LAYER_UNSPECIFIED;
 145         }
 146 }
 147 EXPORT_SYMBOL(rdma_port_get_link_layer);
 148
 149 /* Protection domains */
 150
 151 struct ib_pd *ib_alloc_pd(struct ib_device *device)
 152 {
 153         struct ib_pd *pd;
 154
 155         pd = device->alloc_pd(device, NULL, NULL);
 156
 157         if (!IS_ERR(pd)) {
 158                 pd->device  = device;
 159                 pd->uobject = NULL;
 160                 atomic_set(&pd->usecnt, 0);
 161         }
 162
 163         return pd;
 164 }
 165 EXPORT_SYMBOL(ib_alloc_pd);
 166
 167 int ib_dealloc_pd(struct ib_pd *pd)
 168 {
 169         if (atomic_read(&pd->usecnt))
 170                 return -EBUSY;
 171
 172         return pd->device->dealloc_pd(pd);
 173 }
 174 EXPORT_SYMBOL(ib_dealloc_pd);
 175
 176 /* Address handles */
 177
 178 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
 179 {
 180         struct ib_ah *ah;
 181
 182         ah = pd->device->create_ah(pd, ah_attr);
 183
 184         if (!IS_ERR(ah)) {
 185                 ah->device  = pd->device;
 186                 ah->pd      = pd;
 187                 ah->uobject = NULL;
 188                 atomic_inc(&pd->usecnt);
 189         }
 190
 191         return ah;
 192 }
 193 EXPORT_SYMBOL(ib_create_ah);
 194
 195 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
 196                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
 197 {
 198         u32 flow_class;
 199         u16 gid_index;
 200         int ret;
 201         int is_eth = (rdma_port_get_link_layer(device, port_num) ==
 202                         IB_LINK_LAYER_ETHERNET);
 203
 204         memset(ah_attr, 0, sizeof *ah_attr);
 205         if (is_eth) {
 206                 if (!(wc->wc_flags & IB_WC_GRH))
 207                         return -EPROTOTYPE;
 208
 209                 if (wc->wc_flags & IB_WC_WITH_SMAC &&
 210                     wc->wc_flags & IB_WC_WITH_VLAN) {
 211                         memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
 212                         ah_attr->vlan_id = wc->vlan_id;
 213                 } else {
 214                         ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
 215                                         ah_attr->dmac, &ah_attr->vlan_id);
 216                         if (ret)
 217                                 return ret;
 218                 }
 219         } else {
 220                 ah_attr->vlan_id = 0xffff;
 221         }
 222
 223         ah_attr->dlid = wc->slid;
 224         ah_attr->sl = wc->sl;
 225         ah_attr->src_path_bits = wc->dlid_path_bits;
 226         ah_attr->port_num = port_num;
 227
 228         if (wc->wc_flags & IB_WC_GRH) {
 229                 ah_attr->ah_flags = IB_AH_GRH;
 230                 ah_attr->grh.dgid = grh->sgid;
 231
 232                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
 233                                          &gid_index);
 234                 if (ret)
 235                         return ret;
 236
 237                 ah_attr->grh.sgid_index = (u8) gid_index;
 238                 flow_class = be32_to_cpu(grh->version_tclass_flow);
 239                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
 240                 ah_attr->grh.hop_limit = 0xFF;
 241                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
 242         }
 243         return 0;
 244 }
 245 EXPORT_SYMBOL(ib_init_ah_from_wc);
 246
 247 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
 248                                    struct ib_grh *grh, u8 port_num)
 249 {
 250         struct ib_ah_attr ah_attr;
 251         int ret;
 252
 253         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
 254         if (ret)
 255                 return ERR_PTR(ret);
 256
 257         return ib_create_ah(pd, &ah_attr);
 258 }
 259 EXPORT_SYMBOL(ib_create_ah_from_wc);
 260
 261 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 262 {
 263         return ah->device->modify_ah ?
 264                 ah->device->modify_ah(ah, ah_attr) :
 265                 -ENOSYS;
 266 }
 267 EXPORT_SYMBOL(ib_modify_ah);
 268
 269 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
 270 {
 271         return ah->device->query_ah ?
 272                 ah->device->query_ah(ah, ah_attr) :
 273                 -ENOSYS;
 274 }
 275 EXPORT_SYMBOL(ib_query_ah);
 276
 277 int ib_destroy_ah(struct ib_ah *ah)
 278 {
 279         struct ib_pd *pd;
 280         int ret;
 281
 282         pd = ah->pd;
 283         ret = ah->device->destroy_ah(ah);
 284         if (!ret)
 285                 atomic_dec(&pd->usecnt);
 286
 287         return ret;
 288 }
 289 EXPORT_SYMBOL(ib_destroy_ah);
 290
 291 /* Shared receive queues */
 292
 293 struct ib_srq *ib_create_srq(struct ib_pd *pd,
 294                              struct ib_srq_init_attr *srq_init_attr)
 295 {
 296         struct ib_srq *srq;
 297
 298         if (!pd->device->create_srq)
 299                 return ERR_PTR(-ENOSYS);
 300
 301         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
 302
 303         if (!IS_ERR(srq)) {
 304                 srq->device        = pd->device;
 305                 srq->pd            = pd;
 306                 srq->uobject       = NULL;
 307                 srq->event_handler = srq_init_attr->event_handler;
 308                 srq->srq_context   = srq_init_attr->srq_context;
 309                 srq->srq_type      = srq_init_attr->srq_type;
 310                 if (srq->srq_type == IB_SRQT_XRC) {
 311                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
 312                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
 313                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
 314                         atomic_inc(&srq->ext.xrc.cq->usecnt);
 315                 }
 316                 atomic_inc(&pd->usecnt);
 317                 atomic_set(&srq->usecnt, 0);
 318         }
 319
 320         return srq;
 321 }
 322 EXPORT_SYMBOL(ib_create_srq);
 323
 324 int ib_modify_srq(struct ib_srq *srq,
 325                   struct ib_srq_attr *srq_attr,
 326                   enum ib_srq_attr_mask srq_attr_mask)
 327 {
 328         return srq->device->modify_srq ?
 329                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
 330                 -ENOSYS;
 331 }
 332 EXPORT_SYMBOL(ib_modify_srq);
 333
 334 int ib_query_srq(struct ib_srq *srq,
 335                  struct ib_srq_attr *srq_attr)
 336 {
 337         return srq->device->query_srq ?
 338                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
 339 }
 340 EXPORT_SYMBOL(ib_query_srq);
 341
 342 int ib_destroy_srq(struct ib_srq *srq)
 343 {
 344         struct ib_pd *pd;
 345         enum ib_srq_type srq_type;
 346         struct ib_xrcd *uninitialized_var(xrcd);
 347         struct ib_cq *uninitialized_var(cq);
 348         int ret;
 349
 350         if (atomic_read(&srq->usecnt))
 351                 return -EBUSY;
 352
 353         pd = srq->pd;
 354         srq_type = srq->srq_type;
 355         if (srq_type == IB_SRQT_XRC) {
 356                 xrcd = srq->ext.xrc.xrcd;
 357                 cq = srq->ext.xrc.cq;
 358         }
 359
 360         ret = srq->device->destroy_srq(srq);
 361         if (!ret) {
 362                 atomic_dec(&pd->usecnt);
 363                 if (srq_type == IB_SRQT_XRC) {
 364                         atomic_dec(&xrcd->usecnt);
 365                         atomic_dec(&cq->usecnt);
 366                 }
 367         }
 368
 369         return ret;
 370 }
 371 EXPORT_SYMBOL(ib_destroy_srq);
 372
 373 /* Queue pairs */
 374
 375 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
 376 {
 377         struct ib_qp *qp = context;
 378         unsigned long flags;
 379
 380         spin_lock_irqsave(&qp->device->event_handler_lock, flags);
 381         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
 382                 if (event->element.qp->event_handler)
 383                         event->element.qp->event_handler(event, event->element.qp->qp_context);
 384         spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
 385 }
 386
 387 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
 388 {
 389         mutex_lock(&xrcd->tgt_qp_mutex);
 390         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
 391         mutex_unlock(&xrcd->tgt_qp_mutex);
 392 }
 393
 394 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
 395                                   void (*event_handler)(struct ib_event *, void *),
 396                                   void *qp_context)
 397 {
 398         struct ib_qp *qp;
 399         unsigned long flags;
 400
 401         qp = kzalloc(sizeof *qp, GFP_KERNEL);
 402         if (!qp)
 403                 return ERR_PTR(-ENOMEM);
 404
 405         qp->real_qp = real_qp;
 406         atomic_inc(&real_qp->usecnt);
 407         qp->device = real_qp->device;
 408         qp->event_handler = event_handler;
 409         qp->qp_context = qp_context;
 410         qp->qp_num = real_qp->qp_num;
 411         qp->qp_type = real_qp->qp_type;
 412
 413         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 414         list_add(&qp->open_list, &real_qp->open_list);
 415         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 416
 417         return qp;
 418 }
 419
 420 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
 421                          struct ib_qp_open_attr *qp_open_attr)
 422 {
 423         struct ib_qp *qp, *real_qp;
 424
 425         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
 426                 return ERR_PTR(-EINVAL);
 427
 428         qp = ERR_PTR(-EINVAL);
 429         mutex_lock(&xrcd->tgt_qp_mutex);
 430         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
 431                 if (real_qp->qp_num == qp_open_attr->qp_num) {
 432                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
 433                                           qp_open_attr->qp_context);
 434                         break;
 435                 }
 436         }
 437         mutex_unlock(&xrcd->tgt_qp_mutex);
 438         return qp;
 439 }
 440 EXPORT_SYMBOL(ib_open_qp);
 441
 442 struct ib_qp *ib_create_qp(struct ib_pd *pd,
 443                            struct ib_qp_init_attr *qp_init_attr)
 444 {
 445         struct ib_qp *qp, *real_qp;
 446         struct ib_device *device;
 447
 448         device = pd ? pd->device : qp_init_attr->xrcd->device;
 449         qp = device->create_qp(pd, qp_init_attr, NULL);
 450
 451         if (!IS_ERR(qp)) {
 452                 qp->device     = device;
 453                 qp->real_qp    = qp;
 454                 qp->uobject    = NULL;
 455                 qp->qp_type    = qp_init_attr->qp_type;
 456
 457                 atomic_set(&qp->usecnt, 0);
 458                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
 459                         qp->event_handler = __ib_shared_qp_event_handler;
 460                         qp->qp_context = qp;
 461                         qp->pd = NULL;
 462                         qp->send_cq = qp->recv_cq = NULL;
 463                         qp->srq = NULL;
 464                         qp->xrcd = qp_init_attr->xrcd;
 465                         atomic_inc(&qp_init_attr->xrcd->usecnt);
 466                         INIT_LIST_HEAD(&qp->open_list);
 467
 468                         real_qp = qp;
 469                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
 470                                           qp_init_attr->qp_context);
 471                         if (!IS_ERR(qp))
 472                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
 473                         else
 474                                 real_qp->device->destroy_qp(real_qp);
 475                 } else {
 476                         qp->event_handler = qp_init_attr->event_handler;
 477                         qp->qp_context = qp_init_attr->qp_context;
 478                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
 479                                 qp->recv_cq = NULL;
 480                                 qp->srq = NULL;
 481                         } else {
 482                                 qp->recv_cq = qp_init_attr->recv_cq;
 483                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
 484                                 qp->srq = qp_init_attr->srq;
 485                                 if (qp->srq)
 486                                         atomic_inc(&qp_init_attr->srq->usecnt);
 487                         }
 488
 489                         qp->pd      = pd;
 490                         qp->send_cq = qp_init_attr->send_cq;
 491                         qp->xrcd    = NULL;
 492
 493                         atomic_inc(&pd->usecnt);
 494                         atomic_inc(&qp_init_attr->send_cq->usecnt);
 495                 }
 496         }
 497
 498         return qp;
 499 }
 500 EXPORT_SYMBOL(ib_create_qp);
 501
 502 static const struct {
 503         int                     valid;
 504         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
 505         enum ib_qp_attr_mask    req_param_add_eth[IB_QPT_MAX];
 506         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
 507         enum ib_qp_attr_mask    opt_param_add_eth[IB_QPT_MAX];
 508 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
 509         [IB_QPS_RESET] = {
 510                 [IB_QPS_RESET] = { .valid = 1 },
 511                 [IB_QPS_INIT]  = {
 512                         .valid = 1,
 513                         .req_param = {
 514                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 515                                                 IB_QP_PORT                      |
 516                                                 IB_QP_QKEY),
 517                                 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
 518                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 519                                                 IB_QP_PORT                      |
 520                                                 IB_QP_ACCESS_FLAGS),
 521                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 522                                                 IB_QP_PORT                      |
 523                                                 IB_QP_ACCESS_FLAGS),
 524                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 525                                                 IB_QP_PORT                      |
 526                                                 IB_QP_ACCESS_FLAGS),
 527                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 528                                                 IB_QP_PORT                      |
 529                                                 IB_QP_ACCESS_FLAGS),
 530                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 531                                                 IB_QP_QKEY),
 532                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 533                                                 IB_QP_QKEY),
 534                         }
 535                 },
 536         },
 537         [IB_QPS_INIT]  = {
 538                 [IB_QPS_RESET] = { .valid = 1 },
 539                 [IB_QPS_ERR] =   { .valid = 1 },
 540                 [IB_QPS_INIT]  = {
 541                         .valid = 1,
 542                         .opt_param = {
 543                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 544                                                 IB_QP_PORT                      |
 545                                                 IB_QP_QKEY),
 546                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
 547                                                 IB_QP_PORT                      |
 548                                                 IB_QP_ACCESS_FLAGS),
 549                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
 550                                                 IB_QP_PORT                      |
 551                                                 IB_QP_ACCESS_FLAGS),
 552                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
 553                                                 IB_QP_PORT                      |
 554                                                 IB_QP_ACCESS_FLAGS),
 555                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
 556                                                 IB_QP_PORT                      |
 557                                                 IB_QP_ACCESS_FLAGS),
 558                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 559                                                 IB_QP_QKEY),
 560                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 561                                                 IB_QP_QKEY),
 562                         }
 563                 },
 564                 [IB_QPS_RTR]   = {
 565                         .valid = 1,
 566                         .req_param = {
 567                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 568                                                 IB_QP_PATH_MTU                  |
 569                                                 IB_QP_DEST_QPN                  |
 570                                                 IB_QP_RQ_PSN),
 571                                 [IB_QPT_RC]  = (IB_QP_AV                        |
 572                                                 IB_QP_PATH_MTU                  |
 573                                                 IB_QP_DEST_QPN                  |
 574                                                 IB_QP_RQ_PSN                    |
 575                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 576                                                 IB_QP_MIN_RNR_TIMER),
 577                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
 578                                                 IB_QP_PATH_MTU                  |
 579                                                 IB_QP_DEST_QPN                  |
 580                                                 IB_QP_RQ_PSN),
 581                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
 582                                                 IB_QP_PATH_MTU                  |
 583                                                 IB_QP_DEST_QPN                  |
 584                                                 IB_QP_RQ_PSN                    |
 585                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 586                                                 IB_QP_MIN_RNR_TIMER),
 587                         },
 588                         .req_param_add_eth = {
 589                                 [IB_QPT_RC]  = (IB_QP_SMAC),
 590                                 [IB_QPT_UC]  = (IB_QP_SMAC),
 591                                 [IB_QPT_XRC_INI]  = (IB_QP_SMAC),
 592                                 [IB_QPT_XRC_TGT]  = (IB_QP_SMAC)
 593                         },
 594                         .opt_param = {
 595                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
 596                                                  IB_QP_QKEY),
 597                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
 598                                                  IB_QP_ACCESS_FLAGS             |
 599                                                  IB_QP_PKEY_INDEX),
 600                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
 601                                                  IB_QP_ACCESS_FLAGS             |
 602                                                  IB_QP_PKEY_INDEX),
 603                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
 604                                                  IB_QP_ACCESS_FLAGS             |
 605                                                  IB_QP_PKEY_INDEX),
 606                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
 607                                                  IB_QP_ACCESS_FLAGS             |
 608                                                  IB_QP_PKEY_INDEX),
 609                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
 610                                                  IB_QP_QKEY),
 611                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
 612                                                  IB_QP_QKEY),
 613                          },
 614                         .opt_param_add_eth = {
 615                                 [IB_QPT_RC]  = (IB_QP_ALT_SMAC                  |
 616                                                 IB_QP_VID                       |
 617                                                 IB_QP_ALT_VID),
 618                                 [IB_QPT_UC]  = (IB_QP_ALT_SMAC                  |
 619                                                 IB_QP_VID                       |
 620                                                 IB_QP_ALT_VID),
 621                                 [IB_QPT_XRC_INI]  = (IB_QP_ALT_SMAC                     |
 622                                                 IB_QP_VID                       |
 623                                                 IB_QP_ALT_VID),
 624                                 [IB_QPT_XRC_TGT]  = (IB_QP_ALT_SMAC                     |
 625                                                 IB_QP_VID                       |
 626                                                 IB_QP_ALT_VID)
 627                         }
 628                 }
 629         },
 630         [IB_QPS_RTR]   = {
 631                 [IB_QPS_RESET] = { .valid = 1 },
 632                 [IB_QPS_ERR] =   { .valid = 1 },
 633                 [IB_QPS_RTS]   = {
 634                         .valid = 1,
 635                         .req_param = {
 636                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
 637                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
 638                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
 639                                                 IB_QP_RETRY_CNT                 |
 640                                                 IB_QP_RNR_RETRY                 |
 641                                                 IB_QP_SQ_PSN                    |
 642                                                 IB_QP_MAX_QP_RD_ATOMIC),
 643                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
 644                                                 IB_QP_RETRY_CNT                 |
 645                                                 IB_QP_RNR_RETRY                 |
 646                                                 IB_QP_SQ_PSN                    |
 647                                                 IB_QP_MAX_QP_RD_ATOMIC),
 648                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
 649                                                 IB_QP_SQ_PSN),
 650                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
 651                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
 652                         },
 653                         .opt_param = {
 654                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
 655                                                  IB_QP_QKEY),
 656                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
 657                                                  IB_QP_ALT_PATH                 |
 658                                                  IB_QP_ACCESS_FLAGS             |
 659                                                  IB_QP_PATH_MIG_STATE),
 660                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
 661                                                  IB_QP_ALT_PATH                 |
 662                                                  IB_QP_ACCESS_FLAGS             |
 663                                                  IB_QP_MIN_RNR_TIMER            |
 664                                                  IB_QP_PATH_MIG_STATE),
 665                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
 666                                                  IB_QP_ALT_PATH                 |
 667                                                  IB_QP_ACCESS_FLAGS             |
 668                                                  IB_QP_PATH_MIG_STATE),
 669                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
 670                                                  IB_QP_ALT_PATH                 |
 671                                                  IB_QP_ACCESS_FLAGS             |
 672                                                  IB_QP_MIN_RNR_TIMER            |
 673                                                  IB_QP_PATH_MIG_STATE),
 674                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
 675                                                  IB_QP_QKEY),
 676                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
 677                                                  IB_QP_QKEY),
 678                          }
 679                 }
 680         },
 681         [IB_QPS_RTS]   = {
 682                 [IB_QPS_RESET] = { .valid = 1 },
 683                 [IB_QPS_ERR] =   { .valid = 1 },
 684                 [IB_QPS_RTS]   = {
 685                         .valid = 1,
 686                         .opt_param = {
 687                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 688                                                 IB_QP_QKEY),
 689                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 690                                                 IB_QP_ACCESS_FLAGS              |
 691                                                 IB_QP_ALT_PATH                  |
 692                                                 IB_QP_PATH_MIG_STATE),
 693                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 694                                                 IB_QP_ACCESS_FLAGS              |
 695                                                 IB_QP_ALT_PATH                  |
 696                                                 IB_QP_PATH_MIG_STATE            |
 697                                                 IB_QP_MIN_RNR_TIMER),
 698                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 699                                                 IB_QP_ACCESS_FLAGS              |
 700                                                 IB_QP_ALT_PATH                  |
 701                                                 IB_QP_PATH_MIG_STATE),
 702                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 703                                                 IB_QP_ACCESS_FLAGS              |
 704                                                 IB_QP_ALT_PATH                  |
 705                                                 IB_QP_PATH_MIG_STATE            |
 706                                                 IB_QP_MIN_RNR_TIMER),
 707                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 708                                                 IB_QP_QKEY),
 709                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 710                                                 IB_QP_QKEY),
 711                         }
 712                 },
 713                 [IB_QPS_SQD]   = {
 714                         .valid = 1,
 715                         .opt_param = {
 716                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 717                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 718                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
 719                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 720                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
 721                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
 722                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
 723                         }
 724                 },
 725         },
 726         [IB_QPS_SQD]   = {
 727                 [IB_QPS_RESET] = { .valid = 1 },
 728                 [IB_QPS_ERR] =   { .valid = 1 },
 729                 [IB_QPS_RTS]   = {
 730                         .valid = 1,
 731                         .opt_param = {
 732                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 733                                                 IB_QP_QKEY),
 734                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 735                                                 IB_QP_ALT_PATH                  |
 736                                                 IB_QP_ACCESS_FLAGS              |
 737                                                 IB_QP_PATH_MIG_STATE),
 738                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
 739                                                 IB_QP_ALT_PATH                  |
 740                                                 IB_QP_ACCESS_FLAGS              |
 741                                                 IB_QP_MIN_RNR_TIMER             |
 742                                                 IB_QP_PATH_MIG_STATE),
 743                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
 744                                                 IB_QP_ALT_PATH                  |
 745                                                 IB_QP_ACCESS_FLAGS              |
 746                                                 IB_QP_PATH_MIG_STATE),
 747                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
 748                                                 IB_QP_ALT_PATH                  |
 749                                                 IB_QP_ACCESS_FLAGS              |
 750                                                 IB_QP_MIN_RNR_TIMER             |
 751                                                 IB_QP_PATH_MIG_STATE),
 752                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 753                                                 IB_QP_QKEY),
 754                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 755                                                 IB_QP_QKEY),
 756                         }
 757                 },
 758                 [IB_QPS_SQD]   = {
 759                         .valid = 1,
 760                         .opt_param = {
 761                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
 762                                                 IB_QP_QKEY),
 763                                 [IB_QPT_UC]  = (IB_QP_AV                        |
 764                                                 IB_QP_ALT_PATH                  |
 765                                                 IB_QP_ACCESS_FLAGS              |
 766                                                 IB_QP_PKEY_INDEX                |
 767                                                 IB_QP_PATH_MIG_STATE),
 768                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
 769                                                 IB_QP_AV                        |
 770                                                 IB_QP_TIMEOUT                   |
 771                                                 IB_QP_RETRY_CNT                 |
 772                                                 IB_QP_RNR_RETRY                 |
 773                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 774                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 775                                                 IB_QP_ALT_PATH                  |
 776                                                 IB_QP_ACCESS_FLAGS              |
 777                                                 IB_QP_PKEY_INDEX                |
 778                                                 IB_QP_MIN_RNR_TIMER             |
 779                                                 IB_QP_PATH_MIG_STATE),
 780                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
 781                                                 IB_QP_AV                        |
 782                                                 IB_QP_TIMEOUT                   |
 783                                                 IB_QP_RETRY_CNT                 |
 784                                                 IB_QP_RNR_RETRY                 |
 785                                                 IB_QP_MAX_QP_RD_ATOMIC          |
 786                                                 IB_QP_ALT_PATH                  |
 787                                                 IB_QP_ACCESS_FLAGS              |
 788                                                 IB_QP_PKEY_INDEX                |
 789                                                 IB_QP_PATH_MIG_STATE),
 790                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
 791                                                 IB_QP_AV                        |
 792                                                 IB_QP_TIMEOUT                   |
 793                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
 794                                                 IB_QP_ALT_PATH                  |
 795                                                 IB_QP_ACCESS_FLAGS              |
 796                                                 IB_QP_PKEY_INDEX                |
 797                                                 IB_QP_MIN_RNR_TIMER             |
 798                                                 IB_QP_PATH_MIG_STATE),
 799                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
 800                                                 IB_QP_QKEY),
 801                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
 802                                                 IB_QP_QKEY),
 803                         }
 804                 }
 805         },
 806         [IB_QPS_SQE]   = {
 807                 [IB_QPS_RESET] = { .valid = 1 },
 808                 [IB_QPS_ERR] =   { .valid = 1 },
 809                 [IB_QPS_RTS]   = {
 810                         .valid = 1,
 811                         .opt_param = {
 812                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
 813                                                 IB_QP_QKEY),
 814                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
 815                                                 IB_QP_ACCESS_FLAGS),
 816                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
 817                                                 IB_QP_QKEY),
 818                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
 819                                                 IB_QP_QKEY),
 820                         }
 821                 }
 822         },
 823         [IB_QPS_ERR] = {
 824                 [IB_QPS_RESET] = { .valid = 1 },
 825                 [IB_QPS_ERR] =   { .valid = 1 }
 826         }
 827 };
 828
 829 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
 830                        enum ib_qp_type type, enum ib_qp_attr_mask mask,
 831                        enum rdma_link_layer ll)
 832 {
 833         enum ib_qp_attr_mask req_param, opt_param;
 834
 835         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
 836             next_state < 0 || next_state > IB_QPS_ERR)
 837                 return 0;
 838
 839         if (mask & IB_QP_CUR_STATE  &&
 840             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
 841             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
 842                 return 0;
 843
 844         if (!qp_state_table[cur_state][next_state].valid)
 845                 return 0;
 846
 847         req_param = qp_state_table[cur_state][next_state].req_param[type];
 848         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
 849
 850         if (ll == IB_LINK_LAYER_ETHERNET) {
 851                 req_param |= qp_state_table[cur_state][next_state].
 852                         req_param_add_eth[type];
 853                 opt_param |= qp_state_table[cur_state][next_state].
 854                         opt_param_add_eth[type];
 855         }
 856
 857         if ((mask & req_param) != req_param)
 858                 return 0;
 859
 860         if (mask & ~(req_param | opt_param | IB_QP_STATE))
 861                 return 0;
 862
 863         return 1;
 864 }
 865 EXPORT_SYMBOL(ib_modify_qp_is_ok);
 866
 867 int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
 868                             struct ib_qp_attr *qp_attr, int *qp_attr_mask)
 869 {
 870         int           ret = 0;
 871         union ib_gid  sgid;
 872
 873         if ((*qp_attr_mask & IB_QP_AV)  &&
 874             (rdma_port_get_link_layer(qp->device, qp_attr->ah_attr.port_num) == IB_LINK_LAYER_ETHERNET)) {
 875                 ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
 876                                    qp_attr->ah_attr.grh.sgid_index, &sgid);
 877                 if (ret)
 878                         goto out;
 879                 if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
 880                         rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
 881                         rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr->smac);
 882                         if (!(*qp_attr_mask & IB_QP_VID))
 883                                 qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
 884                 } else {
 885                         ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr->ah_attr.grh.dgid,
 886                                         qp_attr->ah_attr.dmac, &qp_attr->vlan_id);
 887                         if (ret)
 888                                 goto out;
 889                         ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr->smac, NULL);
 890                         if (ret)
 891                                 goto out;
 892                 }
 893                 *qp_attr_mask |= IB_QP_SMAC;
 894                 if (qp_attr->vlan_id < 0xFFFF)
 895                         *qp_attr_mask |= IB_QP_VID;
 896         }
 897 out:
 898         return ret;
 899 }
 900 EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);
 901
 902
 903 int ib_modify_qp(struct ib_qp *qp,
 904                  struct ib_qp_attr *qp_attr,
 905                  int qp_attr_mask)
 906 {
 907         int ret;
 908
 909         ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
 910         if (ret)
 911                 return ret;
 912
 913         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
 914 }
 915 EXPORT_SYMBOL(ib_modify_qp);
 916
 917 int ib_query_qp(struct ib_qp *qp,
 918                 struct ib_qp_attr *qp_attr,
 919                 int qp_attr_mask,
 920                 struct ib_qp_init_attr *qp_init_attr)
 921 {
 922         return qp->device->query_qp ?
 923                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
 924                 -ENOSYS;
 925 }
 926 EXPORT_SYMBOL(ib_query_qp);
 927
 928 int ib_close_qp(struct ib_qp *qp)
 929 {
 930         struct ib_qp *real_qp;
 931         unsigned long flags;
 932
 933         real_qp = qp->real_qp;
 934         if (real_qp == qp)
 935                 return -EINVAL;
 936
 937         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
 938         list_del(&qp->open_list);
 939         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
 940
 941         atomic_dec(&real_qp->usecnt);
 942         kfree(qp);
 943
 944         return 0;
 945 }
 946 EXPORT_SYMBOL(ib_close_qp);
 947
 948 static int __ib_destroy_shared_qp(struct ib_qp *qp)
 949 {
 950         struct ib_xrcd *xrcd;
 951         struct ib_qp *real_qp;
 952         int ret;
 953
 954         real_qp = qp->real_qp;
 955         xrcd = real_qp->xrcd;
 956
 957         mutex_lock(&xrcd->tgt_qp_mutex);
 958         ib_close_qp(qp);
 959         if (atomic_read(&real_qp->usecnt) == 0)
 960                 list_del(&real_qp->xrcd_list);
 961         else
 962                 real_qp = NULL;
 963         mutex_unlock(&xrcd->tgt_qp_mutex);
 964
 965         if (real_qp) {
 966                 ret = ib_destroy_qp(real_qp);
 967                 if (!ret)
 968                         atomic_dec(&xrcd->usecnt);
 969                 else
 970                         __ib_insert_xrcd_qp(xrcd, real_qp);
 971         }
 972
 973         return 0;
 974 }
 975
 976 int ib_destroy_qp(struct ib_qp *qp)
 977 {
 978         struct ib_pd *pd;
 979         struct ib_cq *scq, *rcq;
 980         struct ib_srq *srq;
 981         int ret;
 982
 983         if (atomic_read(&qp->usecnt))
 984                 return -EBUSY;
 985
 986         if (qp->real_qp != qp)
 987                 return __ib_destroy_shared_qp(qp);
 988
 989         pd   = qp->pd;
 990         scq  = qp->send_cq;
 991         rcq  = qp->recv_cq;
 992         srq  = qp->srq;
 993
 994         ret = qp->device->destroy_qp(qp);
 995         if (!ret) {
 996                 if (pd)
 997                         atomic_dec(&pd->usecnt);
 998                 if (scq)
 999                         atomic_dec(&scq->usecnt);
1000                 if (rcq)
1001                         atomic_dec(&rcq->usecnt);
1002                 if (srq)
1003                         atomic_dec(&srq->usecnt);
1004         }
1005
1006         return ret;
1007 }
1008 EXPORT_SYMBOL(ib_destroy_qp);
1009
1010 /* Completion queues */
1011
1012 struct ib_cq *ib_create_cq(struct ib_device *device,
1013                            ib_comp_handler comp_handler,
1014                            void (*event_handler)(struct ib_event *, void *),
1015                            void *cq_context, int cqe, int comp_vector)
1016 {
1017         struct ib_cq *cq;
1018
1019         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
1020
1021         if (!IS_ERR(cq)) {
1022                 cq->device        = device;
1023                 cq->uobject       = NULL;
1024                 cq->comp_handler  = comp_handler;
1025                 cq->event_handler = event_handler;
1026                 cq->cq_context    = cq_context;
1027                 atomic_set(&cq->usecnt, 0);
1028         }
1029
1030         return cq;
1031 }
1032 EXPORT_SYMBOL(ib_create_cq);
1033
1034 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1035 {
1036         return cq->device->modify_cq ?
1037                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
1038 }
1039 EXPORT_SYMBOL(ib_modify_cq);
1040
1041 int ib_destroy_cq(struct ib_cq *cq)
1042 {
1043         if (atomic_read(&cq->usecnt))
1044                 return -EBUSY;
1045
1046         return cq->device->destroy_cq(cq);
1047 }
1048 EXPORT_SYMBOL(ib_destroy_cq);
1049
1050 int ib_resize_cq(struct ib_cq *cq, int cqe)
1051 {
1052         return cq->device->resize_cq ?
1053                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
1054 }
1055 EXPORT_SYMBOL(ib_resize_cq);
1056
1057 /* Memory regions */
1058
1059 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
1060 {
1061         struct ib_mr *mr;
1062         int err;
1063
1064         err = ib_check_mr_access(mr_access_flags);
1065         if (err)
1066                 return ERR_PTR(err);
1067
1068         mr = pd->device->get_dma_mr(pd, mr_access_flags);
1069
1070         if (!IS_ERR(mr)) {
1071                 mr->device  = pd->device;
1072                 mr->pd      = pd;
1073                 mr->uobject = NULL;
1074                 atomic_inc(&pd->usecnt);
1075                 atomic_set(&mr->usecnt, 0);
1076         }
1077
1078         return mr;
1079 }
1080 EXPORT_SYMBOL(ib_get_dma_mr);
1081
1082 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
1083                              struct ib_phys_buf *phys_buf_array,
1084                              int num_phys_buf,
1085                              int mr_access_flags,
1086                              u64 *iova_start)
1087 {
1088         struct ib_mr *mr;
1089         int err;
1090
1091         err = ib_check_mr_access(mr_access_flags);
1092         if (err)
1093                 return ERR_PTR(err);
1094
1095         if (!pd->device->reg_phys_mr)
1096                 return ERR_PTR(-ENOSYS);
1097
1098         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
1099                                      mr_access_flags, iova_start);
1100
1101         if (!IS_ERR(mr)) {
1102                 mr->device  = pd->device;
1103                 mr->pd      = pd;
1104                 mr->uobject = NULL;
1105                 atomic_inc(&pd->usecnt);
1106                 atomic_set(&mr->usecnt, 0);
1107         }
1108
1109         return mr;
1110 }
1111 EXPORT_SYMBOL(ib_reg_phys_mr);
1112
1113 int ib_rereg_phys_mr(struct ib_mr *mr,
1114                      int mr_rereg_mask,
1115                      struct ib_pd *pd,
1116                      struct ib_phys_buf *phys_buf_array,
1117                      int num_phys_buf,
1118                      int mr_access_flags,
1119                      u64 *iova_start)
1120 {
1121         struct ib_pd *old_pd;
1122         int ret;
1123
1124         ret = ib_check_mr_access(mr_access_flags);
1125         if (ret)
1126                 return ret;
1127
1128         if (!mr->device->rereg_phys_mr)
1129                 return -ENOSYS;
1130
1131         if (atomic_read(&mr->usecnt))
1132                 return -EBUSY;
1133
1134         old_pd = mr->pd;
1135
1136         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1137                                         phys_buf_array, num_phys_buf,
1138                                         mr_access_flags, iova_start);
1139
1140         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1141                 atomic_dec(&old_pd->usecnt);
1142                 atomic_inc(&pd->usecnt);
1143         }
1144
1145         return ret;
1146 }
1147 EXPORT_SYMBOL(ib_rereg_phys_mr);
1148
1149 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1150 {
1151         return mr->device->query_mr ?
1152                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1153 }
1154 EXPORT_SYMBOL(ib_query_mr);
1155
1156 int ib_dereg_mr(struct ib_mr *mr)
1157 {
1158         struct ib_pd *pd;
1159         int ret;
1160
1161         if (atomic_read(&mr->usecnt))
1162                 return -EBUSY;
1163
1164         pd = mr->pd;
1165         ret = mr->device->dereg_mr(mr);
1166         if (!ret)
1167                 atomic_dec(&pd->usecnt);
1168
1169         return ret;
1170 }
1171 EXPORT_SYMBOL(ib_dereg_mr);
1172
1173 struct ib_mr *ib_create_mr(struct ib_pd *pd,
1174                            struct ib_mr_init_attr *mr_init_attr)
1175 {
1176         struct ib_mr *mr;
1177
1178         if (!pd->device->create_mr)
1179                 return ERR_PTR(-ENOSYS);
1180
1181         mr = pd->device->create_mr(pd, mr_init_attr);
1182
1183         if (!IS_ERR(mr)) {
1184                 mr->device  = pd->device;
1185                 mr->pd      = pd;
1186                 mr->uobject = NULL;
1187                 atomic_inc(&pd->usecnt);
1188                 atomic_set(&mr->usecnt, 0);
1189         }
1190
1191         return mr;
1192 }
1193 EXPORT_SYMBOL(ib_create_mr);
1194
1195 int ib_destroy_mr(struct ib_mr *mr)
1196 {
1197         struct ib_pd *pd;
1198         int ret;
1199
1200         if (atomic_read(&mr->usecnt))
1201                 return -EBUSY;
1202
1203         pd = mr->pd;
1204         ret = mr->device->destroy_mr(mr);
1205         if (!ret)
1206                 atomic_dec(&pd->usecnt);
1207
1208         return ret;
1209 }
1210 EXPORT_SYMBOL(ib_destroy_mr);
1211
1212 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1213 {
1214         struct ib_mr *mr;
1215
1216         if (!pd->device->alloc_fast_reg_mr)
1217                 return ERR_PTR(-ENOSYS);
1218
1219         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1220
1221         if (!IS_ERR(mr)) {
1222                 mr->device  = pd->device;
1223                 mr->pd      = pd;
1224                 mr->uobject = NULL;
1225                 atomic_inc(&pd->usecnt);
1226                 atomic_set(&mr->usecnt, 0);
1227         }
1228
1229         return mr;
1230 }
1231 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1232
1233 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1234                                                           int max_page_list_len)
1235 {
1236         struct ib_fast_reg_page_list *page_list;
1237
1238         if (!device->alloc_fast_reg_page_list)
1239                 return ERR_PTR(-ENOSYS);
1240
1241         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1242
1243         if (!IS_ERR(page_list)) {
1244                 page_list->device = device;
1245                 page_list->max_page_list_len = max_page_list_len;
1246         }
1247
1248         return page_list;
1249 }
1250 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1251
1252 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1253 {
1254         page_list->device->free_fast_reg_page_list(page_list);
1255 }
1256 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1257
1258 /* Memory windows */
1259
1260 struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
1261 {
1262         struct ib_mw *mw;
1263
1264         if (!pd->device->alloc_mw)
1265                 return ERR_PTR(-ENOSYS);
1266
1267         mw = pd->device->alloc_mw(pd, type);
1268         if (!IS_ERR(mw)) {
1269                 mw->device  = pd->device;
1270                 mw->pd      = pd;
1271                 mw->uobject = NULL;
1272                 mw->type    = type;
1273                 atomic_inc(&pd->usecnt);
1274         }
1275
1276         return mw;
1277 }
1278 EXPORT_SYMBOL(ib_alloc_mw);
1279
1280 int ib_dealloc_mw(struct ib_mw *mw)
1281 {
1282         struct ib_pd *pd;
1283         int ret;
1284
1285         pd = mw->pd;
1286         ret = mw->device->dealloc_mw(mw);
1287         if (!ret)
1288                 atomic_dec(&pd->usecnt);
1289
1290         return ret;
1291 }
1292 EXPORT_SYMBOL(ib_dealloc_mw);
1293
1294 /* "Fast" memory regions */
1295
1296 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1297                             int mr_access_flags,
1298                             struct ib_fmr_attr *fmr_attr)
1299 {
1300         struct ib_fmr *fmr;
1301
1302         if (!pd->device->alloc_fmr)
1303                 return ERR_PTR(-ENOSYS);
1304
1305         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1306         if (!IS_ERR(fmr)) {
1307                 fmr->device = pd->device;
1308                 fmr->pd     = pd;
1309                 atomic_inc(&pd->usecnt);
1310         }
1311
1312         return fmr;
1313 }
1314 EXPORT_SYMBOL(ib_alloc_fmr);
1315
1316 int ib_unmap_fmr(struct list_head *fmr_list)
1317 {
1318         struct ib_fmr *fmr;
1319
1320         if (list_empty(fmr_list))
1321                 return 0;
1322
1323         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1324         return fmr->device->unmap_fmr(fmr_list);
1325 }
1326 EXPORT_SYMBOL(ib_unmap_fmr);
1327
1328 int ib_dealloc_fmr(struct ib_fmr *fmr)
1329 {
1330         struct ib_pd *pd;
1331         int ret;
1332
1333         pd = fmr->pd;
1334         ret = fmr->device->dealloc_fmr(fmr);
1335         if (!ret)
1336                 atomic_dec(&pd->usecnt);
1337
1338         return ret;
1339 }
1340 EXPORT_SYMBOL(ib_dealloc_fmr);
1341
1342 /* Multicast groups */
1343
1344 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1345 {
1346         int ret;
1347
1348         if (!qp->device->attach_mcast)
1349                 return -ENOSYS;
1350         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1351                 return -EINVAL;
1352
1353         ret = qp->device->attach_mcast(qp, gid, lid);
1354         if (!ret)
1355                 atomic_inc(&qp->usecnt);
1356         return ret;
1357 }
1358 EXPORT_SYMBOL(ib_attach_mcast);
1359
1360 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1361 {
1362         int ret;
1363
1364         if (!qp->device->detach_mcast)
1365                 return -ENOSYS;
1366         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1367                 return -EINVAL;
1368
1369         ret = qp->device->detach_mcast(qp, gid, lid);
1370         if (!ret)
1371                 atomic_dec(&qp->usecnt);
1372         return ret;
1373 }
1374 EXPORT_SYMBOL(ib_detach_mcast);
1375
1376 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1377 {
1378         struct ib_xrcd *xrcd;
1379
1380         if (!device->alloc_xrcd)
1381                 return ERR_PTR(-ENOSYS);
1382
1383         xrcd = device->alloc_xrcd(device, NULL, NULL);
1384         if (!IS_ERR(xrcd)) {
1385                 xrcd->device = device;
1386                 xrcd->inode = NULL;
1387                 atomic_set(&xrcd->usecnt, 0);
1388                 mutex_init(&xrcd->tgt_qp_mutex);
1389                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1390         }
1391
1392         return xrcd;
1393 }
1394 EXPORT_SYMBOL(ib_alloc_xrcd);
1395
1396 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1397 {
1398         struct ib_qp *qp;
1399         int ret;
1400
1401         if (atomic_read(&xrcd->usecnt))
1402                 return -EBUSY;
1403
1404         while (!list_empty(&xrcd->tgt_qp_list)) {
1405                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1406                 ret = ib_destroy_qp(qp);
1407                 if (ret)
1408                         return ret;
1409         }
1410
1411         return xrcd->device->dealloc_xrcd(xrcd);
1412 }
1413 EXPORT_SYMBOL(ib_dealloc_xrcd);
1414
1415 struct ib_flow *ib_create_flow(struct ib_qp *qp,
1416                                struct ib_flow_attr *flow_attr,
1417                                int domain)
1418 {
1419         struct ib_flow *flow_id;
1420         if (!qp->device->create_flow)
1421                 return ERR_PTR(-ENOSYS);
1422
1423         flow_id = qp->device->create_flow(qp, flow_attr, domain);
1424         if (!IS_ERR(flow_id))
1425                 atomic_inc(&qp->usecnt);
1426         return flow_id;
1427 }
1428 EXPORT_SYMBOL(ib_create_flow);
1429
1430 int ib_destroy_flow(struct ib_flow *flow_id)
1431 {
1432         int err;
1433         struct ib_qp *qp = flow_id->qp;
1434
1435         err = qp->device->destroy_flow(flow_id);
1436         if (!err)
1437                 atomic_dec(&qp->usecnt);
1438         return err;
1439 }
1440 EXPORT_SYMBOL(ib_destroy_flow);
1441
1442 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
1443                        struct ib_mr_status *mr_status)
1444 {
1445         return mr->device->check_mr_status ?
1446                 mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
1447 }
1448 EXPORT_SYMBOL(ib_check_mr_status);