kernel/drivers/staging/rdma/hfi1/srq.c

   1 /*
   2  *
   3  * This file is provided under a dual BSD/GPLv2 license.  When using or
   4  * redistributing this file, you may do so under either license.
   5  *
   6  * GPL LICENSE SUMMARY
   7  *
   8  * Copyright(c) 2015 Intel Corporation.
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of version 2 of the GNU General Public License as
  12  * published by the Free Software Foundation.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * General Public License for more details.
  18  *
  19  * BSD LICENSE
  20  *
  21  * Copyright(c) 2015 Intel Corporation.
  22  *
  23  * Redistribution and use in source and binary forms, with or without
  24  * modification, are permitted provided that the following conditions
  25  * are met:
  26  *
  27  *  - Redistributions of source code must retain the above copyright
  28  *    notice, this list of conditions and the following disclaimer.
  29  *  - Redistributions in binary form must reproduce the above copyright
  30  *    notice, this list of conditions and the following disclaimer in
  31  *    the documentation and/or other materials provided with the
  32  *    distribution.
  33  *  - Neither the name of Intel Corporation nor the names of its
  34  *    contributors may be used to endorse or promote products derived
  35  *    from this software without specific prior written permission.
  36  *
  37  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  38  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  39  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  40  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  41  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  42  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  43  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  44  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  45  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  46  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  47  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  48  *
  49  */
  50
  51 #include <linux/err.h>
  52 #include <linux/slab.h>
  53 #include <linux/vmalloc.h>
  54
  55 #include "verbs.h"
  56
  57 /**
  58  * hfi1_post_srq_receive - post a receive on a shared receive queue
  59  * @ibsrq: the SRQ to post the receive on
  60  * @wr: the list of work requests to post
  61  * @bad_wr: A pointer to the first WR to cause a problem is put here
  62  *
  63  * This may be called from interrupt context.
  64  */
  65 int hfi1_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
  66                           struct ib_recv_wr **bad_wr)
  67 {
  68         struct hfi1_srq *srq = to_isrq(ibsrq);
  69         struct hfi1_rwq *wq;
  70         unsigned long flags;
  71         int ret;
  72
  73         for (; wr; wr = wr->next) {
  74                 struct hfi1_rwqe *wqe;
  75                 u32 next;
  76                 int i;
  77
  78                 if ((unsigned) wr->num_sge > srq->rq.max_sge) {
  79                         *bad_wr = wr;
  80                         ret = -EINVAL;
  81                         goto bail;
  82                 }
  83
  84                 spin_lock_irqsave(&srq->rq.lock, flags);
  85                 wq = srq->rq.wq;
  86                 next = wq->head + 1;
  87                 if (next >= srq->rq.size)
  88                         next = 0;
  89                 if (next == wq->tail) {
  90                         spin_unlock_irqrestore(&srq->rq.lock, flags);
  91                         *bad_wr = wr;
  92                         ret = -ENOMEM;
  93                         goto bail;
  94                 }
  95
  96                 wqe = get_rwqe_ptr(&srq->rq, wq->head);
  97                 wqe->wr_id = wr->wr_id;
  98                 wqe->num_sge = wr->num_sge;
  99                 for (i = 0; i < wr->num_sge; i++)
 100                         wqe->sg_list[i] = wr->sg_list[i];
 101                 /* Make sure queue entry is written before the head index. */
 102                 smp_wmb();
 103                 wq->head = next;
 104                 spin_unlock_irqrestore(&srq->rq.lock, flags);
 105         }
 106         ret = 0;
 107
 108 bail:
 109         return ret;
 110 }
 111
 112 /**
 113  * hfi1_create_srq - create a shared receive queue
 114  * @ibpd: the protection domain of the SRQ to create
 115  * @srq_init_attr: the attributes of the SRQ
 116  * @udata: data from libibverbs when creating a user SRQ
 117  */
 118 struct ib_srq *hfi1_create_srq(struct ib_pd *ibpd,
 119                                struct ib_srq_init_attr *srq_init_attr,
 120                                struct ib_udata *udata)
 121 {
 122         struct hfi1_ibdev *dev = to_idev(ibpd->device);
 123         struct hfi1_srq *srq;
 124         u32 sz;
 125         struct ib_srq *ret;
 126
 127         if (srq_init_attr->srq_type != IB_SRQT_BASIC) {
 128                 ret = ERR_PTR(-ENOSYS);
 129                 goto done;
 130         }
 131
 132         if (srq_init_attr->attr.max_sge == 0 ||
 133             srq_init_attr->attr.max_sge > hfi1_max_srq_sges ||
 134             srq_init_attr->attr.max_wr == 0 ||
 135             srq_init_attr->attr.max_wr > hfi1_max_srq_wrs) {
 136                 ret = ERR_PTR(-EINVAL);
 137                 goto done;
 138         }
 139
 140         srq = kmalloc(sizeof(*srq), GFP_KERNEL);
 141         if (!srq) {
 142                 ret = ERR_PTR(-ENOMEM);
 143                 goto done;
 144         }
 145
 146         /*
 147          * Need to use vmalloc() if we want to support large #s of entries.
 148          */
 149         srq->rq.size = srq_init_attr->attr.max_wr + 1;
 150         srq->rq.max_sge = srq_init_attr->attr.max_sge;
 151         sz = sizeof(struct ib_sge) * srq->rq.max_sge +
 152                 sizeof(struct hfi1_rwqe);
 153         srq->rq.wq = vmalloc_user(sizeof(struct hfi1_rwq) + srq->rq.size * sz);
 154         if (!srq->rq.wq) {
 155                 ret = ERR_PTR(-ENOMEM);
 156                 goto bail_srq;
 157         }
 158
 159         /*
 160          * Return the address of the RWQ as the offset to mmap.
 161          * See hfi1_mmap() for details.
 162          */
 163         if (udata && udata->outlen >= sizeof(__u64)) {
 164                 int err;
 165                 u32 s = sizeof(struct hfi1_rwq) + srq->rq.size * sz;
 166
 167                 srq->ip =
 168                     hfi1_create_mmap_info(dev, s, ibpd->uobject->context,
 169                                           srq->rq.wq);
 170                 if (!srq->ip) {
 171                         ret = ERR_PTR(-ENOMEM);
 172                         goto bail_wq;
 173                 }
 174
 175                 err = ib_copy_to_udata(udata, &srq->ip->offset,
 176                                        sizeof(srq->ip->offset));
 177                 if (err) {
 178                         ret = ERR_PTR(err);
 179                         goto bail_ip;
 180                 }
 181         } else
 182                 srq->ip = NULL;
 183
 184         /*
 185          * ib_create_srq() will initialize srq->ibsrq.
 186          */
 187         spin_lock_init(&srq->rq.lock);
 188         srq->rq.wq->head = 0;
 189         srq->rq.wq->tail = 0;
 190         srq->limit = srq_init_attr->attr.srq_limit;
 191
 192         spin_lock(&dev->n_srqs_lock);
 193         if (dev->n_srqs_allocated == hfi1_max_srqs) {
 194                 spin_unlock(&dev->n_srqs_lock);
 195                 ret = ERR_PTR(-ENOMEM);
 196                 goto bail_ip;
 197         }
 198
 199         dev->n_srqs_allocated++;
 200         spin_unlock(&dev->n_srqs_lock);
 201
 202         if (srq->ip) {
 203                 spin_lock_irq(&dev->pending_lock);
 204                 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps);
 205                 spin_unlock_irq(&dev->pending_lock);
 206         }
 207
 208         ret = &srq->ibsrq;
 209         goto done;
 210
 211 bail_ip:
 212         kfree(srq->ip);
 213 bail_wq:
 214         vfree(srq->rq.wq);
 215 bail_srq:
 216         kfree(srq);
 217 done:
 218         return ret;
 219 }
 220
 221 /**
 222  * hfi1_modify_srq - modify a shared receive queue
 223  * @ibsrq: the SRQ to modify
 224  * @attr: the new attributes of the SRQ
 225  * @attr_mask: indicates which attributes to modify
 226  * @udata: user data for libibverbs.so
 227  */
 228 int hfi1_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 229                     enum ib_srq_attr_mask attr_mask,
 230                     struct ib_udata *udata)
 231 {
 232         struct hfi1_srq *srq = to_isrq(ibsrq);
 233         struct hfi1_rwq *wq;
 234         int ret = 0;
 235
 236         if (attr_mask & IB_SRQ_MAX_WR) {
 237                 struct hfi1_rwq *owq;
 238                 struct hfi1_rwqe *p;
 239                 u32 sz, size, n, head, tail;
 240
 241                 /* Check that the requested sizes are below the limits. */
 242                 if ((attr->max_wr > hfi1_max_srq_wrs) ||
 243                     ((attr_mask & IB_SRQ_LIMIT) ?
 244                      attr->srq_limit : srq->limit) > attr->max_wr) {
 245                         ret = -EINVAL;
 246                         goto bail;
 247                 }
 248
 249                 sz = sizeof(struct hfi1_rwqe) +
 250                         srq->rq.max_sge * sizeof(struct ib_sge);
 251                 size = attr->max_wr + 1;
 252                 wq = vmalloc_user(sizeof(struct hfi1_rwq) + size * sz);
 253                 if (!wq) {
 254                         ret = -ENOMEM;
 255                         goto bail;
 256                 }
 257
 258                 /* Check that we can write the offset to mmap. */
 259                 if (udata && udata->inlen >= sizeof(__u64)) {
 260                         __u64 offset_addr;
 261                         __u64 offset = 0;
 262
 263                         ret = ib_copy_from_udata(&offset_addr, udata,
 264                                                  sizeof(offset_addr));
 265                         if (ret)
 266                                 goto bail_free;
 267                         udata->outbuf =
 268                                 (void __user *) (unsigned long) offset_addr;
 269                         ret = ib_copy_to_udata(udata, &offset,
 270                                                sizeof(offset));
 271                         if (ret)
 272                                 goto bail_free;
 273                 }
 274
 275                 spin_lock_irq(&srq->rq.lock);
 276                 /*
 277                  * validate head and tail pointer values and compute
 278                  * the number of remaining WQEs.
 279                  */
 280                 owq = srq->rq.wq;
 281                 head = owq->head;
 282                 tail = owq->tail;
 283                 if (head >= srq->rq.size || tail >= srq->rq.size) {
 284                         ret = -EINVAL;
 285                         goto bail_unlock;
 286                 }
 287                 n = head;
 288                 if (n < tail)
 289                         n += srq->rq.size - tail;
 290                 else
 291                         n -= tail;
 292                 if (size <= n) {
 293                         ret = -EINVAL;
 294                         goto bail_unlock;
 295                 }
 296                 n = 0;
 297                 p = wq->wq;
 298                 while (tail != head) {
 299                         struct hfi1_rwqe *wqe;
 300                         int i;
 301
 302                         wqe = get_rwqe_ptr(&srq->rq, tail);
 303                         p->wr_id = wqe->wr_id;
 304                         p->num_sge = wqe->num_sge;
 305                         for (i = 0; i < wqe->num_sge; i++)
 306                                 p->sg_list[i] = wqe->sg_list[i];
 307                         n++;
 308                         p = (struct hfi1_rwqe *)((char *)p + sz);
 309                         if (++tail >= srq->rq.size)
 310                                 tail = 0;
 311                 }
 312                 srq->rq.wq = wq;
 313                 srq->rq.size = size;
 314                 wq->head = n;
 315                 wq->tail = 0;
 316                 if (attr_mask & IB_SRQ_LIMIT)
 317                         srq->limit = attr->srq_limit;
 318                 spin_unlock_irq(&srq->rq.lock);
 319
 320                 vfree(owq);
 321
 322                 if (srq->ip) {
 323                         struct hfi1_mmap_info *ip = srq->ip;
 324                         struct hfi1_ibdev *dev = to_idev(srq->ibsrq.device);
 325                         u32 s = sizeof(struct hfi1_rwq) + size * sz;
 326
 327                         hfi1_update_mmap_info(dev, ip, s, wq);
 328
 329                         /*
 330                          * Return the offset to mmap.
 331                          * See hfi1_mmap() for details.
 332                          */
 333                         if (udata && udata->inlen >= sizeof(__u64)) {
 334                                 ret = ib_copy_to_udata(udata, &ip->offset,
 335                                                        sizeof(ip->offset));
 336                                 if (ret)
 337                                         goto bail;
 338                         }
 339
 340                         /*
 341                          * Put user mapping info onto the pending list
 342                          * unless it already is on the list.
 343                          */
 344                         spin_lock_irq(&dev->pending_lock);
 345                         if (list_empty(&ip->pending_mmaps))
 346                                 list_add(&ip->pending_mmaps,
 347                                          &dev->pending_mmaps);
 348                         spin_unlock_irq(&dev->pending_lock);
 349                 }
 350         } else if (attr_mask & IB_SRQ_LIMIT) {
 351                 spin_lock_irq(&srq->rq.lock);
 352                 if (attr->srq_limit >= srq->rq.size)
 353                         ret = -EINVAL;
 354                 else
 355                         srq->limit = attr->srq_limit;
 356                 spin_unlock_irq(&srq->rq.lock);
 357         }
 358         goto bail;
 359
 360 bail_unlock:
 361         spin_unlock_irq(&srq->rq.lock);
 362 bail_free:
 363         vfree(wq);
 364 bail:
 365         return ret;
 366 }
 367
 368 int hfi1_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 369 {
 370         struct hfi1_srq *srq = to_isrq(ibsrq);
 371
 372         attr->max_wr = srq->rq.size - 1;
 373         attr->max_sge = srq->rq.max_sge;
 374         attr->srq_limit = srq->limit;
 375         return 0;
 376 }
 377
 378 /**
 379  * hfi1_destroy_srq - destroy a shared receive queue
 380  * @ibsrq: the SRQ to destroy
 381  */
 382 int hfi1_destroy_srq(struct ib_srq *ibsrq)
 383 {
 384         struct hfi1_srq *srq = to_isrq(ibsrq);
 385         struct hfi1_ibdev *dev = to_idev(ibsrq->device);
 386
 387         spin_lock(&dev->n_srqs_lock);
 388         dev->n_srqs_allocated--;
 389         spin_unlock(&dev->n_srqs_lock);
 390         if (srq->ip)
 391                 kref_put(&srq->ip->ref, hfi1_release_mmap_info);
 392         else
 393                 vfree(srq->rq.wq);
 394         kfree(srq);
 395
 396         return 0;
 397 }