kernel/drivers/s390/block/scm_blk.c

   1 /*
   2  * Block driver for s390 storage class memory.
   3  *
   4  * Copyright IBM Corp. 2012
   5  * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
   6  */
   7
   8 #define KMSG_COMPONENT "scm_block"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/interrupt.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/mempool.h>
  14 #include <linux/module.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/genhd.h>
  17 #include <linux/slab.h>
  18 #include <linux/list.h>
  19 #include <asm/eadm.h>
  20 #include "scm_blk.h"
  21
  22 debug_info_t *scm_debug;
  23 static int scm_major;
  24 static mempool_t *aidaw_pool;
  25 static DEFINE_SPINLOCK(list_lock);
  26 static LIST_HEAD(inactive_requests);
  27 static unsigned int nr_requests = 64;
  28 static unsigned int nr_requests_per_io = 8;
  29 static atomic_t nr_devices = ATOMIC_INIT(0);
  30 module_param(nr_requests, uint, S_IRUGO);
  31 MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
  32
  33 module_param(nr_requests_per_io, uint, S_IRUGO);
  34 MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO.");
  35
  36 MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
  37 MODULE_LICENSE("GPL");
  38 MODULE_ALIAS("scm:scmdev*");
  39
  40 static void __scm_free_rq(struct scm_request *scmrq)
  41 {
  42         struct aob_rq_header *aobrq = to_aobrq(scmrq);
  43
  44         free_page((unsigned long) scmrq->aob);
  45         __scm_free_rq_cluster(scmrq);
  46         kfree(scmrq->request);
  47         kfree(aobrq);
  48 }
  49
  50 static void scm_free_rqs(void)
  51 {
  52         struct list_head *iter, *safe;
  53         struct scm_request *scmrq;
  54
  55         spin_lock_irq(&list_lock);
  56         list_for_each_safe(iter, safe, &inactive_requests) {
  57                 scmrq = list_entry(iter, struct scm_request, list);
  58                 list_del(&scmrq->list);
  59                 __scm_free_rq(scmrq);
  60         }
  61         spin_unlock_irq(&list_lock);
  62
  63         mempool_destroy(aidaw_pool);
  64 }
  65
  66 static int __scm_alloc_rq(void)
  67 {
  68         struct aob_rq_header *aobrq;
  69         struct scm_request *scmrq;
  70
  71         aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
  72         if (!aobrq)
  73                 return -ENOMEM;
  74
  75         scmrq = (void *) aobrq->data;
  76         scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
  77         if (!scmrq->aob)
  78                 goto free;
  79
  80         scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),
  81                                  GFP_KERNEL);
  82         if (!scmrq->request)
  83                 goto free;
  84
  85         if (__scm_alloc_rq_cluster(scmrq))
  86                 goto free;
  87
  88         INIT_LIST_HEAD(&scmrq->list);
  89         spin_lock_irq(&list_lock);
  90         list_add(&scmrq->list, &inactive_requests);
  91         spin_unlock_irq(&list_lock);
  92
  93         return 0;
  94 free:
  95         __scm_free_rq(scmrq);
  96         return -ENOMEM;
  97 }
  98
  99 static int scm_alloc_rqs(unsigned int nrqs)
 100 {
 101         int ret = 0;
 102
 103         aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
 104         if (!aidaw_pool)
 105                 return -ENOMEM;
 106
 107         while (nrqs-- && !ret)
 108                 ret = __scm_alloc_rq();
 109
 110         return ret;
 111 }
 112
 113 static struct scm_request *scm_request_fetch(void)
 114 {
 115         struct scm_request *scmrq = NULL;
 116
 117         spin_lock(&list_lock);
 118         if (list_empty(&inactive_requests))
 119                 goto out;
 120         scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
 121         list_del(&scmrq->list);
 122 out:
 123         spin_unlock(&list_lock);
 124         return scmrq;
 125 }
 126
 127 static void scm_request_done(struct scm_request *scmrq)
 128 {
 129         unsigned long flags;
 130         struct msb *msb;
 131         u64 aidaw;
 132         int i;
 133
 134         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 135                 msb = &scmrq->aob->msb[i];
 136                 aidaw = msb->data_addr;
 137
 138                 if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
 139                     IS_ALIGNED(aidaw, PAGE_SIZE))
 140                         mempool_free(virt_to_page(aidaw), aidaw_pool);
 141         }
 142
 143         spin_lock_irqsave(&list_lock, flags);
 144         list_add(&scmrq->list, &inactive_requests);
 145         spin_unlock_irqrestore(&list_lock, flags);
 146 }
 147
 148 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
 149 {
 150         return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
 151 }
 152
 153 static inline struct aidaw *scm_aidaw_alloc(void)
 154 {
 155         struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);
 156
 157         return page ? page_address(page) : NULL;
 158 }
 159
 160 static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
 161 {
 162         unsigned long _aidaw = (unsigned long) aidaw;
 163         unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;
 164
 165         return (bytes / sizeof(*aidaw)) * PAGE_SIZE;
 166 }
 167
 168 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)
 169 {
 170         struct aidaw *aidaw;
 171
 172         if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
 173                 return scmrq->next_aidaw;
 174
 175         aidaw = scm_aidaw_alloc();
 176         if (aidaw)
 177                 memset(aidaw, 0, PAGE_SIZE);
 178         return aidaw;
 179 }
 180
 181 static int scm_request_prepare(struct scm_request *scmrq)
 182 {
 183         struct scm_blk_dev *bdev = scmrq->bdev;
 184         struct scm_device *scmdev = bdev->gendisk->private_data;
 185         int pos = scmrq->aob->request.msb_count;
 186         struct msb *msb = &scmrq->aob->msb[pos];
 187         struct request *req = scmrq->request[pos];
 188         struct req_iterator iter;
 189         struct aidaw *aidaw;
 190         struct bio_vec bv;
 191
 192         aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
 193         if (!aidaw)
 194                 return -ENOMEM;
 195
 196         msb->bs = MSB_BS_4K;
 197         scmrq->aob->request.msb_count++;
 198         msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
 199         msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
 200         msb->flags |= MSB_FLAG_IDA;
 201         msb->data_addr = (u64) aidaw;
 202
 203         rq_for_each_segment(bv, req, iter) {
 204                 WARN_ON(bv.bv_offset);
 205                 msb->blk_count += bv.bv_len >> 12;
 206                 aidaw->data_addr = (u64) page_address(bv.bv_page);
 207                 aidaw++;
 208         }
 209
 210         scmrq->next_aidaw = aidaw;
 211         return 0;
 212 }
 213
 214 static inline void scm_request_set(struct scm_request *scmrq,
 215                                    struct request *req)
 216 {
 217         scmrq->request[scmrq->aob->request.msb_count] = req;
 218 }
 219
 220 static inline void scm_request_init(struct scm_blk_dev *bdev,
 221                                     struct scm_request *scmrq)
 222 {
 223         struct aob_rq_header *aobrq = to_aobrq(scmrq);
 224         struct aob *aob = scmrq->aob;
 225
 226         memset(scmrq->request, 0,
 227                nr_requests_per_io * sizeof(scmrq->request[0]));
 228         memset(aob, 0, sizeof(*aob));
 229         aobrq->scmdev = bdev->scmdev;
 230         aob->request.cmd_code = ARQB_CMD_MOVE;
 231         aob->request.data = (u64) aobrq;
 232         scmrq->bdev = bdev;
 233         scmrq->retries = 4;
 234         scmrq->error = 0;
 235         /* We don't use all msbs - place aidaws at the end of the aob page. */
 236         scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];
 237         scm_request_cluster_init(scmrq);
 238 }
 239
 240 static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
 241 {
 242         if (atomic_read(&bdev->queued_reqs)) {
 243                 /* Queue restart is triggered by the next interrupt. */
 244                 return;
 245         }
 246         blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
 247 }
 248
 249 void scm_request_requeue(struct scm_request *scmrq)
 250 {
 251         struct scm_blk_dev *bdev = scmrq->bdev;
 252         int i;
 253
 254         scm_release_cluster(scmrq);
 255         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
 256                 blk_requeue_request(bdev->rq, scmrq->request[i]);
 257
 258         atomic_dec(&bdev->queued_reqs);
 259         scm_request_done(scmrq);
 260         scm_ensure_queue_restart(bdev);
 261 }
 262
 263 void scm_request_finish(struct scm_request *scmrq)
 264 {
 265         struct scm_blk_dev *bdev = scmrq->bdev;
 266         int i;
 267
 268         scm_release_cluster(scmrq);
 269         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
 270                 blk_end_request_all(scmrq->request[i], scmrq->error);
 271
 272         atomic_dec(&bdev->queued_reqs);
 273         scm_request_done(scmrq);
 274 }
 275
 276 static int scm_request_start(struct scm_request *scmrq)
 277 {
 278         struct scm_blk_dev *bdev = scmrq->bdev;
 279         int ret;
 280
 281         atomic_inc(&bdev->queued_reqs);
 282         if (!scmrq->aob->request.msb_count) {
 283                 scm_request_requeue(scmrq);
 284                 return -EINVAL;
 285         }
 286
 287         ret = eadm_start_aob(scmrq->aob);
 288         if (ret) {
 289                 SCM_LOG(5, "no subchannel");
 290                 scm_request_requeue(scmrq);
 291         }
 292         return ret;
 293 }
 294
 295 static void scm_blk_request(struct request_queue *rq)
 296 {
 297         struct scm_device *scmdev = rq->queuedata;
 298         struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
 299         struct scm_request *scmrq = NULL;
 300         struct request *req;
 301
 302         while ((req = blk_peek_request(rq))) {
 303                 if (req->cmd_type != REQ_TYPE_FS) {
 304                         blk_start_request(req);
 305                         blk_dump_rq_flags(req, KMSG_COMPONENT " bad request");
 306                         blk_end_request_all(req, -EIO);
 307                         continue;
 308                 }
 309
 310                 if (!scm_permit_request(bdev, req))
 311                         goto out;
 312
 313                 if (!scmrq) {
 314                         scmrq = scm_request_fetch();
 315                         if (!scmrq) {
 316                                 SCM_LOG(5, "no request");
 317                                 goto out;
 318                         }
 319                         scm_request_init(bdev, scmrq);
 320                 }
 321                 scm_request_set(scmrq, req);
 322
 323                 if (!scm_reserve_cluster(scmrq)) {
 324                         SCM_LOG(5, "cluster busy");
 325                         scm_request_set(scmrq, NULL);
 326                         if (scmrq->aob->request.msb_count)
 327                                 goto out;
 328
 329                         scm_request_done(scmrq);
 330                         return;
 331                 }
 332
 333                 if (scm_need_cluster_request(scmrq)) {
 334                         if (scmrq->aob->request.msb_count) {
 335                                 /* Start cluster requests separately. */
 336                                 scm_request_set(scmrq, NULL);
 337                                 if (scm_request_start(scmrq))
 338                                         return;
 339                         } else {
 340                                 atomic_inc(&bdev->queued_reqs);
 341                                 blk_start_request(req);
 342                                 scm_initiate_cluster_request(scmrq);
 343                         }
 344                         scmrq = NULL;
 345                         continue;
 346                 }
 347
 348                 if (scm_request_prepare(scmrq)) {
 349                         SCM_LOG(5, "aidaw alloc failed");
 350                         scm_request_set(scmrq, NULL);
 351                         goto out;
 352                 }
 353                 blk_start_request(req);
 354
 355                 if (scmrq->aob->request.msb_count < nr_requests_per_io)
 356                         continue;
 357
 358                 if (scm_request_start(scmrq))
 359                         return;
 360
 361                 scmrq = NULL;
 362         }
 363 out:
 364         if (scmrq)
 365                 scm_request_start(scmrq);
 366         else
 367                 scm_ensure_queue_restart(bdev);
 368 }
 369
 370 static void __scmrq_log_error(struct scm_request *scmrq)
 371 {
 372         struct aob *aob = scmrq->aob;
 373
 374         if (scmrq->error == -ETIMEDOUT)
 375                 SCM_LOG(1, "Request timeout");
 376         else {
 377                 SCM_LOG(1, "Request error");
 378                 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
 379         }
 380         if (scmrq->retries)
 381                 SCM_LOG(1, "Retry request");
 382         else
 383                 pr_err("An I/O operation to SCM failed with rc=%d\n",
 384                        scmrq->error);
 385 }
 386
 387 void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
 388 {
 389         struct scm_request *scmrq = data;
 390         struct scm_blk_dev *bdev = scmrq->bdev;
 391
 392         scmrq->error = error;
 393         if (error)
 394                 __scmrq_log_error(scmrq);
 395
 396         spin_lock(&bdev->lock);
 397         list_add_tail(&scmrq->list, &bdev->finished_requests);
 398         spin_unlock(&bdev->lock);
 399         tasklet_hi_schedule(&bdev->tasklet);
 400 }
 401
 402 static void scm_blk_handle_error(struct scm_request *scmrq)
 403 {
 404         struct scm_blk_dev *bdev = scmrq->bdev;
 405         unsigned long flags;
 406
 407         if (scmrq->error != -EIO)
 408                 goto restart;
 409
 410         /* For -EIO the response block is valid. */
 411         switch (scmrq->aob->response.eqc) {
 412         case EQC_WR_PROHIBIT:
 413                 spin_lock_irqsave(&bdev->lock, flags);
 414                 if (bdev->state != SCM_WR_PROHIBIT)
 415                         pr_info("%lx: Write access to the SCM increment is suspended\n",
 416                                 (unsigned long) bdev->scmdev->address);
 417                 bdev->state = SCM_WR_PROHIBIT;
 418                 spin_unlock_irqrestore(&bdev->lock, flags);
 419                 goto requeue;
 420         default:
 421                 break;
 422         }
 423
 424 restart:
 425         if (!eadm_start_aob(scmrq->aob))
 426                 return;
 427
 428 requeue:
 429         spin_lock_irqsave(&bdev->rq_lock, flags);
 430         scm_request_requeue(scmrq);
 431         spin_unlock_irqrestore(&bdev->rq_lock, flags);
 432 }
 433
 434 static void scm_blk_tasklet(struct scm_blk_dev *bdev)
 435 {
 436         struct scm_request *scmrq;
 437         unsigned long flags;
 438
 439         spin_lock_irqsave(&bdev->lock, flags);
 440         while (!list_empty(&bdev->finished_requests)) {
 441                 scmrq = list_first_entry(&bdev->finished_requests,
 442                                          struct scm_request, list);
 443                 list_del(&scmrq->list);
 444                 spin_unlock_irqrestore(&bdev->lock, flags);
 445
 446                 if (scmrq->error && scmrq->retries-- > 0) {
 447                         scm_blk_handle_error(scmrq);
 448
 449                         /* Request restarted or requeued, handle next. */
 450                         spin_lock_irqsave(&bdev->lock, flags);
 451                         continue;
 452                 }
 453
 454                 if (scm_test_cluster_request(scmrq)) {
 455                         scm_cluster_request_irq(scmrq);
 456                         spin_lock_irqsave(&bdev->lock, flags);
 457                         continue;
 458                 }
 459
 460                 scm_request_finish(scmrq);
 461                 spin_lock_irqsave(&bdev->lock, flags);
 462         }
 463         spin_unlock_irqrestore(&bdev->lock, flags);
 464         /* Look out for more requests. */
 465         blk_run_queue(bdev->rq);
 466 }
 467
 468 static const struct block_device_operations scm_blk_devops = {
 469         .owner = THIS_MODULE,
 470 };
 471
 472 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
 473 {
 474         struct request_queue *rq;
 475         int len, ret = -ENOMEM;
 476         unsigned int devindex, nr_max_blk;
 477
 478         devindex = atomic_inc_return(&nr_devices) - 1;
 479         /* scma..scmz + scmaa..scmzz */
 480         if (devindex > 701) {
 481                 ret = -ENODEV;
 482                 goto out;
 483         }
 484
 485         bdev->scmdev = scmdev;
 486         bdev->state = SCM_OPER;
 487         spin_lock_init(&bdev->rq_lock);
 488         spin_lock_init(&bdev->lock);
 489         INIT_LIST_HEAD(&bdev->finished_requests);
 490         atomic_set(&bdev->queued_reqs, 0);
 491         tasklet_init(&bdev->tasklet,
 492                      (void (*)(unsigned long)) scm_blk_tasklet,
 493                      (unsigned long) bdev);
 494
 495         rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
 496         if (!rq)
 497                 goto out;
 498
 499         bdev->rq = rq;
 500         nr_max_blk = min(scmdev->nr_max_block,
 501                          (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
 502
 503         blk_queue_logical_block_size(rq, 1 << 12);
 504         blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
 505         blk_queue_max_segments(rq, nr_max_blk);
 506         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
 507         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
 508         scm_blk_dev_cluster_setup(bdev);
 509
 510         bdev->gendisk = alloc_disk(SCM_NR_PARTS);
 511         if (!bdev->gendisk)
 512                 goto out_queue;
 513
 514         rq->queuedata = scmdev;
 515         bdev->gendisk->driverfs_dev = &scmdev->dev;
 516         bdev->gendisk->private_data = scmdev;
 517         bdev->gendisk->fops = &scm_blk_devops;
 518         bdev->gendisk->queue = rq;
 519         bdev->gendisk->major = scm_major;
 520         bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
 521
 522         len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
 523         if (devindex > 25) {
 524                 len += snprintf(bdev->gendisk->disk_name + len,
 525                                 DISK_NAME_LEN - len, "%c",
 526                                 'a' + (devindex / 26) - 1);
 527                 devindex = devindex % 26;
 528         }
 529         snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
 530                  'a' + devindex);
 531
 532         /* 512 byte sectors */
 533         set_capacity(bdev->gendisk, scmdev->size >> 9);
 534         add_disk(bdev->gendisk);
 535         return 0;
 536
 537 out_queue:
 538         blk_cleanup_queue(rq);
 539 out:
 540         atomic_dec(&nr_devices);
 541         return ret;
 542 }
 543
 544 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
 545 {
 546         tasklet_kill(&bdev->tasklet);
 547         del_gendisk(bdev->gendisk);
 548         blk_cleanup_queue(bdev->gendisk->queue);
 549         put_disk(bdev->gendisk);
 550 }
 551
 552 void scm_blk_set_available(struct scm_blk_dev *bdev)
 553 {
 554         unsigned long flags;
 555
 556         spin_lock_irqsave(&bdev->lock, flags);
 557         if (bdev->state == SCM_WR_PROHIBIT)
 558                 pr_info("%lx: Write access to the SCM increment is restored\n",
 559                         (unsigned long) bdev->scmdev->address);
 560         bdev->state = SCM_OPER;
 561         spin_unlock_irqrestore(&bdev->lock, flags);
 562 }
 563
 564 static bool __init scm_blk_params_valid(void)
 565 {
 566         if (!nr_requests_per_io || nr_requests_per_io > 64)
 567                 return false;
 568
 569         return scm_cluster_size_valid();
 570 }
 571
 572 static int __init scm_blk_init(void)
 573 {
 574         int ret = -EINVAL;
 575
 576         if (!scm_blk_params_valid())
 577                 goto out;
 578
 579         ret = register_blkdev(0, "scm");
 580         if (ret < 0)
 581                 goto out;
 582
 583         scm_major = ret;
 584         ret = scm_alloc_rqs(nr_requests);
 585         if (ret)
 586                 goto out_free;
 587
 588         scm_debug = debug_register("scm_log", 16, 1, 16);
 589         if (!scm_debug) {
 590                 ret = -ENOMEM;
 591                 goto out_free;
 592         }
 593
 594         debug_register_view(scm_debug, &debug_hex_ascii_view);
 595         debug_set_level(scm_debug, 2);
 596
 597         ret = scm_drv_init();
 598         if (ret)
 599                 goto out_dbf;
 600
 601         return ret;
 602
 603 out_dbf:
 604         debug_unregister(scm_debug);
 605 out_free:
 606         scm_free_rqs();
 607         unregister_blkdev(scm_major, "scm");
 608 out:
 609         return ret;
 610 }
 611 module_init(scm_blk_init);
 612
 613 static void __exit scm_blk_cleanup(void)
 614 {
 615         scm_drv_cleanup();
 616         debug_unregister(scm_debug);
 617         scm_free_rqs();
 618         unregister_blkdev(scm_major, "scm");
 619 }
 620 module_exit(scm_blk_cleanup);