kernel/drivers/soc/ti/knav_qmss_acc.c

   1 /*
   2  * Keystone accumulator queue manager
   3  *
   4  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
   5  * Author:      Sandeep Nair <sandeep_n@ti.com>
   6  *              Cyril Chemparathy <cyril@ti.com>
   7  *              Santosh Shilimkar <santosh.shilimkar@ti.com>
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * version 2 as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful, but
  14  * WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/module.h>
  21 #include <linux/device.h>
  22 #include <linux/io.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/bitops.h>
  25 #include <linux/slab.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/soc/ti/knav_qmss.h>
  28 #include <linux/platform_device.h>
  29 #include <linux/dma-mapping.h>
  30 #include <linux/of.h>
  31 #include <linux/of_device.h>
  32 #include <linux/of_address.h>
  33 #include <linux/firmware.h>
  34
  35 #include "knav_qmss.h"
  36
  37 #define knav_range_offset_to_inst(kdev, range, q)       \
  38         (range->queue_base_inst + (q << kdev->inst_shift))
  39
  40 static void __knav_acc_notify(struct knav_range_info *range,
  41                                 struct knav_acc_channel *acc)
  42 {
  43         struct knav_device *kdev = range->kdev;
  44         struct knav_queue_inst *inst;
  45         int range_base, queue;
  46
  47         range_base = kdev->base_id + range->queue_base;
  48
  49         if (range->flags & RANGE_MULTI_QUEUE) {
  50                 for (queue = 0; queue < range->num_queues; queue++) {
  51                         inst = knav_range_offset_to_inst(kdev, range,
  52                                                                 queue);
  53                         if (inst->notify_needed) {
  54                                 inst->notify_needed = 0;
  55                                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  56                                         range_base + queue);
  57                                 knav_queue_notify(inst);
  58                         }
  59                 }
  60         } else {
  61                 queue = acc->channel - range->acc_info.start_channel;
  62                 inst = knav_range_offset_to_inst(kdev, range, queue);
  63                 dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
  64                         range_base + queue);
  65                 knav_queue_notify(inst);
  66         }
  67 }
  68
  69 static int knav_acc_set_notify(struct knav_range_info *range,
  70                                 struct knav_queue_inst *kq,
  71                                 bool enabled)
  72 {
  73         struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
  74         struct knav_device *kdev = range->kdev;
  75         u32 mask, offset;
  76
  77         /*
  78          * when enabling, we need to re-trigger an interrupt if we
  79          * have descriptors pending
  80          */
  81         if (!enabled || atomic_read(&kq->desc_count) <= 0)
  82                 return 0;
  83
  84         kq->notify_needed = 1;
  85         atomic_inc(&kq->acc->retrigger_count);
  86         mask = BIT(kq->acc->channel % 32);
  87         offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
  88         dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
  89                 kq->acc->name);
  90         writel_relaxed(mask, pdsp->intd + offset);
  91         return 0;
  92 }
  93
  94 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
  95 {
  96         struct knav_acc_channel *acc;
  97         struct knav_queue_inst *kq = NULL;
  98         struct knav_range_info *range;
  99         struct knav_pdsp_info *pdsp;
 100         struct knav_acc_info *info;
 101         struct knav_device *kdev;
 102
 103         u32 *list, *list_cpu, val, idx, notifies;
 104         int range_base, channel, queue = 0;
 105         dma_addr_t list_dma;
 106
 107         range = _instdata;
 108         info  = &range->acc_info;
 109         kdev  = range->kdev;
 110         pdsp  = range->acc_info.pdsp;
 111         acc   = range->acc;
 112
 113         range_base = kdev->base_id + range->queue_base;
 114         if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
 115                 for (queue = 0; queue < range->num_irqs; queue++)
 116                         if (range->irqs[queue].irq == irq)
 117                                 break;
 118                 kq = knav_range_offset_to_inst(kdev, range, queue);
 119                 acc += queue;
 120         }
 121
 122         channel = acc->channel;
 123         list_dma = acc->list_dma[acc->list_index];
 124         list_cpu = acc->list_cpu[acc->list_index];
 125         dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, phys %x\n",
 126                 channel, acc->list_index, list_cpu, list_dma);
 127         if (atomic_read(&acc->retrigger_count)) {
 128                 atomic_dec(&acc->retrigger_count);
 129                 __knav_acc_notify(range, acc);
 130                 writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 131                 /* ack the interrupt */
 132                 writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 133                                pdsp->intd + ACC_INTD_OFFSET_EOI);
 134
 135                 return IRQ_HANDLED;
 136         }
 137
 138         notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 139         WARN_ON(!notifies);
 140         dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
 141                                 DMA_FROM_DEVICE);
 142
 143         for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
 144              list += ACC_LIST_ENTRY_WORDS) {
 145                 if (ACC_LIST_ENTRY_WORDS == 1) {
 146                         dev_dbg(kdev->dev,
 147                                 "acc-irq: list %d, entry @%p, %08x\n",
 148                                 acc->list_index, list, list[0]);
 149                 } else if (ACC_LIST_ENTRY_WORDS == 2) {
 150                         dev_dbg(kdev->dev,
 151                                 "acc-irq: list %d, entry @%p, %08x %08x\n",
 152                                 acc->list_index, list, list[0], list[1]);
 153                 } else if (ACC_LIST_ENTRY_WORDS == 4) {
 154                         dev_dbg(kdev->dev,
 155                                 "acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
 156                                 acc->list_index, list, list[0], list[1],
 157                                 list[2], list[3]);
 158                 }
 159
 160                 val = list[ACC_LIST_ENTRY_DESC_IDX];
 161                 if (!val)
 162                         break;
 163
 164                 if (range->flags & RANGE_MULTI_QUEUE) {
 165                         queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
 166                         if (queue < range_base ||
 167                             queue >= range_base + range->num_queues) {
 168                                 dev_err(kdev->dev,
 169                                         "bad queue %d, expecting %d-%d\n",
 170                                         queue, range_base,
 171                                         range_base + range->num_queues);
 172                                 break;
 173                         }
 174                         queue -= range_base;
 175                         kq = knav_range_offset_to_inst(kdev, range,
 176                                                                 queue);
 177                 }
 178
 179                 if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
 180                         atomic_dec(&kq->desc_count);
 181                         dev_err(kdev->dev,
 182                                 "acc-irq: queue %d full, entry dropped\n",
 183                                 queue + range_base);
 184                         continue;
 185                 }
 186
 187                 idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
 188                 kq->descs[idx] = val;
 189                 kq->notify_needed = 1;
 190                 dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
 191                         val, idx, queue + range_base);
 192         }
 193
 194         __knav_acc_notify(range, acc);
 195         memset(list_cpu, 0, info->list_size);
 196         dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
 197                                    DMA_TO_DEVICE);
 198
 199         /* flip to the other list */
 200         acc->list_index ^= 1;
 201
 202         /* reset the interrupt counter */
 203         writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
 204
 205         /* ack the interrupt */
 206         writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
 207                        pdsp->intd + ACC_INTD_OFFSET_EOI);
 208
 209         return IRQ_HANDLED;
 210 }
 211
 212 static int knav_range_setup_acc_irq(struct knav_range_info *range,
 213                                 int queue, bool enabled)
 214 {
 215         struct knav_device *kdev = range->kdev;
 216         struct knav_acc_channel *acc;
 217         unsigned long cpu_map;
 218         int ret = 0, irq;
 219         u32 old, new;
 220
 221         if (range->flags & RANGE_MULTI_QUEUE) {
 222                 acc = range->acc;
 223                 irq = range->irqs[0].irq;
 224                 cpu_map = range->irqs[0].cpu_map;
 225         } else {
 226                 acc = range->acc + queue;
 227                 irq = range->irqs[queue].irq;
 228                 cpu_map = range->irqs[queue].cpu_map;
 229         }
 230
 231         old = acc->open_mask;
 232         if (enabled)
 233                 new = old | BIT(queue);
 234         else
 235                 new = old & ~BIT(queue);
 236         acc->open_mask = new;
 237
 238         dev_dbg(kdev->dev,
 239                 "setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
 240                 old, new, acc->name);
 241
 242         if (likely(new == old))
 243                 return 0;
 244
 245         if (new && !old) {
 246                 dev_dbg(kdev->dev,
 247                         "setup-acc-irq: requesting %s for channel %s\n",
 248                         acc->name, acc->name);
 249                 ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
 250                                   range);
 251                 if (!ret && cpu_map) {
 252                         ret = irq_set_affinity_hint(irq, to_cpumask(&cpu_map));
 253                         if (ret) {
 254                                 dev_warn(range->kdev->dev,
 255                                          "Failed to set IRQ affinity\n");
 256                                 return ret;
 257                         }
 258                 }
 259         }
 260
 261         if (old && !new) {
 262                 dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
 263                         acc->name, acc->name);
 264                 free_irq(irq, range);
 265         }
 266
 267         return ret;
 268 }
 269
 270 static const char *knav_acc_result_str(enum knav_acc_result result)
 271 {
 272         static const char * const result_str[] = {
 273                 [ACC_RET_IDLE]                  = "idle",
 274                 [ACC_RET_SUCCESS]               = "success",
 275                 [ACC_RET_INVALID_COMMAND]       = "invalid command",
 276                 [ACC_RET_INVALID_CHANNEL]       = "invalid channel",
 277                 [ACC_RET_INACTIVE_CHANNEL]      = "inactive channel",
 278                 [ACC_RET_ACTIVE_CHANNEL]        = "active channel",
 279                 [ACC_RET_INVALID_QUEUE]         = "invalid queue",
 280                 [ACC_RET_INVALID_RET]           = "invalid return code",
 281         };
 282
 283         if (result >= ARRAY_SIZE(result_str))
 284                 return result_str[ACC_RET_INVALID_RET];
 285         else
 286                 return result_str[result];
 287 }
 288
 289 static enum knav_acc_result
 290 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
 291                 struct knav_reg_acc_command *cmd)
 292 {
 293         u32 result;
 294
 295         dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
 296                 cmd->command, cmd->queue_mask, cmd->list_phys,
 297                 cmd->queue_num, cmd->timer_config);
 298
 299         writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
 300         writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
 301         writel_relaxed(cmd->list_phys, &pdsp->acc_command->list_phys);
 302         writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
 303         writel_relaxed(cmd->command, &pdsp->acc_command->command);
 304
 305         /* wait for the command to clear */
 306         do {
 307                 result = readl_relaxed(&pdsp->acc_command->command);
 308         } while ((result >> 8) & 0xff);
 309
 310         return (result >> 24) & 0xff;
 311 }
 312
 313 static void knav_acc_setup_cmd(struct knav_device *kdev,
 314                                 struct knav_range_info *range,
 315                                 struct knav_reg_acc_command *cmd,
 316                                 int queue)
 317 {
 318         struct knav_acc_info *info = &range->acc_info;
 319         struct knav_acc_channel *acc;
 320         int queue_base;
 321         u32 queue_mask;
 322
 323         if (range->flags & RANGE_MULTI_QUEUE) {
 324                 acc = range->acc;
 325                 queue_base = range->queue_base;
 326                 queue_mask = BIT(range->num_queues) - 1;
 327         } else {
 328                 acc = range->acc + queue;
 329                 queue_base = range->queue_base + queue;
 330                 queue_mask = 0;
 331         }
 332
 333         memset(cmd, 0, sizeof(*cmd));
 334         cmd->command    = acc->channel;
 335         cmd->queue_mask = queue_mask;
 336         cmd->list_phys  = acc->list_dma[0];
 337         cmd->queue_num  = info->list_entries << 16;
 338         cmd->queue_num |= queue_base;
 339
 340         cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
 341         if (range->flags & RANGE_MULTI_QUEUE)
 342                 cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
 343         cmd->timer_config |= info->pacing_mode << 16;
 344         cmd->timer_config |= info->timer_count;
 345 }
 346
 347 static void knav_acc_stop(struct knav_device *kdev,
 348                                 struct knav_range_info *range,
 349                                 int queue)
 350 {
 351         struct knav_reg_acc_command cmd;
 352         struct knav_acc_channel *acc;
 353         enum knav_acc_result result;
 354
 355         acc = range->acc + queue;
 356
 357         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 358         cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
 359         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 360
 361         dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
 362                 acc->name, knav_acc_result_str(result));
 363 }
 364
 365 static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
 366                                                 struct knav_range_info *range,
 367                                                 int queue)
 368 {
 369         struct knav_reg_acc_command cmd;
 370         struct knav_acc_channel *acc;
 371         enum knav_acc_result result;
 372
 373         acc = range->acc + queue;
 374
 375         knav_acc_setup_cmd(kdev, range, &cmd, queue);
 376         cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
 377         result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
 378
 379         dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
 380                 acc->name, knav_acc_result_str(result));
 381
 382         return result;
 383 }
 384
 385 static int knav_acc_init_range(struct knav_range_info *range)
 386 {
 387         struct knav_device *kdev = range->kdev;
 388         struct knav_acc_channel *acc;
 389         enum knav_acc_result result;
 390         int queue;
 391
 392         for (queue = 0; queue < range->num_queues; queue++) {
 393                 acc = range->acc + queue;
 394
 395                 knav_acc_stop(kdev, range, queue);
 396                 acc->list_index = 0;
 397                 result = knav_acc_start(kdev, range, queue);
 398
 399                 if (result != ACC_RET_SUCCESS)
 400                         return -EIO;
 401
 402                 if (range->flags & RANGE_MULTI_QUEUE)
 403                         return 0;
 404         }
 405         return 0;
 406 }
 407
 408 static int knav_acc_init_queue(struct knav_range_info *range,
 409                                 struct knav_queue_inst *kq)
 410 {
 411         unsigned id = kq->id - range->queue_base;
 412
 413         kq->descs = devm_kzalloc(range->kdev->dev,
 414                                  ACC_DESCS_MAX * sizeof(u32), GFP_KERNEL);
 415         if (!kq->descs)
 416                 return -ENOMEM;
 417
 418         kq->acc = range->acc;
 419         if ((range->flags & RANGE_MULTI_QUEUE) == 0)
 420                 kq->acc += id;
 421         return 0;
 422 }
 423
 424 static int knav_acc_open_queue(struct knav_range_info *range,
 425                                 struct knav_queue_inst *inst, unsigned flags)
 426 {
 427         unsigned id = inst->id - range->queue_base;
 428
 429         return knav_range_setup_acc_irq(range, id, true);
 430 }
 431
 432 static int knav_acc_close_queue(struct knav_range_info *range,
 433                                         struct knav_queue_inst *inst)
 434 {
 435         unsigned id = inst->id - range->queue_base;
 436
 437         return knav_range_setup_acc_irq(range, id, false);
 438 }
 439
 440 static int knav_acc_free_range(struct knav_range_info *range)
 441 {
 442         struct knav_device *kdev = range->kdev;
 443         struct knav_acc_channel *acc;
 444         struct knav_acc_info *info;
 445         int channel, channels;
 446
 447         info = &range->acc_info;
 448
 449         if (range->flags & RANGE_MULTI_QUEUE)
 450                 channels = 1;
 451         else
 452                 channels = range->num_queues;
 453
 454         for (channel = 0; channel < channels; channel++) {
 455                 acc = range->acc + channel;
 456                 if (!acc->list_cpu[0])
 457                         continue;
 458                 dma_unmap_single(kdev->dev, acc->list_dma[0],
 459                                  info->mem_size, DMA_BIDIRECTIONAL);
 460                 free_pages_exact(acc->list_cpu[0], info->mem_size);
 461         }
 462         devm_kfree(range->kdev->dev, range->acc);
 463         return 0;
 464 }
 465
 466 struct knav_range_ops knav_acc_range_ops = {
 467         .set_notify     = knav_acc_set_notify,
 468         .init_queue     = knav_acc_init_queue,
 469         .open_queue     = knav_acc_open_queue,
 470         .close_queue    = knav_acc_close_queue,
 471         .init_range     = knav_acc_init_range,
 472         .free_range     = knav_acc_free_range,
 473 };
 474
 475 /**
 476  * knav_init_acc_range: Initialise accumulator ranges
 477  *
 478  * @kdev:               qmss device
 479  * @node:               device node
 480  * @range:              qmms range information
 481  *
 482  * Return 0 on success or error
 483  */
 484 int knav_init_acc_range(struct knav_device *kdev,
 485                                 struct device_node *node,
 486                                 struct knav_range_info *range)
 487 {
 488         struct knav_acc_channel *acc;
 489         struct knav_pdsp_info *pdsp;
 490         struct knav_acc_info *info;
 491         int ret, channel, channels;
 492         int list_size, mem_size;
 493         dma_addr_t list_dma;
 494         void *list_mem;
 495         u32 config[5];
 496
 497         range->flags |= RANGE_HAS_ACCUMULATOR;
 498         info = &range->acc_info;
 499
 500         ret = of_property_read_u32_array(node, "accumulator", config, 5);
 501         if (ret)
 502                 return ret;
 503
 504         info->pdsp_id           = config[0];
 505         info->start_channel     = config[1];
 506         info->list_entries      = config[2];
 507         info->pacing_mode       = config[3];
 508         info->timer_count       = config[4] / ACC_DEFAULT_PERIOD;
 509
 510         if (info->start_channel > ACC_MAX_CHANNEL) {
 511                 dev_err(kdev->dev, "channel %d invalid for range %s\n",
 512                         info->start_channel, range->name);
 513                 return -EINVAL;
 514         }
 515
 516         if (info->pacing_mode > 3) {
 517                 dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
 518                         info->pacing_mode, range->name);
 519                 return -EINVAL;
 520         }
 521
 522         pdsp = knav_find_pdsp(kdev, info->pdsp_id);
 523         if (!pdsp) {
 524                 dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
 525                         info->pdsp_id, range->name);
 526                 return -EINVAL;
 527         }
 528
 529         info->pdsp = pdsp;
 530         channels = range->num_queues;
 531         if (of_get_property(node, "multi-queue", NULL)) {
 532                 range->flags |= RANGE_MULTI_QUEUE;
 533                 channels = 1;
 534                 if (range->queue_base & (32 - 1)) {
 535                         dev_err(kdev->dev,
 536                                 "misaligned multi-queue accumulator range %s\n",
 537                                 range->name);
 538                         return -EINVAL;
 539                 }
 540                 if (range->num_queues > 32) {
 541                         dev_err(kdev->dev,
 542                                 "too many queues in accumulator range %s\n",
 543                                 range->name);
 544                         return -EINVAL;
 545                 }
 546         }
 547
 548         /* figure out list size */
 549         list_size  = info->list_entries;
 550         list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
 551         info->list_size = list_size;
 552         mem_size   = PAGE_ALIGN(list_size * 2);
 553         info->mem_size  = mem_size;
 554         range->acc = devm_kzalloc(kdev->dev, channels * sizeof(*range->acc),
 555                                   GFP_KERNEL);
 556         if (!range->acc)
 557                 return -ENOMEM;
 558
 559         for (channel = 0; channel < channels; channel++) {
 560                 acc = range->acc + channel;
 561                 acc->channel = info->start_channel + channel;
 562
 563                 /* allocate memory for the two lists */
 564                 list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
 565                 if (!list_mem)
 566                         return -ENOMEM;
 567
 568                 list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
 569                                           DMA_BIDIRECTIONAL);
 570                 if (dma_mapping_error(kdev->dev, list_dma)) {
 571                         free_pages_exact(list_mem, mem_size);
 572                         return -ENOMEM;
 573                 }
 574
 575                 memset(list_mem, 0, mem_size);
 576                 dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
 577                                            DMA_TO_DEVICE);
 578                 scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
 579                           acc->channel);
 580                 acc->list_cpu[0] = list_mem;
 581                 acc->list_cpu[1] = list_mem + list_size;
 582                 acc->list_dma[0] = list_dma;
 583                 acc->list_dma[1] = list_dma + list_size;
 584                 dev_dbg(kdev->dev, "%s: channel %d, phys %08x, virt %8p\n",
 585                         acc->name, acc->channel, list_dma, list_mem);
 586         }
 587
 588         range->ops = &knav_acc_range_ops;
 589         return 0;
 590 }
 591 EXPORT_SYMBOL_GPL(knav_init_acc_range);