kernel/drivers/gpu/host1x/job.c

   1 /*
   2  * Tegra host1x Job
   3  *
   4  * Copyright (c) 2010-2013, NVIDIA Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  */
  18
  19 #include <linux/dma-mapping.h>
  20 #include <linux/err.h>
  21 #include <linux/host1x.h>
  22 #include <linux/kref.h>
  23 #include <linux/module.h>
  24 #include <linux/scatterlist.h>
  25 #include <linux/slab.h>
  26 #include <linux/vmalloc.h>
  27 #include <trace/events/host1x.h>
  28
  29 #include "channel.h"
  30 #include "dev.h"
  31 #include "job.h"
  32 #include "syncpt.h"
  33
  34 struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
  35                                     u32 num_cmdbufs, u32 num_relocs,
  36                                     u32 num_waitchks)
  37 {
  38         struct host1x_job *job = NULL;
  39         unsigned int num_unpins = num_cmdbufs + num_relocs;
  40         u64 total;
  41         void *mem;
  42
  43         /* Check that we're not going to overflow */
  44         total = sizeof(struct host1x_job) +
  45                 (u64)num_relocs * sizeof(struct host1x_reloc) +
  46                 (u64)num_unpins * sizeof(struct host1x_job_unpin_data) +
  47                 (u64)num_waitchks * sizeof(struct host1x_waitchk) +
  48                 (u64)num_cmdbufs * sizeof(struct host1x_job_gather) +
  49                 (u64)num_unpins * sizeof(dma_addr_t) +
  50                 (u64)num_unpins * sizeof(u32 *);
  51         if (total > ULONG_MAX)
  52                 return NULL;
  53
  54         mem = job = kzalloc(total, GFP_KERNEL);
  55         if (!job)
  56                 return NULL;
  57
  58         kref_init(&job->ref);
  59         job->channel = ch;
  60
  61         /* Redistribute memory to the structs  */
  62         mem += sizeof(struct host1x_job);
  63         job->relocarray = num_relocs ? mem : NULL;
  64         mem += num_relocs * sizeof(struct host1x_reloc);
  65         job->unpins = num_unpins ? mem : NULL;
  66         mem += num_unpins * sizeof(struct host1x_job_unpin_data);
  67         job->waitchk = num_waitchks ? mem : NULL;
  68         mem += num_waitchks * sizeof(struct host1x_waitchk);
  69         job->gathers = num_cmdbufs ? mem : NULL;
  70         mem += num_cmdbufs * sizeof(struct host1x_job_gather);
  71         job->addr_phys = num_unpins ? mem : NULL;
  72
  73         job->reloc_addr_phys = job->addr_phys;
  74         job->gather_addr_phys = &job->addr_phys[num_relocs];
  75
  76         return job;
  77 }
  78 EXPORT_SYMBOL(host1x_job_alloc);
  79
  80 struct host1x_job *host1x_job_get(struct host1x_job *job)
  81 {
  82         kref_get(&job->ref);
  83         return job;
  84 }
  85 EXPORT_SYMBOL(host1x_job_get);
  86
  87 static void job_free(struct kref *ref)
  88 {
  89         struct host1x_job *job = container_of(ref, struct host1x_job, ref);
  90
  91         kfree(job);
  92 }
  93
  94 void host1x_job_put(struct host1x_job *job)
  95 {
  96         kref_put(&job->ref, job_free);
  97 }
  98 EXPORT_SYMBOL(host1x_job_put);
  99
 100 void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *bo,
 101                            u32 words, u32 offset)
 102 {
 103         struct host1x_job_gather *cur_gather = &job->gathers[job->num_gathers];
 104
 105         cur_gather->words = words;
 106         cur_gather->bo = bo;
 107         cur_gather->offset = offset;
 108         job->num_gathers++;
 109 }
 110 EXPORT_SYMBOL(host1x_job_add_gather);
 111
 112 /*
 113  * NULL an already satisfied WAIT_SYNCPT host method, by patching its
 114  * args in the command stream. The method data is changed to reference
 115  * a reserved (never given out or incr) HOST1X_SYNCPT_RESERVED syncpt
 116  * with a matching threshold value of 0, so is guaranteed to be popped
 117  * by the host HW.
 118  */
 119 static void host1x_syncpt_patch_offset(struct host1x_syncpt *sp,
 120                                        struct host1x_bo *h, u32 offset)
 121 {
 122         void *patch_addr = NULL;
 123
 124         /* patch the wait */
 125         patch_addr = host1x_bo_kmap(h, offset >> PAGE_SHIFT);
 126         if (patch_addr) {
 127                 host1x_syncpt_patch_wait(sp,
 128                                          patch_addr + (offset & ~PAGE_MASK));
 129                 host1x_bo_kunmap(h, offset >> PAGE_SHIFT, patch_addr);
 130         } else
 131                 pr_err("Could not map cmdbuf for wait check\n");
 132 }
 133
 134 /*
 135  * Check driver supplied waitchk structs for syncpt thresholds
 136  * that have already been satisfied and NULL the comparison (to
 137  * avoid a wrap condition in the HW).
 138  */
 139 static int do_waitchks(struct host1x_job *job, struct host1x *host,
 140                        struct host1x_bo *patch)
 141 {
 142         int i;
 143
 144         /* compare syncpt vs wait threshold */
 145         for (i = 0; i < job->num_waitchk; i++) {
 146                 struct host1x_waitchk *wait = &job->waitchk[i];
 147                 struct host1x_syncpt *sp =
 148                         host1x_syncpt_get(host, wait->syncpt_id);
 149
 150                 /* validate syncpt id */
 151                 if (wait->syncpt_id > host1x_syncpt_nb_pts(host))
 152                         continue;
 153
 154                 /* skip all other gathers */
 155                 if (patch != wait->bo)
 156                         continue;
 157
 158                 trace_host1x_syncpt_wait_check(wait->bo, wait->offset,
 159                                                wait->syncpt_id, wait->thresh,
 160                                                host1x_syncpt_read_min(sp));
 161
 162                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
 163                         dev_dbg(host->dev,
 164                                 "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
 165                                 wait->syncpt_id, sp->name, wait->thresh,
 166                                 host1x_syncpt_read_min(sp));
 167
 168                         host1x_syncpt_patch_offset(sp, patch, wait->offset);
 169                 }
 170
 171                 wait->bo = NULL;
 172         }
 173
 174         return 0;
 175 }
 176
 177 static unsigned int pin_job(struct host1x_job *job)
 178 {
 179         unsigned int i;
 180
 181         job->num_unpins = 0;
 182
 183         for (i = 0; i < job->num_relocs; i++) {
 184                 struct host1x_reloc *reloc = &job->relocarray[i];
 185                 struct sg_table *sgt;
 186                 dma_addr_t phys_addr;
 187
 188                 reloc->target.bo = host1x_bo_get(reloc->target.bo);
 189                 if (!reloc->target.bo)
 190                         goto unpin;
 191
 192                 phys_addr = host1x_bo_pin(reloc->target.bo, &sgt);
 193                 if (!phys_addr)
 194                         goto unpin;
 195
 196                 job->addr_phys[job->num_unpins] = phys_addr;
 197                 job->unpins[job->num_unpins].bo = reloc->target.bo;
 198                 job->unpins[job->num_unpins].sgt = sgt;
 199                 job->num_unpins++;
 200         }
 201
 202         for (i = 0; i < job->num_gathers; i++) {
 203                 struct host1x_job_gather *g = &job->gathers[i];
 204                 struct sg_table *sgt;
 205                 dma_addr_t phys_addr;
 206
 207                 g->bo = host1x_bo_get(g->bo);
 208                 if (!g->bo)
 209                         goto unpin;
 210
 211                 phys_addr = host1x_bo_pin(g->bo, &sgt);
 212                 if (!phys_addr)
 213                         goto unpin;
 214
 215                 job->addr_phys[job->num_unpins] = phys_addr;
 216                 job->unpins[job->num_unpins].bo = g->bo;
 217                 job->unpins[job->num_unpins].sgt = sgt;
 218                 job->num_unpins++;
 219         }
 220
 221         return job->num_unpins;
 222
 223 unpin:
 224         host1x_job_unpin(job);
 225         return 0;
 226 }
 227
 228 static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 229 {
 230         int i = 0;
 231         u32 last_page = ~0;
 232         void *cmdbuf_page_addr = NULL;
 233
 234         /* pin & patch the relocs for one gather */
 235         for (i = 0; i < job->num_relocs; i++) {
 236                 struct host1x_reloc *reloc = &job->relocarray[i];
 237                 u32 reloc_addr = (job->reloc_addr_phys[i] +
 238                                   reloc->target.offset) >> reloc->shift;
 239                 u32 *target;
 240
 241                 /* skip all other gathers */
 242                 if (cmdbuf != reloc->cmdbuf.bo)
 243                         continue;
 244
 245                 if (last_page != reloc->cmdbuf.offset >> PAGE_SHIFT) {
 246                         if (cmdbuf_page_addr)
 247                                 host1x_bo_kunmap(cmdbuf, last_page,
 248                                                  cmdbuf_page_addr);
 249
 250                         cmdbuf_page_addr = host1x_bo_kmap(cmdbuf,
 251                                         reloc->cmdbuf.offset >> PAGE_SHIFT);
 252                         last_page = reloc->cmdbuf.offset >> PAGE_SHIFT;
 253
 254                         if (unlikely(!cmdbuf_page_addr)) {
 255                                 pr_err("Could not map cmdbuf for relocation\n");
 256                                 return -ENOMEM;
 257                         }
 258                 }
 259
 260                 target = cmdbuf_page_addr + (reloc->cmdbuf.offset & ~PAGE_MASK);
 261                 *target = reloc_addr;
 262         }
 263
 264         if (cmdbuf_page_addr)
 265                 host1x_bo_kunmap(cmdbuf, last_page, cmdbuf_page_addr);
 266
 267         return 0;
 268 }
 269
 270 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
 271                         unsigned int offset)
 272 {
 273         offset *= sizeof(u32);
 274
 275         if (reloc->cmdbuf.bo != cmdbuf || reloc->cmdbuf.offset != offset)
 276                 return false;
 277
 278         return true;
 279 }
 280
 281 struct host1x_firewall {
 282         struct host1x_job *job;
 283         struct device *dev;
 284
 285         unsigned int num_relocs;
 286         struct host1x_reloc *reloc;
 287
 288         struct host1x_bo *cmdbuf;
 289         unsigned int offset;
 290
 291         u32 words;
 292         u32 class;
 293         u32 reg;
 294         u32 mask;
 295         u32 count;
 296 };
 297
 298 static int check_register(struct host1x_firewall *fw, unsigned long offset)
 299 {
 300         if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
 301                 if (!fw->num_relocs)
 302                         return -EINVAL;
 303
 304                 if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
 305                         return -EINVAL;
 306
 307                 fw->num_relocs--;
 308                 fw->reloc++;
 309         }
 310
 311         return 0;
 312 }
 313
 314 static int check_mask(struct host1x_firewall *fw)
 315 {
 316         u32 mask = fw->mask;
 317         u32 reg = fw->reg;
 318         int ret;
 319
 320         while (mask) {
 321                 if (fw->words == 0)
 322                         return -EINVAL;
 323
 324                 if (mask & 1) {
 325                         ret = check_register(fw, reg);
 326                         if (ret < 0)
 327                                 return ret;
 328
 329                         fw->words--;
 330                         fw->offset++;
 331                 }
 332                 mask >>= 1;
 333                 reg++;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static int check_incr(struct host1x_firewall *fw)
 340 {
 341         u32 count = fw->count;
 342         u32 reg = fw->reg;
 343         int ret;
 344
 345         while (count) {
 346                 if (fw->words == 0)
 347                         return -EINVAL;
 348
 349                 ret = check_register(fw, reg);
 350                 if (ret < 0)
 351                         return ret;
 352
 353                 reg++;
 354                 fw->words--;
 355                 fw->offset++;
 356                 count--;
 357         }
 358
 359         return 0;
 360 }
 361
 362 static int check_nonincr(struct host1x_firewall *fw)
 363 {
 364         u32 count = fw->count;
 365         int ret;
 366
 367         while (count) {
 368                 if (fw->words == 0)
 369                         return -EINVAL;
 370
 371                 ret = check_register(fw, fw->reg);
 372                 if (ret < 0)
 373                         return ret;
 374
 375                 fw->words--;
 376                 fw->offset++;
 377                 count--;
 378         }
 379
 380         return 0;
 381 }
 382
 383 static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
 384 {
 385         u32 *cmdbuf_base = (u32 *)fw->job->gather_copy_mapped +
 386                 (g->offset / sizeof(u32));
 387         int err = 0;
 388
 389         if (!fw->job->is_addr_reg)
 390                 return 0;
 391
 392         fw->words = g->words;
 393         fw->cmdbuf = g->bo;
 394         fw->offset = 0;
 395
 396         while (fw->words && !err) {
 397                 u32 word = cmdbuf_base[fw->offset];
 398                 u32 opcode = (word & 0xf0000000) >> 28;
 399
 400                 fw->mask = 0;
 401                 fw->reg = 0;
 402                 fw->count = 0;
 403                 fw->words--;
 404                 fw->offset++;
 405
 406                 switch (opcode) {
 407                 case 0:
 408                         fw->class = word >> 6 & 0x3ff;
 409                         fw->mask = word & 0x3f;
 410                         fw->reg = word >> 16 & 0xfff;
 411                         err = check_mask(fw);
 412                         if (err)
 413                                 goto out;
 414                         break;
 415                 case 1:
 416                         fw->reg = word >> 16 & 0xfff;
 417                         fw->count = word & 0xffff;
 418                         err = check_incr(fw);
 419                         if (err)
 420                                 goto out;
 421                         break;
 422
 423                 case 2:
 424                         fw->reg = word >> 16 & 0xfff;
 425                         fw->count = word & 0xffff;
 426                         err = check_nonincr(fw);
 427                         if (err)
 428                                 goto out;
 429                         break;
 430
 431                 case 3:
 432                         fw->mask = word & 0xffff;
 433                         fw->reg = word >> 16 & 0xfff;
 434                         err = check_mask(fw);
 435                         if (err)
 436                                 goto out;
 437                         break;
 438                 case 4:
 439                 case 5:
 440                 case 14:
 441                         break;
 442                 default:
 443                         err = -EINVAL;
 444                         break;
 445                 }
 446         }
 447
 448 out:
 449         return err;
 450 }
 451
 452 static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 453 {
 454         struct host1x_firewall fw;
 455         size_t size = 0;
 456         size_t offset = 0;
 457         int i;
 458
 459         fw.job = job;
 460         fw.dev = dev;
 461         fw.reloc = job->relocarray;
 462         fw.num_relocs = job->num_relocs;
 463         fw.class = 0;
 464
 465         for (i = 0; i < job->num_gathers; i++) {
 466                 struct host1x_job_gather *g = &job->gathers[i];
 467                 size += g->words * sizeof(u32);
 468         }
 469
 470         job->gather_copy_mapped = dma_alloc_writecombine(dev, size,
 471                                                          &job->gather_copy,
 472                                                          GFP_KERNEL);
 473         if (!job->gather_copy_mapped) {
 474                 job->gather_copy_mapped = NULL;
 475                 return -ENOMEM;
 476         }
 477
 478         job->gather_copy_size = size;
 479
 480         for (i = 0; i < job->num_gathers; i++) {
 481                 struct host1x_job_gather *g = &job->gathers[i];
 482                 void *gather;
 483
 484                 /* Copy the gather */
 485                 gather = host1x_bo_mmap(g->bo);
 486                 memcpy(job->gather_copy_mapped + offset, gather + g->offset,
 487                        g->words * sizeof(u32));
 488                 host1x_bo_munmap(g->bo, gather);
 489
 490                 /* Store the location in the buffer */
 491                 g->base = job->gather_copy;
 492                 g->offset = offset;
 493
 494                 /* Validate the job */
 495                 if (validate(&fw, g))
 496                         return -EINVAL;
 497
 498                 offset += g->words * sizeof(u32);
 499         }
 500
 501         /* No relocs should remain at this point */
 502         if (fw.num_relocs)
 503                 return -EINVAL;
 504
 505         return 0;
 506 }
 507
 508 int host1x_job_pin(struct host1x_job *job, struct device *dev)
 509 {
 510         int err;
 511         unsigned int i, j;
 512         struct host1x *host = dev_get_drvdata(dev->parent);
 513         DECLARE_BITMAP(waitchk_mask, host1x_syncpt_nb_pts(host));
 514
 515         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
 516         for (i = 0; i < job->num_waitchk; i++) {
 517                 u32 syncpt_id = job->waitchk[i].syncpt_id;
 518                 if (syncpt_id < host1x_syncpt_nb_pts(host))
 519                         set_bit(syncpt_id, waitchk_mask);
 520         }
 521
 522         /* get current syncpt values for waitchk */
 523         for_each_set_bit(i, waitchk_mask, host1x_syncpt_nb_pts(host))
 524                 host1x_syncpt_load(host->syncpt + i);
 525
 526         /* pin memory */
 527         err = pin_job(job);
 528         if (!err)
 529                 goto out;
 530
 531         /* patch gathers */
 532         for (i = 0; i < job->num_gathers; i++) {
 533                 struct host1x_job_gather *g = &job->gathers[i];
 534
 535                 /* process each gather mem only once */
 536                 if (g->handled)
 537                         continue;
 538
 539                 g->base = job->gather_addr_phys[i];
 540
 541                 for (j = i + 1; j < job->num_gathers; j++)
 542                         if (job->gathers[j].bo == g->bo)
 543                                 job->gathers[j].handled = true;
 544
 545                 err = do_relocs(job, g->bo);
 546                 if (err)
 547                         break;
 548
 549                 err = do_waitchks(job, host, g->bo);
 550                 if (err)
 551                         break;
 552         }
 553
 554         if (IS_ENABLED(CONFIG_TEGRA_HOST1X_FIREWALL) && !err) {
 555                 err = copy_gathers(job, dev);
 556                 if (err) {
 557                         host1x_job_unpin(job);
 558                         return err;
 559                 }
 560         }
 561
 562 out:
 563         wmb();
 564
 565         return err;
 566 }
 567 EXPORT_SYMBOL(host1x_job_pin);
 568
 569 void host1x_job_unpin(struct host1x_job *job)
 570 {
 571         unsigned int i;
 572
 573         for (i = 0; i < job->num_unpins; i++) {
 574                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
 575                 host1x_bo_unpin(unpin->bo, unpin->sgt);
 576                 host1x_bo_put(unpin->bo);
 577         }
 578         job->num_unpins = 0;
 579
 580         if (job->gather_copy_size)
 581                 dma_free_writecombine(job->channel->dev, job->gather_copy_size,
 582                                       job->gather_copy_mapped,
 583                                       job->gather_copy);
 584 }
 585 EXPORT_SYMBOL(host1x_job_unpin);
 586
 587 /*
 588  * Debug routine used to dump job entries
 589  */
 590 void host1x_job_dump(struct device *dev, struct host1x_job *job)
 591 {
 592         dev_dbg(dev, "    SYNCPT_ID   %d\n", job->syncpt_id);
 593         dev_dbg(dev, "    SYNCPT_VAL  %d\n", job->syncpt_end);
 594         dev_dbg(dev, "    FIRST_GET   0x%x\n", job->first_get);
 595         dev_dbg(dev, "    TIMEOUT     %d\n", job->timeout);
 596         dev_dbg(dev, "    NUM_SLOTS   %d\n", job->num_slots);
 597         dev_dbg(dev, "    NUM_HANDLES %d\n", job->num_unpins);
 598 }