kernel/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/slab.h>
  25 #include <linux/list.h>
  26 #include <linux/types.h>
  27 #include <linux/printk.h>
  28 #include <linux/bitops.h>
  29 #include <linux/sched.h>
  30 #include "kfd_priv.h"
  31 #include "kfd_device_queue_manager.h"
  32 #include "kfd_mqd_manager.h"
  33 #include "cik_regs.h"
  34 #include "kfd_kernel_queue.h"
  35
  36 /* Size of the per-pipe EOP queue */
  37 #define CIK_HPD_EOP_BYTES_LOG2 11
  38 #define CIK_HPD_EOP_BYTES (1U << CIK_HPD_EOP_BYTES_LOG2)
  39
  40 static int set_pasid_vmid_mapping(struct device_queue_manager *dqm,
  41                                         unsigned int pasid, unsigned int vmid);
  42
  43 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
  44                                         struct queue *q,
  45                                         struct qcm_process_device *qpd);
  46
  47 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  48 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock);
  49
  50 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
  51                                         struct queue *q,
  52                                         struct qcm_process_device *qpd);
  53
  54 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
  55                                 unsigned int sdma_queue_id);
  56
  57 static inline
  58 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
  59 {
  60         if (type == KFD_QUEUE_TYPE_SDMA)
  61                 return KFD_MQD_TYPE_SDMA;
  62         return KFD_MQD_TYPE_CP;
  63 }
  64
  65 unsigned int get_first_pipe(struct device_queue_manager *dqm)
  66 {
  67         BUG_ON(!dqm || !dqm->dev);
  68         return dqm->dev->shared_resources.first_compute_pipe;
  69 }
  70
  71 unsigned int get_pipes_num(struct device_queue_manager *dqm)
  72 {
  73         BUG_ON(!dqm || !dqm->dev);
  74         return dqm->dev->shared_resources.compute_pipe_count;
  75 }
  76
  77 static inline unsigned int get_pipes_num_cpsch(void)
  78 {
  79         return PIPE_PER_ME_CP_SCHEDULING;
  80 }
  81
  82 void program_sh_mem_settings(struct device_queue_manager *dqm,
  83                                         struct qcm_process_device *qpd)
  84 {
  85         return dqm->dev->kfd2kgd->program_sh_mem_settings(
  86                                                 dqm->dev->kgd, qpd->vmid,
  87                                                 qpd->sh_mem_config,
  88                                                 qpd->sh_mem_ape1_base,
  89                                                 qpd->sh_mem_ape1_limit,
  90                                                 qpd->sh_mem_bases);
  91 }
  92
  93 static int allocate_vmid(struct device_queue_manager *dqm,
  94                         struct qcm_process_device *qpd,
  95                         struct queue *q)
  96 {
  97         int bit, allocated_vmid;
  98
  99         if (dqm->vmid_bitmap == 0)
 100                 return -ENOMEM;
 101
 102         bit = find_first_bit((unsigned long *)&dqm->vmid_bitmap, CIK_VMID_NUM);
 103         clear_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 104
 105         /* Kaveri kfd vmid's starts from vmid 8 */
 106         allocated_vmid = bit + KFD_VMID_START_OFFSET;
 107         pr_debug("kfd: vmid allocation %d\n", allocated_vmid);
 108         qpd->vmid = allocated_vmid;
 109         q->properties.vmid = allocated_vmid;
 110
 111         set_pasid_vmid_mapping(dqm, q->process->pasid, q->properties.vmid);
 112         program_sh_mem_settings(dqm, qpd);
 113
 114         return 0;
 115 }
 116
 117 static void deallocate_vmid(struct device_queue_manager *dqm,
 118                                 struct qcm_process_device *qpd,
 119                                 struct queue *q)
 120 {
 121         int bit = qpd->vmid - KFD_VMID_START_OFFSET;
 122
 123         /* Release the vmid mapping */
 124         set_pasid_vmid_mapping(dqm, 0, qpd->vmid);
 125
 126         set_bit(bit, (unsigned long *)&dqm->vmid_bitmap);
 127         qpd->vmid = 0;
 128         q->properties.vmid = 0;
 129 }
 130
 131 static int create_queue_nocpsch(struct device_queue_manager *dqm,
 132                                 struct queue *q,
 133                                 struct qcm_process_device *qpd,
 134                                 int *allocated_vmid)
 135 {
 136         int retval;
 137
 138         BUG_ON(!dqm || !q || !qpd || !allocated_vmid);
 139
 140         pr_debug("kfd: In func %s\n", __func__);
 141         print_queue(q);
 142
 143         mutex_lock(&dqm->lock);
 144
 145         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 146                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 147                                 dqm->total_queue_count);
 148                 mutex_unlock(&dqm->lock);
 149                 return -EPERM;
 150         }
 151
 152         if (list_empty(&qpd->queues_list)) {
 153                 retval = allocate_vmid(dqm, qpd, q);
 154                 if (retval != 0) {
 155                         mutex_unlock(&dqm->lock);
 156                         return retval;
 157                 }
 158         }
 159         *allocated_vmid = qpd->vmid;
 160         q->properties.vmid = qpd->vmid;
 161
 162         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE)
 163                 retval = create_compute_queue_nocpsch(dqm, q, qpd);
 164         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 165                 retval = create_sdma_queue_nocpsch(dqm, q, qpd);
 166
 167         if (retval != 0) {
 168                 if (list_empty(&qpd->queues_list)) {
 169                         deallocate_vmid(dqm, qpd, q);
 170                         *allocated_vmid = 0;
 171                 }
 172                 mutex_unlock(&dqm->lock);
 173                 return retval;
 174         }
 175
 176         list_add(&q->list, &qpd->queues_list);
 177         if (q->properties.is_active)
 178                 dqm->queue_count++;
 179
 180         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 181                 dqm->sdma_queue_count++;
 182
 183         /*
 184          * Unconditionally increment this counter, regardless of the queue's
 185          * type or whether the queue is active.
 186          */
 187         dqm->total_queue_count++;
 188         pr_debug("Total of %d queues are accountable so far\n",
 189                         dqm->total_queue_count);
 190
 191         mutex_unlock(&dqm->lock);
 192         return 0;
 193 }
 194
 195 static int allocate_hqd(struct device_queue_manager *dqm, struct queue *q)
 196 {
 197         bool set;
 198         int pipe, bit, i;
 199
 200         set = false;
 201
 202         for (pipe = dqm->next_pipe_to_allocate, i = 0; i < get_pipes_num(dqm);
 203                         pipe = ((pipe + 1) % get_pipes_num(dqm)), ++i) {
 204                 if (dqm->allocated_queues[pipe] != 0) {
 205                         bit = find_first_bit(
 206                                 (unsigned long *)&dqm->allocated_queues[pipe],
 207                                 QUEUES_PER_PIPE);
 208
 209                         clear_bit(bit,
 210                                 (unsigned long *)&dqm->allocated_queues[pipe]);
 211                         q->pipe = pipe;
 212                         q->queue = bit;
 213                         set = true;
 214                         break;
 215                 }
 216         }
 217
 218         if (set == false)
 219                 return -EBUSY;
 220
 221         pr_debug("kfd: DQM %s hqd slot - pipe (%d) queue(%d)\n",
 222                                 __func__, q->pipe, q->queue);
 223         /* horizontal hqd allocation */
 224         dqm->next_pipe_to_allocate = (pipe + 1) % get_pipes_num(dqm);
 225
 226         return 0;
 227 }
 228
 229 static inline void deallocate_hqd(struct device_queue_manager *dqm,
 230                                 struct queue *q)
 231 {
 232         set_bit(q->queue, (unsigned long *)&dqm->allocated_queues[q->pipe]);
 233 }
 234
 235 static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
 236                                         struct queue *q,
 237                                         struct qcm_process_device *qpd)
 238 {
 239         int retval;
 240         struct mqd_manager *mqd;
 241
 242         BUG_ON(!dqm || !q || !qpd);
 243
 244         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 245         if (mqd == NULL)
 246                 return -ENOMEM;
 247
 248         retval = allocate_hqd(dqm, q);
 249         if (retval != 0)
 250                 return retval;
 251
 252         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 253                                 &q->gart_mqd_addr, &q->properties);
 254         if (retval != 0) {
 255                 deallocate_hqd(dqm, q);
 256                 return retval;
 257         }
 258
 259         pr_debug("kfd: loading mqd to hqd on pipe (%d) queue (%d)\n",
 260                         q->pipe,
 261                         q->queue);
 262
 263         retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
 264                         q->queue, (uint32_t __user *) q->properties.write_ptr);
 265         if (retval != 0) {
 266                 deallocate_hqd(dqm, q);
 267                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 268                 return retval;
 269         }
 270
 271         return 0;
 272 }
 273
 274 static int destroy_queue_nocpsch(struct device_queue_manager *dqm,
 275                                 struct qcm_process_device *qpd,
 276                                 struct queue *q)
 277 {
 278         int retval;
 279         struct mqd_manager *mqd;
 280
 281         BUG_ON(!dqm || !q || !q->mqd || !qpd);
 282
 283         retval = 0;
 284
 285         pr_debug("kfd: In Func %s\n", __func__);
 286
 287         mutex_lock(&dqm->lock);
 288
 289         if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 290                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 291                 if (mqd == NULL) {
 292                         retval = -ENOMEM;
 293                         goto out;
 294                 }
 295                 deallocate_hqd(dqm, q);
 296         } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) {
 297                 mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 298                 if (mqd == NULL) {
 299                         retval = -ENOMEM;
 300                         goto out;
 301                 }
 302                 dqm->sdma_queue_count--;
 303                 deallocate_sdma_queue(dqm, q->sdma_id);
 304         } else {
 305                 pr_debug("q->properties.type is invalid (%d)\n",
 306                                 q->properties.type);
 307                 retval = -EINVAL;
 308                 goto out;
 309         }
 310
 311         retval = mqd->destroy_mqd(mqd, q->mqd,
 312                                 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 313                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS,
 314                                 q->pipe, q->queue);
 315
 316         if (retval != 0)
 317                 goto out;
 318
 319         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 320
 321         list_del(&q->list);
 322         if (list_empty(&qpd->queues_list))
 323                 deallocate_vmid(dqm, qpd, q);
 324         if (q->properties.is_active)
 325                 dqm->queue_count--;
 326
 327         /*
 328          * Unconditionally decrement this counter, regardless of the queue's
 329          * type
 330          */
 331         dqm->total_queue_count--;
 332         pr_debug("Total of %d queues are accountable so far\n",
 333                         dqm->total_queue_count);
 334
 335 out:
 336         mutex_unlock(&dqm->lock);
 337         return retval;
 338 }
 339
 340 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 341 {
 342         int retval;
 343         struct mqd_manager *mqd;
 344         bool prev_active = false;
 345
 346         BUG_ON(!dqm || !q || !q->mqd);
 347
 348         mutex_lock(&dqm->lock);
 349         mqd = dqm->ops.get_mqd_manager(dqm,
 350                         get_mqd_type_from_queue_type(q->properties.type));
 351         if (mqd == NULL) {
 352                 mutex_unlock(&dqm->lock);
 353                 return -ENOMEM;
 354         }
 355
 356         if (q->properties.is_active == true)
 357                 prev_active = true;
 358
 359         /*
 360          *
 361          * check active state vs. the previous state
 362          * and modify counter accordingly
 363          */
 364         retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
 365         if ((q->properties.is_active == true) && (prev_active == false))
 366                 dqm->queue_count++;
 367         else if ((q->properties.is_active == false) && (prev_active == true))
 368                 dqm->queue_count--;
 369
 370         if (sched_policy != KFD_SCHED_POLICY_NO_HWS)
 371                 retval = execute_queues_cpsch(dqm, false);
 372
 373         mutex_unlock(&dqm->lock);
 374         return retval;
 375 }
 376
 377 static struct mqd_manager *get_mqd_manager_nocpsch(
 378                 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 379 {
 380         struct mqd_manager *mqd;
 381
 382         BUG_ON(!dqm || type >= KFD_MQD_TYPE_MAX);
 383
 384         pr_debug("kfd: In func %s mqd type %d\n", __func__, type);
 385
 386         mqd = dqm->mqds[type];
 387         if (!mqd) {
 388                 mqd = mqd_manager_init(type, dqm->dev);
 389                 if (mqd == NULL)
 390                         pr_err("kfd: mqd manager is NULL");
 391                 dqm->mqds[type] = mqd;
 392         }
 393
 394         return mqd;
 395 }
 396
 397 static int register_process_nocpsch(struct device_queue_manager *dqm,
 398                                         struct qcm_process_device *qpd)
 399 {
 400         struct device_process_node *n;
 401         int retval;
 402
 403         BUG_ON(!dqm || !qpd);
 404
 405         pr_debug("kfd: In func %s\n", __func__);
 406
 407         n = kzalloc(sizeof(struct device_process_node), GFP_KERNEL);
 408         if (!n)
 409                 return -ENOMEM;
 410
 411         n->qpd = qpd;
 412
 413         mutex_lock(&dqm->lock);
 414         list_add(&n->list, &dqm->queues);
 415
 416         retval = dqm->ops_asic_specific.register_process(dqm, qpd);
 417
 418         dqm->processes_count++;
 419
 420         mutex_unlock(&dqm->lock);
 421
 422         return retval;
 423 }
 424
 425 static int unregister_process_nocpsch(struct device_queue_manager *dqm,
 426                                         struct qcm_process_device *qpd)
 427 {
 428         int retval;
 429         struct device_process_node *cur, *next;
 430
 431         BUG_ON(!dqm || !qpd);
 432
 433         pr_debug("In func %s\n", __func__);
 434
 435         pr_debug("qpd->queues_list is %s\n",
 436                         list_empty(&qpd->queues_list) ? "empty" : "not empty");
 437
 438         retval = 0;
 439         mutex_lock(&dqm->lock);
 440
 441         list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 442                 if (qpd == cur->qpd) {
 443                         list_del(&cur->list);
 444                         kfree(cur);
 445                         dqm->processes_count--;
 446                         goto out;
 447                 }
 448         }
 449         /* qpd not found in dqm list */
 450         retval = 1;
 451 out:
 452         mutex_unlock(&dqm->lock);
 453         return retval;
 454 }
 455
 456 static int
 457 set_pasid_vmid_mapping(struct device_queue_manager *dqm, unsigned int pasid,
 458                         unsigned int vmid)
 459 {
 460         uint32_t pasid_mapping;
 461
 462         pasid_mapping = (pasid == 0) ? 0 :
 463                 (uint32_t)pasid |
 464                 ATC_VMID_PASID_MAPPING_VALID;
 465
 466         return dqm->dev->kfd2kgd->set_pasid_vmid_mapping(
 467                                                 dqm->dev->kgd, pasid_mapping,
 468                                                 vmid);
 469 }
 470
 471 int init_pipelines(struct device_queue_manager *dqm,
 472                         unsigned int pipes_num, unsigned int first_pipe)
 473 {
 474         void *hpdptr;
 475         struct mqd_manager *mqd;
 476         unsigned int i, err, inx;
 477         uint64_t pipe_hpd_addr;
 478
 479         BUG_ON(!dqm || !dqm->dev);
 480
 481         pr_debug("kfd: In func %s\n", __func__);
 482
 483         /*
 484          * Allocate memory for the HPDs. This is hardware-owned per-pipe data.
 485          * The driver never accesses this memory after zeroing it.
 486          * It doesn't even have to be saved/restored on suspend/resume
 487          * because it contains no data when there are no active queues.
 488          */
 489
 490         err = kfd_gtt_sa_allocate(dqm->dev, CIK_HPD_EOP_BYTES * pipes_num,
 491                                         &dqm->pipeline_mem);
 492
 493         if (err) {
 494                 pr_err("kfd: error allocate vidmem num pipes: %d\n",
 495                         pipes_num);
 496                 return -ENOMEM;
 497         }
 498
 499         hpdptr = dqm->pipeline_mem->cpu_ptr;
 500         dqm->pipelines_addr = dqm->pipeline_mem->gpu_addr;
 501
 502         memset(hpdptr, 0, CIK_HPD_EOP_BYTES * pipes_num);
 503
 504         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
 505         if (mqd == NULL) {
 506                 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 507                 return -ENOMEM;
 508         }
 509
 510         for (i = 0; i < pipes_num; i++) {
 511                 inx = i + first_pipe;
 512                 /*
 513                  * HPD buffer on GTT is allocated by amdkfd, no need to waste
 514                  * space in GTT for pipelines we don't initialize
 515                  */
 516                 pipe_hpd_addr = dqm->pipelines_addr + i * CIK_HPD_EOP_BYTES;
 517                 pr_debug("kfd: pipeline address %llX\n", pipe_hpd_addr);
 518                 /* = log2(bytes/4)-1 */
 519                 dqm->dev->kfd2kgd->init_pipeline(dqm->dev->kgd, inx,
 520                                 CIK_HPD_EOP_BYTES_LOG2 - 3, pipe_hpd_addr);
 521         }
 522
 523         return 0;
 524 }
 525
 526 static int init_scheduler(struct device_queue_manager *dqm)
 527 {
 528         int retval;
 529
 530         BUG_ON(!dqm);
 531
 532         pr_debug("kfd: In %s\n", __func__);
 533
 534         retval = init_pipelines(dqm, get_pipes_num(dqm), get_first_pipe(dqm));
 535         return retval;
 536 }
 537
 538 static int initialize_nocpsch(struct device_queue_manager *dqm)
 539 {
 540         int i;
 541
 542         BUG_ON(!dqm);
 543
 544         pr_debug("kfd: In func %s num of pipes: %d\n",
 545                         __func__, get_pipes_num(dqm));
 546
 547         mutex_init(&dqm->lock);
 548         INIT_LIST_HEAD(&dqm->queues);
 549         dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 550         dqm->sdma_queue_count = 0;
 551         dqm->allocated_queues = kcalloc(get_pipes_num(dqm),
 552                                         sizeof(unsigned int), GFP_KERNEL);
 553         if (!dqm->allocated_queues) {
 554                 mutex_destroy(&dqm->lock);
 555                 return -ENOMEM;
 556         }
 557
 558         for (i = 0; i < get_pipes_num(dqm); i++)
 559                 dqm->allocated_queues[i] = (1 << QUEUES_PER_PIPE) - 1;
 560
 561         dqm->vmid_bitmap = (1 << VMID_PER_DEVICE) - 1;
 562         dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
 563
 564         init_scheduler(dqm);
 565         return 0;
 566 }
 567
 568 static void uninitialize_nocpsch(struct device_queue_manager *dqm)
 569 {
 570         int i;
 571
 572         BUG_ON(!dqm);
 573
 574         BUG_ON(dqm->queue_count > 0 || dqm->processes_count > 0);
 575
 576         kfree(dqm->allocated_queues);
 577         for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
 578                 kfree(dqm->mqds[i]);
 579         mutex_destroy(&dqm->lock);
 580         kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 581 }
 582
 583 static int start_nocpsch(struct device_queue_manager *dqm)
 584 {
 585         return 0;
 586 }
 587
 588 static int stop_nocpsch(struct device_queue_manager *dqm)
 589 {
 590         return 0;
 591 }
 592
 593 static int allocate_sdma_queue(struct device_queue_manager *dqm,
 594                                 unsigned int *sdma_queue_id)
 595 {
 596         int bit;
 597
 598         if (dqm->sdma_bitmap == 0)
 599                 return -ENOMEM;
 600
 601         bit = find_first_bit((unsigned long *)&dqm->sdma_bitmap,
 602                                 CIK_SDMA_QUEUES);
 603
 604         clear_bit(bit, (unsigned long *)&dqm->sdma_bitmap);
 605         *sdma_queue_id = bit;
 606
 607         return 0;
 608 }
 609
 610 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 611                                 unsigned int sdma_queue_id)
 612 {
 613         if (sdma_queue_id >= CIK_SDMA_QUEUES)
 614                 return;
 615         set_bit(sdma_queue_id, (unsigned long *)&dqm->sdma_bitmap);
 616 }
 617
 618 static void init_sdma_vm(struct device_queue_manager *dqm, struct queue *q,
 619                                 struct qcm_process_device *qpd)
 620 {
 621         uint32_t value = SDMA_ATC;
 622
 623         if (q->process->is_32bit_user_mode)
 624                 value |= SDMA_VA_PTR32 | get_sh_mem_bases_32(qpd_to_pdd(qpd));
 625         else
 626                 value |= SDMA_VA_SHARED_BASE(get_sh_mem_bases_nybble_64(
 627                                                         qpd_to_pdd(qpd)));
 628         q->properties.sdma_vm_addr = value;
 629 }
 630
 631 static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm,
 632                                         struct queue *q,
 633                                         struct qcm_process_device *qpd)
 634 {
 635         struct mqd_manager *mqd;
 636         int retval;
 637
 638         mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
 639         if (!mqd)
 640                 return -ENOMEM;
 641
 642         retval = allocate_sdma_queue(dqm, &q->sdma_id);
 643         if (retval != 0)
 644                 return retval;
 645
 646         q->properties.sdma_queue_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
 647         q->properties.sdma_engine_id = q->sdma_id / CIK_SDMA_ENGINE_NUM;
 648
 649         pr_debug("kfd: sdma id is:    %d\n", q->sdma_id);
 650         pr_debug("     sdma queue id: %d\n", q->properties.sdma_queue_id);
 651         pr_debug("     sdma engine id: %d\n", q->properties.sdma_engine_id);
 652
 653         init_sdma_vm(dqm, q, qpd);
 654         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 655                                 &q->gart_mqd_addr, &q->properties);
 656         if (retval != 0) {
 657                 deallocate_sdma_queue(dqm, q->sdma_id);
 658                 return retval;
 659         }
 660
 661         retval = mqd->load_mqd(mqd, q->mqd, 0,
 662                                 0, NULL);
 663         if (retval != 0) {
 664                 deallocate_sdma_queue(dqm, q->sdma_id);
 665                 mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
 666                 return retval;
 667         }
 668
 669         return 0;
 670 }
 671
 672 /*
 673  * Device Queue Manager implementation for cp scheduler
 674  */
 675
 676 static int set_sched_resources(struct device_queue_manager *dqm)
 677 {
 678         struct scheduling_resources res;
 679         unsigned int queue_num, queue_mask;
 680
 681         BUG_ON(!dqm);
 682
 683         pr_debug("kfd: In func %s\n", __func__);
 684
 685         queue_num = get_pipes_num_cpsch() * QUEUES_PER_PIPE;
 686         queue_mask = (1 << queue_num) - 1;
 687         res.vmid_mask = (1 << VMID_PER_DEVICE) - 1;
 688         res.vmid_mask <<= KFD_VMID_START_OFFSET;
 689         res.queue_mask = queue_mask << (get_first_pipe(dqm) * QUEUES_PER_PIPE);
 690         res.gws_mask = res.oac_mask = res.gds_heap_base =
 691                                                 res.gds_heap_size = 0;
 692
 693         pr_debug("kfd: scheduling resources:\n"
 694                         "      vmid mask: 0x%8X\n"
 695                         "      queue mask: 0x%8llX\n",
 696                         res.vmid_mask, res.queue_mask);
 697
 698         return pm_send_set_resources(&dqm->packets, &res);
 699 }
 700
 701 static int initialize_cpsch(struct device_queue_manager *dqm)
 702 {
 703         int retval;
 704
 705         BUG_ON(!dqm);
 706
 707         pr_debug("kfd: In func %s num of pipes: %d\n",
 708                         __func__, get_pipes_num_cpsch());
 709
 710         mutex_init(&dqm->lock);
 711         INIT_LIST_HEAD(&dqm->queues);
 712         dqm->queue_count = dqm->processes_count = 0;
 713         dqm->sdma_queue_count = 0;
 714         dqm->active_runlist = false;
 715         retval = dqm->ops_asic_specific.initialize(dqm);
 716         if (retval != 0)
 717                 goto fail_init_pipelines;
 718
 719         return 0;
 720
 721 fail_init_pipelines:
 722         mutex_destroy(&dqm->lock);
 723         return retval;
 724 }
 725
 726 static int start_cpsch(struct device_queue_manager *dqm)
 727 {
 728         struct device_process_node *node;
 729         int retval;
 730
 731         BUG_ON(!dqm);
 732
 733         retval = 0;
 734
 735         retval = pm_init(&dqm->packets, dqm);
 736         if (retval != 0)
 737                 goto fail_packet_manager_init;
 738
 739         retval = set_sched_resources(dqm);
 740         if (retval != 0)
 741                 goto fail_set_sched_resources;
 742
 743         pr_debug("kfd: allocating fence memory\n");
 744
 745         /* allocate fence memory on the gart */
 746         retval = kfd_gtt_sa_allocate(dqm->dev, sizeof(*dqm->fence_addr),
 747                                         &dqm->fence_mem);
 748
 749         if (retval != 0)
 750                 goto fail_allocate_vidmem;
 751
 752         dqm->fence_addr = dqm->fence_mem->cpu_ptr;
 753         dqm->fence_gpu_addr = dqm->fence_mem->gpu_addr;
 754         list_for_each_entry(node, &dqm->queues, list)
 755                 if (node->qpd->pqm->process && dqm->dev)
 756                         kfd_bind_process_to_device(dqm->dev,
 757                                                 node->qpd->pqm->process);
 758
 759         execute_queues_cpsch(dqm, true);
 760
 761         return 0;
 762 fail_allocate_vidmem:
 763 fail_set_sched_resources:
 764         pm_uninit(&dqm->packets);
 765 fail_packet_manager_init:
 766         return retval;
 767 }
 768
 769 static int stop_cpsch(struct device_queue_manager *dqm)
 770 {
 771         struct device_process_node *node;
 772         struct kfd_process_device *pdd;
 773
 774         BUG_ON(!dqm);
 775
 776         destroy_queues_cpsch(dqm, true);
 777
 778         list_for_each_entry(node, &dqm->queues, list) {
 779                 pdd = qpd_to_pdd(node->qpd);
 780                 pdd->bound = false;
 781         }
 782         kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
 783         pm_uninit(&dqm->packets);
 784
 785         return 0;
 786 }
 787
 788 static int create_kernel_queue_cpsch(struct device_queue_manager *dqm,
 789                                         struct kernel_queue *kq,
 790                                         struct qcm_process_device *qpd)
 791 {
 792         BUG_ON(!dqm || !kq || !qpd);
 793
 794         pr_debug("kfd: In func %s\n", __func__);
 795
 796         mutex_lock(&dqm->lock);
 797         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 798                 pr_warn("amdkfd: Can't create new kernel queue because %d queues were already created\n",
 799                                 dqm->total_queue_count);
 800                 mutex_unlock(&dqm->lock);
 801                 return -EPERM;
 802         }
 803
 804         /*
 805          * Unconditionally increment this counter, regardless of the queue's
 806          * type or whether the queue is active.
 807          */
 808         dqm->total_queue_count++;
 809         pr_debug("Total of %d queues are accountable so far\n",
 810                         dqm->total_queue_count);
 811
 812         list_add(&kq->list, &qpd->priv_queue_list);
 813         dqm->queue_count++;
 814         qpd->is_debug = true;
 815         execute_queues_cpsch(dqm, false);
 816         mutex_unlock(&dqm->lock);
 817
 818         return 0;
 819 }
 820
 821 static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm,
 822                                         struct kernel_queue *kq,
 823                                         struct qcm_process_device *qpd)
 824 {
 825         BUG_ON(!dqm || !kq);
 826
 827         pr_debug("kfd: In %s\n", __func__);
 828
 829         mutex_lock(&dqm->lock);
 830         destroy_queues_cpsch(dqm, false);
 831         list_del(&kq->list);
 832         dqm->queue_count--;
 833         qpd->is_debug = false;
 834         execute_queues_cpsch(dqm, false);
 835         /*
 836          * Unconditionally decrement this counter, regardless of the queue's
 837          * type.
 838          */
 839         dqm->total_queue_count--;
 840         pr_debug("Total of %d queues are accountable so far\n",
 841                         dqm->total_queue_count);
 842         mutex_unlock(&dqm->lock);
 843 }
 844
 845 static void select_sdma_engine_id(struct queue *q)
 846 {
 847         static int sdma_id;
 848
 849         q->sdma_id = sdma_id;
 850         sdma_id = (sdma_id + 1) % 2;
 851 }
 852
 853 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 854                         struct qcm_process_device *qpd, int *allocate_vmid)
 855 {
 856         int retval;
 857         struct mqd_manager *mqd;
 858
 859         BUG_ON(!dqm || !q || !qpd);
 860
 861         retval = 0;
 862
 863         if (allocate_vmid)
 864                 *allocate_vmid = 0;
 865
 866         mutex_lock(&dqm->lock);
 867
 868         if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 869                 pr_warn("amdkfd: Can't create new usermode queue because %d queues were already created\n",
 870                                 dqm->total_queue_count);
 871                 retval = -EPERM;
 872                 goto out;
 873         }
 874
 875         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 876                 select_sdma_engine_id(q);
 877
 878         mqd = dqm->ops.get_mqd_manager(dqm,
 879                         get_mqd_type_from_queue_type(q->properties.type));
 880
 881         if (mqd == NULL) {
 882                 mutex_unlock(&dqm->lock);
 883                 return -ENOMEM;
 884         }
 885
 886         init_sdma_vm(dqm, q, qpd);
 887
 888         retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
 889                                 &q->gart_mqd_addr, &q->properties);
 890         if (retval != 0)
 891                 goto out;
 892
 893         list_add(&q->list, &qpd->queues_list);
 894         if (q->properties.is_active) {
 895                 dqm->queue_count++;
 896                 retval = execute_queues_cpsch(dqm, false);
 897         }
 898
 899         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 900                         dqm->sdma_queue_count++;
 901         /*
 902          * Unconditionally increment this counter, regardless of the queue's
 903          * type or whether the queue is active.
 904          */
 905         dqm->total_queue_count++;
 906
 907         pr_debug("Total of %d queues are accountable so far\n",
 908                         dqm->total_queue_count);
 909
 910 out:
 911         mutex_unlock(&dqm->lock);
 912         return retval;
 913 }
 914
 915 static int amdkfd_fence_wait_timeout(unsigned int *fence_addr,
 916                                 unsigned int fence_value,
 917                                 unsigned long timeout)
 918 {
 919         BUG_ON(!fence_addr);
 920         timeout += jiffies;
 921
 922         while (*fence_addr != fence_value) {
 923                 if (time_after(jiffies, timeout)) {
 924                         pr_err("kfd: qcm fence wait loop timeout expired\n");
 925                         return -ETIME;
 926                 }
 927                 schedule();
 928         }
 929
 930         return 0;
 931 }
 932
 933 static int destroy_sdma_queues(struct device_queue_manager *dqm,
 934                                 unsigned int sdma_engine)
 935 {
 936         return pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA,
 937                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false,
 938                         sdma_engine);
 939 }
 940
 941 static int destroy_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 942 {
 943         int retval;
 944
 945         BUG_ON(!dqm);
 946
 947         retval = 0;
 948
 949         if (lock)
 950                 mutex_lock(&dqm->lock);
 951         if (dqm->active_runlist == false)
 952                 goto out;
 953
 954         pr_debug("kfd: Before destroying queues, sdma queue count is : %u\n",
 955                 dqm->sdma_queue_count);
 956
 957         if (dqm->sdma_queue_count > 0) {
 958                 destroy_sdma_queues(dqm, 0);
 959                 destroy_sdma_queues(dqm, 1);
 960         }
 961
 962         retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE,
 963                         KFD_PREEMPT_TYPE_FILTER_ALL_QUEUES, 0, false, 0);
 964         if (retval != 0)
 965                 goto out;
 966
 967         *dqm->fence_addr = KFD_FENCE_INIT;
 968         pm_send_query_status(&dqm->packets, dqm->fence_gpu_addr,
 969                                 KFD_FENCE_COMPLETED);
 970         /* should be timed out */
 971         amdkfd_fence_wait_timeout(dqm->fence_addr, KFD_FENCE_COMPLETED,
 972                                 QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS);
 973         pm_release_ib(&dqm->packets);
 974         dqm->active_runlist = false;
 975
 976 out:
 977         if (lock)
 978                 mutex_unlock(&dqm->lock);
 979         return retval;
 980 }
 981
 982 static int execute_queues_cpsch(struct device_queue_manager *dqm, bool lock)
 983 {
 984         int retval;
 985
 986         BUG_ON(!dqm);
 987
 988         if (lock)
 989                 mutex_lock(&dqm->lock);
 990
 991         retval = destroy_queues_cpsch(dqm, false);
 992         if (retval != 0) {
 993                 pr_err("kfd: the cp might be in an unrecoverable state due to an unsuccessful queues preemption");
 994                 goto out;
 995         }
 996
 997         if (dqm->queue_count <= 0 || dqm->processes_count <= 0) {
 998                 retval = 0;
 999                 goto out;
1000         }
1001
1002         if (dqm->active_runlist) {
1003                 retval = 0;
1004                 goto out;
1005         }
1006
1007         retval = pm_send_runlist(&dqm->packets, &dqm->queues);
1008         if (retval != 0) {
1009                 pr_err("kfd: failed to execute runlist");
1010                 goto out;
1011         }
1012         dqm->active_runlist = true;
1013
1014 out:
1015         if (lock)
1016                 mutex_unlock(&dqm->lock);
1017         return retval;
1018 }
1019
1020 static int destroy_queue_cpsch(struct device_queue_manager *dqm,
1021                                 struct qcm_process_device *qpd,
1022                                 struct queue *q)
1023 {
1024         int retval;
1025         struct mqd_manager *mqd;
1026
1027         BUG_ON(!dqm || !qpd || !q);
1028
1029         retval = 0;
1030
1031         /* remove queue from list to prevent rescheduling after preemption */
1032         mutex_lock(&dqm->lock);
1033         mqd = dqm->ops.get_mqd_manager(dqm,
1034                         get_mqd_type_from_queue_type(q->properties.type));
1035         if (!mqd) {
1036                 retval = -ENOMEM;
1037                 goto failed;
1038         }
1039
1040         if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
1041                 dqm->sdma_queue_count--;
1042
1043         list_del(&q->list);
1044         if (q->properties.is_active)
1045                 dqm->queue_count--;
1046
1047         execute_queues_cpsch(dqm, false);
1048
1049         mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1050
1051         /*
1052          * Unconditionally decrement this counter, regardless of the queue's
1053          * type
1054          */
1055         dqm->total_queue_count--;
1056         pr_debug("Total of %d queues are accountable so far\n",
1057                         dqm->total_queue_count);
1058
1059         mutex_unlock(&dqm->lock);
1060
1061         return 0;
1062
1063 failed:
1064         mutex_unlock(&dqm->lock);
1065         return retval;
1066 }
1067
1068 /*
1069  * Low bits must be 0000/FFFF as required by HW, high bits must be 0 to
1070  * stay in user mode.
1071  */
1072 #define APE1_FIXED_BITS_MASK 0xFFFF80000000FFFFULL
1073 /* APE1 limit is inclusive and 64K aligned. */
1074 #define APE1_LIMIT_ALIGNMENT 0xFFFF
1075
1076 static bool set_cache_memory_policy(struct device_queue_manager *dqm,
1077                                    struct qcm_process_device *qpd,
1078                                    enum cache_policy default_policy,
1079                                    enum cache_policy alternate_policy,
1080                                    void __user *alternate_aperture_base,
1081                                    uint64_t alternate_aperture_size)
1082 {
1083         bool retval;
1084
1085         pr_debug("kfd: In func %s\n", __func__);
1086
1087         mutex_lock(&dqm->lock);
1088
1089         if (alternate_aperture_size == 0) {
1090                 /* base > limit disables APE1 */
1091                 qpd->sh_mem_ape1_base = 1;
1092                 qpd->sh_mem_ape1_limit = 0;
1093         } else {
1094                 /*
1095                  * In FSA64, APE1_Base[63:0] = { 16{SH_MEM_APE1_BASE[31]},
1096                  *                      SH_MEM_APE1_BASE[31:0], 0x0000 }
1097                  * APE1_Limit[63:0] = { 16{SH_MEM_APE1_LIMIT[31]},
1098                  *                      SH_MEM_APE1_LIMIT[31:0], 0xFFFF }
1099                  * Verify that the base and size parameters can be
1100                  * represented in this format and convert them.
1101                  * Additionally restrict APE1 to user-mode addresses.
1102                  */
1103
1104                 uint64_t base = (uintptr_t)alternate_aperture_base;
1105                 uint64_t limit = base + alternate_aperture_size - 1;
1106
1107                 if (limit <= base)
1108                         goto out;
1109
1110                 if ((base & APE1_FIXED_BITS_MASK) != 0)
1111                         goto out;
1112
1113                 if ((limit & APE1_FIXED_BITS_MASK) != APE1_LIMIT_ALIGNMENT)
1114                         goto out;
1115
1116                 qpd->sh_mem_ape1_base = base >> 16;
1117                 qpd->sh_mem_ape1_limit = limit >> 16;
1118         }
1119
1120         retval = dqm->ops_asic_specific.set_cache_memory_policy(
1121                         dqm,
1122                         qpd,
1123                         default_policy,
1124                         alternate_policy,
1125                         alternate_aperture_base,
1126                         alternate_aperture_size);
1127
1128         if ((sched_policy == KFD_SCHED_POLICY_NO_HWS) && (qpd->vmid != 0))
1129                 program_sh_mem_settings(dqm, qpd);
1130
1131         pr_debug("kfd: sh_mem_config: 0x%x, ape1_base: 0x%x, ape1_limit: 0x%x\n",
1132                 qpd->sh_mem_config, qpd->sh_mem_ape1_base,
1133                 qpd->sh_mem_ape1_limit);
1134
1135         mutex_unlock(&dqm->lock);
1136         return retval;
1137
1138 out:
1139         mutex_unlock(&dqm->lock);
1140         return false;
1141 }
1142
1143 struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
1144 {
1145         struct device_queue_manager *dqm;
1146
1147         BUG_ON(!dev);
1148
1149         pr_debug("kfd: loading device queue manager\n");
1150
1151         dqm = kzalloc(sizeof(struct device_queue_manager), GFP_KERNEL);
1152         if (!dqm)
1153                 return NULL;
1154
1155         dqm->dev = dev;
1156         switch (sched_policy) {
1157         case KFD_SCHED_POLICY_HWS:
1158         case KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION:
1159                 /* initialize dqm for cp scheduling */
1160                 dqm->ops.create_queue = create_queue_cpsch;
1161                 dqm->ops.initialize = initialize_cpsch;
1162                 dqm->ops.start = start_cpsch;
1163                 dqm->ops.stop = stop_cpsch;
1164                 dqm->ops.destroy_queue = destroy_queue_cpsch;
1165                 dqm->ops.update_queue = update_queue;
1166                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1167                 dqm->ops.register_process = register_process_nocpsch;
1168                 dqm->ops.unregister_process = unregister_process_nocpsch;
1169                 dqm->ops.uninitialize = uninitialize_nocpsch;
1170                 dqm->ops.create_kernel_queue = create_kernel_queue_cpsch;
1171                 dqm->ops.destroy_kernel_queue = destroy_kernel_queue_cpsch;
1172                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1173                 break;
1174         case KFD_SCHED_POLICY_NO_HWS:
1175                 /* initialize dqm for no cp scheduling */
1176                 dqm->ops.start = start_nocpsch;
1177                 dqm->ops.stop = stop_nocpsch;
1178                 dqm->ops.create_queue = create_queue_nocpsch;
1179                 dqm->ops.destroy_queue = destroy_queue_nocpsch;
1180                 dqm->ops.update_queue = update_queue;
1181                 dqm->ops.get_mqd_manager = get_mqd_manager_nocpsch;
1182                 dqm->ops.register_process = register_process_nocpsch;
1183                 dqm->ops.unregister_process = unregister_process_nocpsch;
1184                 dqm->ops.initialize = initialize_nocpsch;
1185                 dqm->ops.uninitialize = uninitialize_nocpsch;
1186                 dqm->ops.set_cache_memory_policy = set_cache_memory_policy;
1187                 break;
1188         default:
1189                 BUG();
1190                 break;
1191         }
1192
1193         switch (dev->device_info->asic_family) {
1194         case CHIP_CARRIZO:
1195                 device_queue_manager_init_vi(&dqm->ops_asic_specific);
1196                 break;
1197
1198         case CHIP_KAVERI:
1199                 device_queue_manager_init_cik(&dqm->ops_asic_specific);
1200                 break;
1201         }
1202
1203         if (dqm->ops.initialize(dqm) != 0) {
1204                 kfree(dqm);
1205                 return NULL;
1206         }
1207
1208         return dqm;
1209 }
1210
1211 void device_queue_manager_uninit(struct device_queue_manager *dqm)
1212 {
1213         BUG_ON(!dqm);
1214
1215         dqm->ops.uninitialize(dqm);
1216         kfree(dqm);
1217 }