kernel/drivers/vfio/vfio_iommu_spapr_tce.c

   1 /*
   2  * VFIO: IOMMU DMA mapping support for TCE on POWER
   3  *
   4  * Copyright (C) 2013 IBM Corp.  All rights reserved.
   5  *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * Derived from original vfio_iommu_type1.c:
  12  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
  13  *     Author: Alex Williamson <alex.williamson@redhat.com>
  14  */
  15
  16 #include <linux/module.h>
  17 #include <linux/pci.h>
  18 #include <linux/slab.h>
  19 #include <linux/uaccess.h>
  20 #include <linux/err.h>
  21 #include <linux/vfio.h>
  22 #include <asm/iommu.h>
  23 #include <asm/tce.h>
  24
  25 #define DRIVER_VERSION  "0.1"
  26 #define DRIVER_AUTHOR   "aik@ozlabs.ru"
  27 #define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
  28
  29 static void tce_iommu_detach_group(void *iommu_data,
  30                 struct iommu_group *iommu_group);
  31
  32 /*
  33  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
  34  *
  35  * This code handles mapping and unmapping of user data buffers
  36  * into DMA'ble space using the IOMMU
  37  */
  38
  39 /*
  40  * The container descriptor supports only a single group per container.
  41  * Required by the API as the container is not supplied with the IOMMU group
  42  * at the moment of initialization.
  43  */
  44 struct tce_container {
  45         struct mutex lock;
  46         struct iommu_table *tbl;
  47         bool enabled;
  48 };
  49
  50 static int tce_iommu_enable(struct tce_container *container)
  51 {
  52         int ret = 0;
  53         unsigned long locked, lock_limit, npages;
  54         struct iommu_table *tbl = container->tbl;
  55
  56         if (!container->tbl)
  57                 return -ENXIO;
  58
  59         if (!current->mm)
  60                 return -ESRCH; /* process exited */
  61
  62         if (container->enabled)
  63                 return -EBUSY;
  64
  65         /*
  66          * When userspace pages are mapped into the IOMMU, they are effectively
  67          * locked memory, so, theoretically, we need to update the accounting
  68          * of locked pages on each map and unmap.  For powerpc, the map unmap
  69          * paths can be very hot, though, and the accounting would kill
  70          * performance, especially since it would be difficult to impossible
  71          * to handle the accounting in real mode only.
  72          *
  73          * To address that, rather than precisely accounting every page, we
  74          * instead account for a worst case on locked memory when the iommu is
  75          * enabled and disabled.  The worst case upper bound on locked memory
  76          * is the size of the whole iommu window, which is usually relatively
  77          * small (compared to total memory sizes) on POWER hardware.
  78          *
  79          * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
  80          * that would effectively kill the guest at random points, much better
  81          * enforcing the limit based on the max that the guest can map.
  82          */
  83         down_write(&current->mm->mmap_sem);
  84         npages = (tbl->it_size << IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
  85         locked = current->mm->locked_vm + npages;
  86         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  87         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
  88                 pr_warn("RLIMIT_MEMLOCK (%ld) exceeded\n",
  89                                 rlimit(RLIMIT_MEMLOCK));
  90                 ret = -ENOMEM;
  91         } else {
  92
  93                 current->mm->locked_vm += npages;
  94                 container->enabled = true;
  95         }
  96         up_write(&current->mm->mmap_sem);
  97
  98         return ret;
  99 }
 100
 101 static void tce_iommu_disable(struct tce_container *container)
 102 {
 103         if (!container->enabled)
 104                 return;
 105
 106         container->enabled = false;
 107
 108         if (!container->tbl || !current->mm)
 109                 return;
 110
 111         down_write(&current->mm->mmap_sem);
 112         current->mm->locked_vm -= (container->tbl->it_size <<
 113                         IOMMU_PAGE_SHIFT_4K) >> PAGE_SHIFT;
 114         up_write(&current->mm->mmap_sem);
 115 }
 116
 117 static void *tce_iommu_open(unsigned long arg)
 118 {
 119         struct tce_container *container;
 120
 121         if (arg != VFIO_SPAPR_TCE_IOMMU) {
 122                 pr_err("tce_vfio: Wrong IOMMU type\n");
 123                 return ERR_PTR(-EINVAL);
 124         }
 125
 126         container = kzalloc(sizeof(*container), GFP_KERNEL);
 127         if (!container)
 128                 return ERR_PTR(-ENOMEM);
 129
 130         mutex_init(&container->lock);
 131
 132         return container;
 133 }
 134
 135 static void tce_iommu_release(void *iommu_data)
 136 {
 137         struct tce_container *container = iommu_data;
 138
 139         WARN_ON(container->tbl && !container->tbl->it_group);
 140         tce_iommu_disable(container);
 141
 142         if (container->tbl && container->tbl->it_group)
 143                 tce_iommu_detach_group(iommu_data, container->tbl->it_group);
 144
 145         mutex_destroy(&container->lock);
 146
 147         kfree(container);
 148 }
 149
 150 static long tce_iommu_ioctl(void *iommu_data,
 151                                  unsigned int cmd, unsigned long arg)
 152 {
 153         struct tce_container *container = iommu_data;
 154         unsigned long minsz;
 155         long ret;
 156
 157         switch (cmd) {
 158         case VFIO_CHECK_EXTENSION:
 159                 switch (arg) {
 160                 case VFIO_SPAPR_TCE_IOMMU:
 161                         ret = 1;
 162                         break;
 163                 default:
 164                         ret = vfio_spapr_iommu_eeh_ioctl(NULL, cmd, arg);
 165                         break;
 166                 }
 167
 168                 return (ret < 0) ? 0 : ret;
 169
 170         case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
 171                 struct vfio_iommu_spapr_tce_info info;
 172                 struct iommu_table *tbl = container->tbl;
 173
 174                 if (WARN_ON(!tbl))
 175                         return -ENXIO;
 176
 177                 minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
 178                                 dma32_window_size);
 179
 180                 if (copy_from_user(&info, (void __user *)arg, minsz))
 181                         return -EFAULT;
 182
 183                 if (info.argsz < minsz)
 184                         return -EINVAL;
 185
 186                 info.dma32_window_start = tbl->it_offset << IOMMU_PAGE_SHIFT_4K;
 187                 info.dma32_window_size = tbl->it_size << IOMMU_PAGE_SHIFT_4K;
 188                 info.flags = 0;
 189
 190                 if (copy_to_user((void __user *)arg, &info, minsz))
 191                         return -EFAULT;
 192
 193                 return 0;
 194         }
 195         case VFIO_IOMMU_MAP_DMA: {
 196                 struct vfio_iommu_type1_dma_map param;
 197                 struct iommu_table *tbl = container->tbl;
 198                 unsigned long tce, i;
 199
 200                 if (!tbl)
 201                         return -ENXIO;
 202
 203                 BUG_ON(!tbl->it_group);
 204
 205                 minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
 206
 207                 if (copy_from_user(&param, (void __user *)arg, minsz))
 208                         return -EFAULT;
 209
 210                 if (param.argsz < minsz)
 211                         return -EINVAL;
 212
 213                 if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
 214                                 VFIO_DMA_MAP_FLAG_WRITE))
 215                         return -EINVAL;
 216
 217                 if ((param.size & ~IOMMU_PAGE_MASK_4K) ||
 218                                 (param.vaddr & ~IOMMU_PAGE_MASK_4K))
 219                         return -EINVAL;
 220
 221                 /* iova is checked by the IOMMU API */
 222                 tce = param.vaddr;
 223                 if (param.flags & VFIO_DMA_MAP_FLAG_READ)
 224                         tce |= TCE_PCI_READ;
 225                 if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
 226                         tce |= TCE_PCI_WRITE;
 227
 228                 ret = iommu_tce_put_param_check(tbl, param.iova, tce);
 229                 if (ret)
 230                         return ret;
 231
 232                 for (i = 0; i < (param.size >> IOMMU_PAGE_SHIFT_4K); ++i) {
 233                         ret = iommu_put_tce_user_mode(tbl,
 234                                         (param.iova >> IOMMU_PAGE_SHIFT_4K) + i,
 235                                         tce);
 236                         if (ret)
 237                                 break;
 238                         tce += IOMMU_PAGE_SIZE_4K;
 239                 }
 240                 if (ret)
 241                         iommu_clear_tces_and_put_pages(tbl,
 242                                         param.iova >> IOMMU_PAGE_SHIFT_4K, i);
 243
 244                 iommu_flush_tce(tbl);
 245
 246                 return ret;
 247         }
 248         case VFIO_IOMMU_UNMAP_DMA: {
 249                 struct vfio_iommu_type1_dma_unmap param;
 250                 struct iommu_table *tbl = container->tbl;
 251
 252                 if (WARN_ON(!tbl))
 253                         return -ENXIO;
 254
 255                 minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
 256                                 size);
 257
 258                 if (copy_from_user(&param, (void __user *)arg, minsz))
 259                         return -EFAULT;
 260
 261                 if (param.argsz < minsz)
 262                         return -EINVAL;
 263
 264                 /* No flag is supported now */
 265                 if (param.flags)
 266                         return -EINVAL;
 267
 268                 if (param.size & ~IOMMU_PAGE_MASK_4K)
 269                         return -EINVAL;
 270
 271                 ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
 272                                 param.size >> IOMMU_PAGE_SHIFT_4K);
 273                 if (ret)
 274                         return ret;
 275
 276                 ret = iommu_clear_tces_and_put_pages(tbl,
 277                                 param.iova >> IOMMU_PAGE_SHIFT_4K,
 278                                 param.size >> IOMMU_PAGE_SHIFT_4K);
 279                 iommu_flush_tce(tbl);
 280
 281                 return ret;
 282         }
 283         case VFIO_IOMMU_ENABLE:
 284                 mutex_lock(&container->lock);
 285                 ret = tce_iommu_enable(container);
 286                 mutex_unlock(&container->lock);
 287                 return ret;
 288
 289
 290         case VFIO_IOMMU_DISABLE:
 291                 mutex_lock(&container->lock);
 292                 tce_iommu_disable(container);
 293                 mutex_unlock(&container->lock);
 294                 return 0;
 295         case VFIO_EEH_PE_OP:
 296                 if (!container->tbl || !container->tbl->it_group)
 297                         return -ENODEV;
 298
 299                 return vfio_spapr_iommu_eeh_ioctl(container->tbl->it_group,
 300                                                   cmd, arg);
 301         }
 302
 303         return -ENOTTY;
 304 }
 305
 306 static int tce_iommu_attach_group(void *iommu_data,
 307                 struct iommu_group *iommu_group)
 308 {
 309         int ret;
 310         struct tce_container *container = iommu_data;
 311         struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);
 312
 313         BUG_ON(!tbl);
 314         mutex_lock(&container->lock);
 315
 316         /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
 317                         iommu_group_id(iommu_group), iommu_group); */
 318         if (container->tbl) {
 319                 pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n",
 320                                 iommu_group_id(container->tbl->it_group),
 321                                 iommu_group_id(iommu_group));
 322                 ret = -EBUSY;
 323         } else if (container->enabled) {
 324                 pr_err("tce_vfio: attaching group #%u to enabled container\n",
 325                                 iommu_group_id(iommu_group));
 326                 ret = -EBUSY;
 327         } else {
 328                 ret = iommu_take_ownership(tbl);
 329                 if (!ret)
 330                         container->tbl = tbl;
 331         }
 332
 333         mutex_unlock(&container->lock);
 334
 335         return ret;
 336 }
 337
 338 static void tce_iommu_detach_group(void *iommu_data,
 339                 struct iommu_group *iommu_group)
 340 {
 341         struct tce_container *container = iommu_data;
 342         struct iommu_table *tbl = iommu_group_get_iommudata(iommu_group);
 343
 344         BUG_ON(!tbl);
 345         mutex_lock(&container->lock);
 346         if (tbl != container->tbl) {
 347                 pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n",
 348                                 iommu_group_id(iommu_group),
 349                                 iommu_group_id(tbl->it_group));
 350         } else {
 351                 if (container->enabled) {
 352                         pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n",
 353                                         iommu_group_id(tbl->it_group));
 354                         tce_iommu_disable(container);
 355                 }
 356
 357                 /* pr_debug("tce_vfio: detaching group #%u from iommu %p\n",
 358                                 iommu_group_id(iommu_group), iommu_group); */
 359                 container->tbl = NULL;
 360                 iommu_release_ownership(tbl);
 361         }
 362         mutex_unlock(&container->lock);
 363 }
 364
 365 const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
 366         .name           = "iommu-vfio-powerpc",
 367         .owner          = THIS_MODULE,
 368         .open           = tce_iommu_open,
 369         .release        = tce_iommu_release,
 370         .ioctl          = tce_iommu_ioctl,
 371         .attach_group   = tce_iommu_attach_group,
 372         .detach_group   = tce_iommu_detach_group,
 373 };
 374
 375 static int __init tce_iommu_init(void)
 376 {
 377         return vfio_register_iommu_driver(&tce_iommu_driver_ops);
 378 }
 379
 380 static void __exit tce_iommu_cleanup(void)
 381 {
 382         vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
 383 }
 384
 385 module_init(tce_iommu_init);
 386 module_exit(tce_iommu_cleanup);
 387
 388 MODULE_VERSION(DRIVER_VERSION);
 389 MODULE_LICENSE("GPL v2");
 390 MODULE_AUTHOR(DRIVER_AUTHOR);
 391 MODULE_DESCRIPTION(DRIVER_DESC);
 392