kernel/virt/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include <kvm/iodev.h>
  20
  21 #include <linux/kvm_host.h>
  22 #include <linux/kvm.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/percpu.h>
  26 #include <linux/mm.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/vmalloc.h>
  29 #include <linux/reboot.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/highmem.h>
  32 #include <linux/file.h>
  33 #include <linux/syscore_ops.h>
  34 #include <linux/cpu.h>
  35 #include <linux/sched.h>
  36 #include <linux/cpumask.h>
  37 #include <linux/smp.h>
  38 #include <linux/anon_inodes.h>
  39 #include <linux/profile.h>
  40 #include <linux/kvm_para.h>
  41 #include <linux/pagemap.h>
  42 #include <linux/mman.h>
  43 #include <linux/swap.h>
  44 #include <linux/bitops.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/compat.h>
  47 #include <linux/srcu.h>
  48 #include <linux/hugetlb.h>
  49 #include <linux/slab.h>
  50 #include <linux/sort.h>
  51 #include <linux/bsearch.h>
  52
  53 #include <asm/processor.h>
  54 #include <asm/io.h>
  55 #include <asm/ioctl.h>
  56 #include <asm/uaccess.h>
  57 #include <asm/pgtable.h>
  58
  59 #include "coalesced_mmio.h"
  60 #include "async_pf.h"
  61 #include "vfio.h"
  62
  63 #define CREATE_TRACE_POINTS
  64 #include <trace/events/kvm.h>
  65
  66 MODULE_AUTHOR("Qumranet");
  67 MODULE_LICENSE("GPL");
  68
  69 /* Architectures should define their poll value according to the halt latency */
  70 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
  71 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
  72
  73 /* Default doubles per-vcpu halt_poll_ns. */
  74 static unsigned int halt_poll_ns_grow = 2;
  75 module_param(halt_poll_ns_grow, int, S_IRUGO);
  76
  77 /* Default resets per-vcpu halt_poll_ns . */
  78 static unsigned int halt_poll_ns_shrink;
  79 module_param(halt_poll_ns_shrink, int, S_IRUGO);
  80
  81 /*
  82  * Ordering of locks:
  83  *
  84  *      kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  85  */
  86
  87 DEFINE_SPINLOCK(kvm_lock);
  88 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
  89 LIST_HEAD(vm_list);
  90
  91 static cpumask_var_t cpus_hardware_enabled;
  92 static int kvm_usage_count;
  93 static atomic_t hardware_enable_failed;
  94
  95 struct kmem_cache *kvm_vcpu_cache;
  96 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  97
  98 static __read_mostly struct preempt_ops kvm_preempt_ops;
  99
 100 struct dentry *kvm_debugfs_dir;
 101 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
 102
 103 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 104                            unsigned long arg);
 105 #ifdef CONFIG_KVM_COMPAT
 106 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
 107                                   unsigned long arg);
 108 #endif
 109 static int hardware_enable_all(void);
 110 static void hardware_disable_all(void);
 111
 112 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
 113
 114 static void kvm_release_pfn_dirty(pfn_t pfn);
 115 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
 116
 117 __visible bool kvm_rebooting;
 118 EXPORT_SYMBOL_GPL(kvm_rebooting);
 119
 120 static bool largepages_enabled = true;
 121
 122 bool kvm_is_reserved_pfn(pfn_t pfn)
 123 {
 124         if (pfn_valid(pfn))
 125                 return PageReserved(pfn_to_page(pfn));
 126
 127         return true;
 128 }
 129
 130 /*
 131  * Switches to specified vcpu, until a matching vcpu_put()
 132  */
 133 int vcpu_load(struct kvm_vcpu *vcpu)
 134 {
 135         int cpu;
 136
 137         if (mutex_lock_killable(&vcpu->mutex))
 138                 return -EINTR;
 139         cpu = get_cpu();
 140         preempt_notifier_register(&vcpu->preempt_notifier);
 141         kvm_arch_vcpu_load(vcpu, cpu);
 142         put_cpu();
 143         return 0;
 144 }
 145 EXPORT_SYMBOL_GPL(vcpu_load);
 146
 147 void vcpu_put(struct kvm_vcpu *vcpu)
 148 {
 149         preempt_disable();
 150         kvm_arch_vcpu_put(vcpu);
 151         preempt_notifier_unregister(&vcpu->preempt_notifier);
 152         preempt_enable();
 153         mutex_unlock(&vcpu->mutex);
 154 }
 155 EXPORT_SYMBOL_GPL(vcpu_put);
 156
 157 static void ack_flush(void *_completed)
 158 {
 159 }
 160
 161 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
 162 {
 163         int i, cpu, me;
 164         cpumask_var_t cpus;
 165         bool called = true;
 166         struct kvm_vcpu *vcpu;
 167
 168         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
 169
 170         me = get_cpu();
 171         kvm_for_each_vcpu(i, vcpu, kvm) {
 172                 kvm_make_request(req, vcpu);
 173                 cpu = vcpu->cpu;
 174
 175                 /* Set ->requests bit before we read ->mode */
 176                 smp_mb();
 177
 178                 if (cpus != NULL && cpu != -1 && cpu != me &&
 179                       kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
 180                         cpumask_set_cpu(cpu, cpus);
 181         }
 182         if (unlikely(cpus == NULL))
 183                 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 184         else if (!cpumask_empty(cpus))
 185                 smp_call_function_many(cpus, ack_flush, NULL, 1);
 186         else
 187                 called = false;
 188         put_cpu();
 189         free_cpumask_var(cpus);
 190         return called;
 191 }
 192
 193 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
 194 void kvm_flush_remote_tlbs(struct kvm *kvm)
 195 {
 196         long dirty_count = kvm->tlbs_dirty;
 197
 198         smp_mb();
 199         if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 200                 ++kvm->stat.remote_tlb_flush;
 201         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 202 }
 203 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
 204 #endif
 205
 206 void kvm_reload_remote_mmus(struct kvm *kvm)
 207 {
 208         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 209 }
 210
 211 void kvm_make_mclock_inprogress_request(struct kvm *kvm)
 212 {
 213         kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
 214 }
 215
 216 void kvm_make_scan_ioapic_request(struct kvm *kvm)
 217 {
 218         kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
 219 }
 220
 221 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 222 {
 223         struct page *page;
 224         int r;
 225
 226         mutex_init(&vcpu->mutex);
 227         vcpu->cpu = -1;
 228         vcpu->kvm = kvm;
 229         vcpu->vcpu_id = id;
 230         vcpu->pid = NULL;
 231         init_swait_queue_head(&vcpu->wq);
 232         kvm_async_pf_vcpu_init(vcpu);
 233
 234         vcpu->pre_pcpu = -1;
 235         INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
 236
 237         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 238         if (!page) {
 239                 r = -ENOMEM;
 240                 goto fail;
 241         }
 242         vcpu->run = page_address(page);
 243
 244         kvm_vcpu_set_in_spin_loop(vcpu, false);
 245         kvm_vcpu_set_dy_eligible(vcpu, false);
 246         vcpu->preempted = false;
 247
 248         r = kvm_arch_vcpu_init(vcpu);
 249         if (r < 0)
 250                 goto fail_free_run;
 251         return 0;
 252
 253 fail_free_run:
 254         free_page((unsigned long)vcpu->run);
 255 fail:
 256         return r;
 257 }
 258 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 259
 260 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 261 {
 262         put_pid(vcpu->pid);
 263         kvm_arch_vcpu_uninit(vcpu);
 264         free_page((unsigned long)vcpu->run);
 265 }
 266 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 267
 268 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 269 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 270 {
 271         return container_of(mn, struct kvm, mmu_notifier);
 272 }
 273
 274 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 275                                              struct mm_struct *mm,
 276                                              unsigned long address)
 277 {
 278         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 279         int need_tlb_flush, idx;
 280
 281         /*
 282          * When ->invalidate_page runs, the linux pte has been zapped
 283          * already but the page is still allocated until
 284          * ->invalidate_page returns. So if we increase the sequence
 285          * here the kvm page fault will notice if the spte can't be
 286          * established because the page is going to be freed. If
 287          * instead the kvm page fault establishes the spte before
 288          * ->invalidate_page runs, kvm_unmap_hva will release it
 289          * before returning.
 290          *
 291          * The sequence increase only need to be seen at spin_unlock
 292          * time, and not at spin_lock time.
 293          *
 294          * Increasing the sequence after the spin_unlock would be
 295          * unsafe because the kvm page fault could then establish the
 296          * pte after kvm_unmap_hva returned, without noticing the page
 297          * is going to be freed.
 298          */
 299         idx = srcu_read_lock(&kvm->srcu);
 300         spin_lock(&kvm->mmu_lock);
 301
 302         kvm->mmu_notifier_seq++;
 303         need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
 304         /* we've to flush the tlb before the pages can be freed */
 305         if (need_tlb_flush)
 306                 kvm_flush_remote_tlbs(kvm);
 307
 308         spin_unlock(&kvm->mmu_lock);
 309
 310         kvm_arch_mmu_notifier_invalidate_page(kvm, address);
 311
 312         srcu_read_unlock(&kvm->srcu, idx);
 313 }
 314
 315 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 316                                         struct mm_struct *mm,
 317                                         unsigned long address,
 318                                         pte_t pte)
 319 {
 320         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 321         int idx;
 322
 323         idx = srcu_read_lock(&kvm->srcu);
 324         spin_lock(&kvm->mmu_lock);
 325         kvm->mmu_notifier_seq++;
 326         kvm_set_spte_hva(kvm, address, pte);
 327         spin_unlock(&kvm->mmu_lock);
 328         srcu_read_unlock(&kvm->srcu, idx);
 329 }
 330
 331 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 332                                                     struct mm_struct *mm,
 333                                                     unsigned long start,
 334                                                     unsigned long end)
 335 {
 336         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 337         int need_tlb_flush = 0, idx;
 338
 339         idx = srcu_read_lock(&kvm->srcu);
 340         spin_lock(&kvm->mmu_lock);
 341         /*
 342          * The count increase must become visible at unlock time as no
 343          * spte can be established without taking the mmu_lock and
 344          * count is also read inside the mmu_lock critical section.
 345          */
 346         kvm->mmu_notifier_count++;
 347         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
 348         need_tlb_flush |= kvm->tlbs_dirty;
 349         /* we've to flush the tlb before the pages can be freed */
 350         if (need_tlb_flush)
 351                 kvm_flush_remote_tlbs(kvm);
 352
 353         spin_unlock(&kvm->mmu_lock);
 354         srcu_read_unlock(&kvm->srcu, idx);
 355 }
 356
 357 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 358                                                   struct mm_struct *mm,
 359                                                   unsigned long start,
 360                                                   unsigned long end)
 361 {
 362         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 363
 364         spin_lock(&kvm->mmu_lock);
 365         /*
 366          * This sequence increase will notify the kvm page fault that
 367          * the page that is going to be mapped in the spte could have
 368          * been freed.
 369          */
 370         kvm->mmu_notifier_seq++;
 371         smp_wmb();
 372         /*
 373          * The above sequence increase must be visible before the
 374          * below count decrease, which is ensured by the smp_wmb above
 375          * in conjunction with the smp_rmb in mmu_notifier_retry().
 376          */
 377         kvm->mmu_notifier_count--;
 378         spin_unlock(&kvm->mmu_lock);
 379
 380         BUG_ON(kvm->mmu_notifier_count < 0);
 381 }
 382
 383 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 384                                               struct mm_struct *mm,
 385                                               unsigned long start,
 386                                               unsigned long end)
 387 {
 388         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 389         int young, idx;
 390
 391         idx = srcu_read_lock(&kvm->srcu);
 392         spin_lock(&kvm->mmu_lock);
 393
 394         young = kvm_age_hva(kvm, start, end);
 395         if (young)
 396                 kvm_flush_remote_tlbs(kvm);
 397
 398         spin_unlock(&kvm->mmu_lock);
 399         srcu_read_unlock(&kvm->srcu, idx);
 400
 401         return young;
 402 }
 403
 404 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
 405                                         struct mm_struct *mm,
 406                                         unsigned long start,
 407                                         unsigned long end)
 408 {
 409         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 410         int young, idx;
 411
 412         idx = srcu_read_lock(&kvm->srcu);
 413         spin_lock(&kvm->mmu_lock);
 414         /*
 415          * Even though we do not flush TLB, this will still adversely
 416          * affect performance on pre-Haswell Intel EPT, where there is
 417          * no EPT Access Bit to clear so that we have to tear down EPT
 418          * tables instead. If we find this unacceptable, we can always
 419          * add a parameter to kvm_age_hva so that it effectively doesn't
 420          * do anything on clear_young.
 421          *
 422          * Also note that currently we never issue secondary TLB flushes
 423          * from clear_young, leaving this job up to the regular system
 424          * cadence. If we find this inaccurate, we might come up with a
 425          * more sophisticated heuristic later.
 426          */
 427         young = kvm_age_hva(kvm, start, end);
 428         spin_unlock(&kvm->mmu_lock);
 429         srcu_read_unlock(&kvm->srcu, idx);
 430
 431         return young;
 432 }
 433
 434 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
 435                                        struct mm_struct *mm,
 436                                        unsigned long address)
 437 {
 438         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 439         int young, idx;
 440
 441         idx = srcu_read_lock(&kvm->srcu);
 442         spin_lock(&kvm->mmu_lock);
 443         young = kvm_test_age_hva(kvm, address);
 444         spin_unlock(&kvm->mmu_lock);
 445         srcu_read_unlock(&kvm->srcu, idx);
 446
 447         return young;
 448 }
 449
 450 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 451                                      struct mm_struct *mm)
 452 {
 453         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 454         int idx;
 455
 456         idx = srcu_read_lock(&kvm->srcu);
 457         kvm_arch_flush_shadow_all(kvm);
 458         srcu_read_unlock(&kvm->srcu, idx);
 459 }
 460
 461 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 462         .invalidate_page        = kvm_mmu_notifier_invalidate_page,
 463         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
 464         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
 465         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
 466         .clear_young            = kvm_mmu_notifier_clear_young,
 467         .test_young             = kvm_mmu_notifier_test_young,
 468         .change_pte             = kvm_mmu_notifier_change_pte,
 469         .release                = kvm_mmu_notifier_release,
 470 };
 471
 472 static int kvm_init_mmu_notifier(struct kvm *kvm)
 473 {
 474         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 475         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 476 }
 477
 478 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
 479
 480 static int kvm_init_mmu_notifier(struct kvm *kvm)
 481 {
 482         return 0;
 483 }
 484
 485 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 486
 487 static struct kvm_memslots *kvm_alloc_memslots(void)
 488 {
 489         int i;
 490         struct kvm_memslots *slots;
 491
 492         slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
 493         if (!slots)
 494                 return NULL;
 495
 496         /*
 497          * Init kvm generation close to the maximum to easily test the
 498          * code of handling generation number wrap-around.
 499          */
 500         slots->generation = -150;
 501         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
 502                 slots->id_to_index[i] = slots->memslots[i].id = i;
 503
 504         return slots;
 505 }
 506
 507 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 508 {
 509         if (!memslot->dirty_bitmap)
 510                 return;
 511
 512         kvfree(memslot->dirty_bitmap);
 513         memslot->dirty_bitmap = NULL;
 514 }
 515
 516 /*
 517  * Free any memory in @free but not in @dont.
 518  */
 519 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 520                               struct kvm_memory_slot *dont)
 521 {
 522         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 523                 kvm_destroy_dirty_bitmap(free);
 524
 525         kvm_arch_free_memslot(kvm, free, dont);
 526
 527         free->npages = 0;
 528 }
 529
 530 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
 531 {
 532         struct kvm_memory_slot *memslot;
 533
 534         if (!slots)
 535                 return;
 536
 537         kvm_for_each_memslot(memslot, slots)
 538                 kvm_free_memslot(kvm, memslot, NULL);
 539
 540         kvfree(slots);
 541 }
 542
 543 static struct kvm *kvm_create_vm(unsigned long type)
 544 {
 545         int r, i;
 546         struct kvm *kvm = kvm_arch_alloc_vm();
 547
 548         if (!kvm)
 549                 return ERR_PTR(-ENOMEM);
 550
 551         spin_lock_init(&kvm->mmu_lock);
 552         atomic_inc(&current->mm->mm_count);
 553         kvm->mm = current->mm;
 554         kvm_eventfd_init(kvm);
 555         mutex_init(&kvm->lock);
 556         mutex_init(&kvm->irq_lock);
 557         mutex_init(&kvm->slots_lock);
 558         atomic_set(&kvm->users_count, 1);
 559         INIT_LIST_HEAD(&kvm->devices);
 560
 561         r = kvm_arch_init_vm(kvm, type);
 562         if (r)
 563                 goto out_err_no_disable;
 564
 565         r = hardware_enable_all();
 566         if (r)
 567                 goto out_err_no_disable;
 568
 569 #ifdef CONFIG_HAVE_KVM_IRQFD
 570         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 571 #endif
 572
 573         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
 574
 575         r = -ENOMEM;
 576         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
 577                 kvm->memslots[i] = kvm_alloc_memslots();
 578                 if (!kvm->memslots[i])
 579                         goto out_err_no_srcu;
 580         }
 581
 582         if (init_srcu_struct(&kvm->srcu))
 583                 goto out_err_no_srcu;
 584         if (init_srcu_struct(&kvm->irq_srcu))
 585                 goto out_err_no_irq_srcu;
 586         for (i = 0; i < KVM_NR_BUSES; i++) {
 587                 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
 588                                         GFP_KERNEL);
 589                 if (!kvm->buses[i])
 590                         goto out_err;
 591         }
 592
 593         r = kvm_init_mmu_notifier(kvm);
 594         if (r)
 595                 goto out_err;
 596
 597         spin_lock(&kvm_lock);
 598         list_add(&kvm->vm_list, &vm_list);
 599         spin_unlock(&kvm_lock);
 600
 601         preempt_notifier_inc();
 602
 603         return kvm;
 604
 605 out_err:
 606         cleanup_srcu_struct(&kvm->irq_srcu);
 607 out_err_no_irq_srcu:
 608         cleanup_srcu_struct(&kvm->srcu);
 609 out_err_no_srcu:
 610         hardware_disable_all();
 611 out_err_no_disable:
 612         for (i = 0; i < KVM_NR_BUSES; i++)
 613                 kfree(kvm->buses[i]);
 614         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
 615                 kvm_free_memslots(kvm, kvm->memslots[i]);
 616         kvm_arch_free_vm(kvm);
 617         mmdrop(current->mm);
 618         return ERR_PTR(r);
 619 }
 620
 621 /*
 622  * Avoid using vmalloc for a small buffer.
 623  * Should not be used when the size is statically known.
 624  */
 625 void *kvm_kvzalloc(unsigned long size)
 626 {
 627         if (size > PAGE_SIZE)
 628                 return vzalloc(size);
 629         else
 630                 return kzalloc(size, GFP_KERNEL);
 631 }
 632
 633 static void kvm_destroy_devices(struct kvm *kvm)
 634 {
 635         struct list_head *node, *tmp;
 636
 637         list_for_each_safe(node, tmp, &kvm->devices) {
 638                 struct kvm_device *dev =
 639                         list_entry(node, struct kvm_device, vm_node);
 640
 641                 list_del(node);
 642                 dev->ops->destroy(dev);
 643         }
 644 }
 645
 646 static void kvm_destroy_vm(struct kvm *kvm)
 647 {
 648         int i;
 649         struct mm_struct *mm = kvm->mm;
 650
 651         kvm_arch_sync_events(kvm);
 652         spin_lock(&kvm_lock);
 653         list_del(&kvm->vm_list);
 654         spin_unlock(&kvm_lock);
 655         kvm_free_irq_routing(kvm);
 656         for (i = 0; i < KVM_NR_BUSES; i++)
 657                 kvm_io_bus_destroy(kvm->buses[i]);
 658         kvm_coalesced_mmio_free(kvm);
 659 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 660         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 661 #else
 662         kvm_arch_flush_shadow_all(kvm);
 663 #endif
 664         kvm_arch_destroy_vm(kvm);
 665         kvm_destroy_devices(kvm);
 666         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
 667                 kvm_free_memslots(kvm, kvm->memslots[i]);
 668         cleanup_srcu_struct(&kvm->irq_srcu);
 669         cleanup_srcu_struct(&kvm->srcu);
 670         kvm_arch_free_vm(kvm);
 671         preempt_notifier_dec();
 672         hardware_disable_all();
 673         mmdrop(mm);
 674 }
 675
 676 void kvm_get_kvm(struct kvm *kvm)
 677 {
 678         atomic_inc(&kvm->users_count);
 679 }
 680 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 681
 682 void kvm_put_kvm(struct kvm *kvm)
 683 {
 684         if (atomic_dec_and_test(&kvm->users_count))
 685                 kvm_destroy_vm(kvm);
 686 }
 687 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 688
 689
 690 static int kvm_vm_release(struct inode *inode, struct file *filp)
 691 {
 692         struct kvm *kvm = filp->private_data;
 693
 694         kvm_irqfd_release(kvm);
 695
 696         kvm_put_kvm(kvm);
 697         return 0;
 698 }
 699
 700 /*
 701  * Allocation size is twice as large as the actual dirty bitmap size.
 702  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
 703  */
 704 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 705 {
 706         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 707
 708         memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
 709         if (!memslot->dirty_bitmap)
 710                 return -ENOMEM;
 711
 712         return 0;
 713 }
 714
 715 /*
 716  * Insert memslot and re-sort memslots based on their GFN,
 717  * so binary search could be used to lookup GFN.
 718  * Sorting algorithm takes advantage of having initially
 719  * sorted array and known changed memslot position.
 720  */
 721 static void update_memslots(struct kvm_memslots *slots,
 722                             struct kvm_memory_slot *new)
 723 {
 724         int id = new->id;
 725         int i = slots->id_to_index[id];
 726         struct kvm_memory_slot *mslots = slots->memslots;
 727
 728         WARN_ON(mslots[i].id != id);
 729         if (!new->npages) {
 730                 WARN_ON(!mslots[i].npages);
 731                 if (mslots[i].npages)
 732                         slots->used_slots--;
 733         } else {
 734                 if (!mslots[i].npages)
 735                         slots->used_slots++;
 736         }
 737
 738         while (i < KVM_MEM_SLOTS_NUM - 1 &&
 739                new->base_gfn <= mslots[i + 1].base_gfn) {
 740                 if (!mslots[i + 1].npages)
 741                         break;
 742                 mslots[i] = mslots[i + 1];
 743                 slots->id_to_index[mslots[i].id] = i;
 744                 i++;
 745         }
 746
 747         /*
 748          * The ">=" is needed when creating a slot with base_gfn == 0,
 749          * so that it moves before all those with base_gfn == npages == 0.
 750          *
 751          * On the other hand, if new->npages is zero, the above loop has
 752          * already left i pointing to the beginning of the empty part of
 753          * mslots, and the ">=" would move the hole backwards in this
 754          * case---which is wrong.  So skip the loop when deleting a slot.
 755          */
 756         if (new->npages) {
 757                 while (i > 0 &&
 758                        new->base_gfn >= mslots[i - 1].base_gfn) {
 759                         mslots[i] = mslots[i - 1];
 760                         slots->id_to_index[mslots[i].id] = i;
 761                         i--;
 762                 }
 763         } else
 764                 WARN_ON_ONCE(i != slots->used_slots);
 765
 766         mslots[i] = *new;
 767         slots->id_to_index[mslots[i].id] = i;
 768 }
 769
 770 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
 771 {
 772         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
 773
 774 #ifdef __KVM_HAVE_READONLY_MEM
 775         valid_flags |= KVM_MEM_READONLY;
 776 #endif
 777
 778         if (mem->flags & ~valid_flags)
 779                 return -EINVAL;
 780
 781         return 0;
 782 }
 783
 784 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
 785                 int as_id, struct kvm_memslots *slots)
 786 {
 787         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
 788
 789         /*
 790          * Set the low bit in the generation, which disables SPTE caching
 791          * until the end of synchronize_srcu_expedited.
 792          */
 793         WARN_ON(old_memslots->generation & 1);
 794         slots->generation = old_memslots->generation + 1;
 795
 796         rcu_assign_pointer(kvm->memslots[as_id], slots);
 797         synchronize_srcu_expedited(&kvm->srcu);
 798
 799         /*
 800          * Increment the new memslot generation a second time. This prevents
 801          * vm exits that race with memslot updates from caching a memslot
 802          * generation that will (potentially) be valid forever.
 803          */
 804         slots->generation++;
 805
 806         kvm_arch_memslots_updated(kvm, slots);
 807
 808         return old_memslots;
 809 }
 810
 811 /*
 812  * Allocate some memory and give it an address in the guest physical address
 813  * space.
 814  *
 815  * Discontiguous memory is allowed, mostly for framebuffers.
 816  *
 817  * Must be called holding kvm->slots_lock for write.
 818  */
 819 int __kvm_set_memory_region(struct kvm *kvm,
 820                             const struct kvm_userspace_memory_region *mem)
 821 {
 822         int r;
 823         gfn_t base_gfn;
 824         unsigned long npages;
 825         struct kvm_memory_slot *slot;
 826         struct kvm_memory_slot old, new;
 827         struct kvm_memslots *slots = NULL, *old_memslots;
 828         int as_id, id;
 829         enum kvm_mr_change change;
 830
 831         r = check_memory_region_flags(mem);
 832         if (r)
 833                 goto out;
 834
 835         r = -EINVAL;
 836         as_id = mem->slot >> 16;
 837         id = (u16)mem->slot;
 838
 839         /* General sanity checks */
 840         if (mem->memory_size & (PAGE_SIZE - 1))
 841                 goto out;
 842         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 843                 goto out;
 844         /* We can read the guest memory with __xxx_user() later on. */
 845         if ((id < KVM_USER_MEM_SLOTS) &&
 846             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
 847              !access_ok(VERIFY_WRITE,
 848                         (void __user *)(unsigned long)mem->userspace_addr,
 849                         mem->memory_size)))
 850                 goto out;
 851         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
 852                 goto out;
 853         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 854                 goto out;
 855
 856         slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
 857         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 858         npages = mem->memory_size >> PAGE_SHIFT;
 859
 860         if (npages > KVM_MEM_MAX_NR_PAGES)
 861                 goto out;
 862
 863         new = old = *slot;
 864
 865         new.id = id;
 866         new.base_gfn = base_gfn;
 867         new.npages = npages;
 868         new.flags = mem->flags;
 869
 870         if (npages) {
 871                 if (!old.npages)
 872                         change = KVM_MR_CREATE;
 873                 else { /* Modify an existing slot. */
 874                         if ((mem->userspace_addr != old.userspace_addr) ||
 875                             (npages != old.npages) ||
 876                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
 877                                 goto out;
 878
 879                         if (base_gfn != old.base_gfn)
 880                                 change = KVM_MR_MOVE;
 881                         else if (new.flags != old.flags)
 882                                 change = KVM_MR_FLAGS_ONLY;
 883                         else { /* Nothing to change. */
 884                                 r = 0;
 885                                 goto out;
 886                         }
 887                 }
 888         } else {
 889                 if (!old.npages)
 890                         goto out;
 891
 892                 change = KVM_MR_DELETE;
 893                 new.base_gfn = 0;
 894                 new.flags = 0;
 895         }
 896
 897         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 898                 /* Check for overlaps */
 899                 r = -EEXIST;
 900                 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
 901                         if ((slot->id >= KVM_USER_MEM_SLOTS) ||
 902                             (slot->id == id))
 903                                 continue;
 904                         if (!((base_gfn + npages <= slot->base_gfn) ||
 905                               (base_gfn >= slot->base_gfn + slot->npages)))
 906                                 goto out;
 907                 }
 908         }
 909
 910         /* Free page dirty bitmap if unneeded */
 911         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 912                 new.dirty_bitmap = NULL;
 913
 914         r = -ENOMEM;
 915         if (change == KVM_MR_CREATE) {
 916                 new.userspace_addr = mem->userspace_addr;
 917
 918                 if (kvm_arch_create_memslot(kvm, &new, npages))
 919                         goto out_free;
 920         }
 921
 922         /* Allocate page dirty bitmap if needed */
 923         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 924                 if (kvm_create_dirty_bitmap(&new) < 0)
 925                         goto out_free;
 926         }
 927
 928         slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
 929         if (!slots)
 930                 goto out_free;
 931         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
 932
 933         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
 934                 slot = id_to_memslot(slots, id);
 935                 slot->flags |= KVM_MEMSLOT_INVALID;
 936
 937                 old_memslots = install_new_memslots(kvm, as_id, slots);
 938
 939                 /* slot was deleted or moved, clear iommu mapping */
 940                 kvm_iommu_unmap_pages(kvm, &old);
 941                 /* From this point no new shadow pages pointing to a deleted,
 942                  * or moved, memslot will be created.
 943                  *
 944                  * validation of sp->gfn happens in:
 945                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 946                  *      - kvm_is_visible_gfn (mmu_check_roots)
 947                  */
 948                 kvm_arch_flush_shadow_memslot(kvm, slot);
 949
 950                 /*
 951                  * We can re-use the old_memslots from above, the only difference
 952                  * from the currently installed memslots is the invalid flag.  This
 953                  * will get overwritten by update_memslots anyway.
 954                  */
 955                 slots = old_memslots;
 956         }
 957
 958         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
 959         if (r)
 960                 goto out_slots;
 961
 962         /* actual memory is freed via old in kvm_free_memslot below */
 963         if (change == KVM_MR_DELETE) {
 964                 new.dirty_bitmap = NULL;
 965                 memset(&new.arch, 0, sizeof(new.arch));
 966         }
 967
 968         update_memslots(slots, &new);
 969         old_memslots = install_new_memslots(kvm, as_id, slots);
 970
 971         kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
 972
 973         kvm_free_memslot(kvm, &old, &new);
 974         kvfree(old_memslots);
 975
 976         /*
 977          * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
 978          * un-mapped and re-mapped if their base changes.  Since base change
 979          * unmapping is handled above with slot deletion, mapping alone is
 980          * needed here.  Anything else the iommu might care about for existing
 981          * slots (size changes, userspace addr changes and read-only flag
 982          * changes) is disallowed above, so any other attribute changes getting
 983          * here can be skipped.
 984          */
 985         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 986                 r = kvm_iommu_map_pages(kvm, &new);
 987                 return r;
 988         }
 989
 990         return 0;
 991
 992 out_slots:
 993         kvfree(slots);
 994 out_free:
 995         kvm_free_memslot(kvm, &new, &old);
 996 out:
 997         return r;
 998 }
 999 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1000
1001 int kvm_set_memory_region(struct kvm *kvm,
1002                           const struct kvm_userspace_memory_region *mem)
1003 {
1004         int r;
1005
1006         mutex_lock(&kvm->slots_lock);
1007         r = __kvm_set_memory_region(kvm, mem);
1008         mutex_unlock(&kvm->slots_lock);
1009         return r;
1010 }
1011 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1012
1013 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1014                                           struct kvm_userspace_memory_region *mem)
1015 {
1016         if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1017                 return -EINVAL;
1018
1019         return kvm_set_memory_region(kvm, mem);
1020 }
1021
1022 int kvm_get_dirty_log(struct kvm *kvm,
1023                         struct kvm_dirty_log *log, int *is_dirty)
1024 {
1025         struct kvm_memslots *slots;
1026         struct kvm_memory_slot *memslot;
1027         int r, i, as_id, id;
1028         unsigned long n;
1029         unsigned long any = 0;
1030
1031         r = -EINVAL;
1032         as_id = log->slot >> 16;
1033         id = (u16)log->slot;
1034         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1035                 goto out;
1036
1037         slots = __kvm_memslots(kvm, as_id);
1038         memslot = id_to_memslot(slots, id);
1039         r = -ENOENT;
1040         if (!memslot->dirty_bitmap)
1041                 goto out;
1042
1043         n = kvm_dirty_bitmap_bytes(memslot);
1044
1045         for (i = 0; !any && i < n/sizeof(long); ++i)
1046                 any = memslot->dirty_bitmap[i];
1047
1048         r = -EFAULT;
1049         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1050                 goto out;
1051
1052         if (any)
1053                 *is_dirty = 1;
1054
1055         r = 0;
1056 out:
1057         return r;
1058 }
1059 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1060
1061 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1062 /**
1063  * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1064  *      are dirty write protect them for next write.
1065  * @kvm:        pointer to kvm instance
1066  * @log:        slot id and address to which we copy the log
1067  * @is_dirty:   flag set if any page is dirty
1068  *
1069  * We need to keep it in mind that VCPU threads can write to the bitmap
1070  * concurrently. So, to avoid losing track of dirty pages we keep the
1071  * following order:
1072  *
1073  *    1. Take a snapshot of the bit and clear it if needed.
1074  *    2. Write protect the corresponding page.
1075  *    3. Copy the snapshot to the userspace.
1076  *    4. Upon return caller flushes TLB's if needed.
1077  *
1078  * Between 2 and 4, the guest may write to the page using the remaining TLB
1079  * entry.  This is not a problem because the page is reported dirty using
1080  * the snapshot taken before and step 4 ensures that writes done after
1081  * exiting to userspace will be logged for the next call.
1082  *
1083  */
1084 int kvm_get_dirty_log_protect(struct kvm *kvm,
1085                         struct kvm_dirty_log *log, bool *is_dirty)
1086 {
1087         struct kvm_memslots *slots;
1088         struct kvm_memory_slot *memslot;
1089         int r, i, as_id, id;
1090         unsigned long n;
1091         unsigned long *dirty_bitmap;
1092         unsigned long *dirty_bitmap_buffer;
1093
1094         r = -EINVAL;
1095         as_id = log->slot >> 16;
1096         id = (u16)log->slot;
1097         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1098                 goto out;
1099
1100         slots = __kvm_memslots(kvm, as_id);
1101         memslot = id_to_memslot(slots, id);
1102
1103         dirty_bitmap = memslot->dirty_bitmap;
1104         r = -ENOENT;
1105         if (!dirty_bitmap)
1106                 goto out;
1107
1108         n = kvm_dirty_bitmap_bytes(memslot);
1109
1110         dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
1111         memset(dirty_bitmap_buffer, 0, n);
1112
1113         spin_lock(&kvm->mmu_lock);
1114         *is_dirty = false;
1115         for (i = 0; i < n / sizeof(long); i++) {
1116                 unsigned long mask;
1117                 gfn_t offset;
1118
1119                 if (!dirty_bitmap[i])
1120                         continue;
1121
1122                 *is_dirty = true;
1123
1124                 mask = xchg(&dirty_bitmap[i], 0);
1125                 dirty_bitmap_buffer[i] = mask;
1126
1127                 if (mask) {
1128                         offset = i * BITS_PER_LONG;
1129                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1130                                                                 offset, mask);
1131                 }
1132         }
1133
1134         spin_unlock(&kvm->mmu_lock);
1135
1136         r = -EFAULT;
1137         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1138                 goto out;
1139
1140         r = 0;
1141 out:
1142         return r;
1143 }
1144 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1145 #endif
1146
1147 bool kvm_largepages_enabled(void)
1148 {
1149         return largepages_enabled;
1150 }
1151
1152 void kvm_disable_largepages(void)
1153 {
1154         largepages_enabled = false;
1155 }
1156 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1157
1158 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1159 {
1160         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1161 }
1162 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1163
1164 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1165 {
1166         return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1167 }
1168
1169 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1170 {
1171         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1172
1173         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1174               memslot->flags & KVM_MEMSLOT_INVALID)
1175                 return 0;
1176
1177         return 1;
1178 }
1179 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1180
1181 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1182 {
1183         struct vm_area_struct *vma;
1184         unsigned long addr, size;
1185
1186         size = PAGE_SIZE;
1187
1188         addr = gfn_to_hva(kvm, gfn);
1189         if (kvm_is_error_hva(addr))
1190                 return PAGE_SIZE;
1191
1192         down_read(&current->mm->mmap_sem);
1193         vma = find_vma(current->mm, addr);
1194         if (!vma)
1195                 goto out;
1196
1197         size = vma_kernel_pagesize(vma);
1198
1199 out:
1200         up_read(&current->mm->mmap_sem);
1201
1202         return size;
1203 }
1204
1205 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1206 {
1207         return slot->flags & KVM_MEM_READONLY;
1208 }
1209
1210 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1211                                        gfn_t *nr_pages, bool write)
1212 {
1213         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1214                 return KVM_HVA_ERR_BAD;
1215
1216         if (memslot_is_readonly(slot) && write)
1217                 return KVM_HVA_ERR_RO_BAD;
1218
1219         if (nr_pages)
1220                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1221
1222         return __gfn_to_hva_memslot(slot, gfn);
1223 }
1224
1225 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1226                                      gfn_t *nr_pages)
1227 {
1228         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1229 }
1230
1231 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1232                                         gfn_t gfn)
1233 {
1234         return gfn_to_hva_many(slot, gfn, NULL);
1235 }
1236 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1237
1238 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1239 {
1240         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1241 }
1242 EXPORT_SYMBOL_GPL(gfn_to_hva);
1243
1244 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1245 {
1246         return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1247 }
1248 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1249
1250 /*
1251  * If writable is set to false, the hva returned by this function is only
1252  * allowed to be read.
1253  */
1254 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1255                                       gfn_t gfn, bool *writable)
1256 {
1257         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1258
1259         if (!kvm_is_error_hva(hva) && writable)
1260                 *writable = !memslot_is_readonly(slot);
1261
1262         return hva;
1263 }
1264
1265 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1266 {
1267         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1268
1269         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1270 }
1271
1272 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1273 {
1274         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1275
1276         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1277 }
1278
1279 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1280         unsigned long start, int write, struct page **page)
1281 {
1282         int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1283
1284         if (write)
1285                 flags |= FOLL_WRITE;
1286
1287         return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1288 }
1289
1290 static inline int check_user_page_hwpoison(unsigned long addr)
1291 {
1292         int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1293
1294         rc = __get_user_pages(current, current->mm, addr, 1,
1295                               flags, NULL, NULL, NULL);
1296         return rc == -EHWPOISON;
1297 }
1298
1299 /*
1300  * The atomic path to get the writable pfn which will be stored in @pfn,
1301  * true indicates success, otherwise false is returned.
1302  */
1303 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1304                             bool write_fault, bool *writable, pfn_t *pfn)
1305 {
1306         struct page *page[1];
1307         int npages;
1308
1309         if (!(async || atomic))
1310                 return false;
1311
1312         /*
1313          * Fast pin a writable pfn only if it is a write fault request
1314          * or the caller allows to map a writable pfn for a read fault
1315          * request.
1316          */
1317         if (!(write_fault || writable))
1318                 return false;
1319
1320         npages = __get_user_pages_fast(addr, 1, 1, page);
1321         if (npages == 1) {
1322                 *pfn = page_to_pfn(page[0]);
1323
1324                 if (writable)
1325                         *writable = true;
1326                 return true;
1327         }
1328
1329         return false;
1330 }
1331
1332 /*
1333  * The slow path to get the pfn of the specified host virtual address,
1334  * 1 indicates success, -errno is returned if error is detected.
1335  */
1336 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1337                            bool *writable, pfn_t *pfn)
1338 {
1339         struct page *page[1];
1340         int npages = 0;
1341
1342         might_sleep();
1343
1344         if (writable)
1345                 *writable = write_fault;
1346
1347         if (async) {
1348                 down_read(&current->mm->mmap_sem);
1349                 npages = get_user_page_nowait(current, current->mm,
1350                                               addr, write_fault, page);
1351                 up_read(&current->mm->mmap_sem);
1352         } else
1353                 npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
1354                                                    write_fault, 0, page,
1355                                                    FOLL_TOUCH|FOLL_HWPOISON);
1356         if (npages != 1)
1357                 return npages;
1358
1359         /* map read fault as writable if possible */
1360         if (unlikely(!write_fault) && writable) {
1361                 struct page *wpage[1];
1362
1363                 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1364                 if (npages == 1) {
1365                         *writable = true;
1366                         put_page(page[0]);
1367                         page[0] = wpage[0];
1368                 }
1369
1370                 npages = 1;
1371         }
1372         *pfn = page_to_pfn(page[0]);
1373         return npages;
1374 }
1375
1376 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1377 {
1378         if (unlikely(!(vma->vm_flags & VM_READ)))
1379                 return false;
1380
1381         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1382                 return false;
1383
1384         return true;
1385 }
1386
1387 /*
1388  * Pin guest page in memory and return its pfn.
1389  * @addr: host virtual address which maps memory to the guest
1390  * @atomic: whether this function can sleep
1391  * @async: whether this function need to wait IO complete if the
1392  *         host page is not in the memory
1393  * @write_fault: whether we should get a writable host page
1394  * @writable: whether it allows to map a writable host page for !@write_fault
1395  *
1396  * The function will map a writable host page for these two cases:
1397  * 1): @write_fault = true
1398  * 2): @write_fault = false && @writable, @writable will tell the caller
1399  *     whether the mapping is writable.
1400  */
1401 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1402                         bool write_fault, bool *writable)
1403 {
1404         struct vm_area_struct *vma;
1405         pfn_t pfn = 0;
1406         int npages;
1407
1408         /* we can do it either atomically or asynchronously, not both */
1409         BUG_ON(atomic && async);
1410
1411         if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1412                 return pfn;
1413
1414         if (atomic)
1415                 return KVM_PFN_ERR_FAULT;
1416
1417         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1418         if (npages == 1)
1419                 return pfn;
1420
1421         down_read(&current->mm->mmap_sem);
1422         if (npages == -EHWPOISON ||
1423               (!async && check_user_page_hwpoison(addr))) {
1424                 pfn = KVM_PFN_ERR_HWPOISON;
1425                 goto exit;
1426         }
1427
1428         vma = find_vma_intersection(current->mm, addr, addr + 1);
1429
1430         if (vma == NULL)
1431                 pfn = KVM_PFN_ERR_FAULT;
1432         else if ((vma->vm_flags & VM_PFNMAP)) {
1433                 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1434                         vma->vm_pgoff;
1435                 BUG_ON(!kvm_is_reserved_pfn(pfn));
1436         } else {
1437                 if (async && vma_is_valid(vma, write_fault))
1438                         *async = true;
1439                 pfn = KVM_PFN_ERR_FAULT;
1440         }
1441 exit:
1442         up_read(&current->mm->mmap_sem);
1443         return pfn;
1444 }
1445
1446 pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
1447                            bool *async, bool write_fault, bool *writable)
1448 {
1449         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1450
1451         if (addr == KVM_HVA_ERR_RO_BAD)
1452                 return KVM_PFN_ERR_RO_FAULT;
1453
1454         if (kvm_is_error_hva(addr))
1455                 return KVM_PFN_NOSLOT;
1456
1457         /* Do not map writable pfn in the readonly memslot. */
1458         if (writable && memslot_is_readonly(slot)) {
1459                 *writable = false;
1460                 writable = NULL;
1461         }
1462
1463         return hva_to_pfn(addr, atomic, async, write_fault,
1464                           writable);
1465 }
1466 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1467
1468 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1469                       bool *writable)
1470 {
1471         return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1472                                     write_fault, writable);
1473 }
1474 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1475
1476 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1477 {
1478         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1479 }
1480 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1481
1482 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1483 {
1484         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1485 }
1486 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1487
1488 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1489 {
1490         return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1491 }
1492 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1493
1494 pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1495 {
1496         return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1497 }
1498 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1499
1500 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1501 {
1502         return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1503 }
1504 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1505
1506 pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1507 {
1508         return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1509 }
1510 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1511
1512 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1513                             struct page **pages, int nr_pages)
1514 {
1515         unsigned long addr;
1516         gfn_t entry;
1517
1518         addr = gfn_to_hva_many(slot, gfn, &entry);
1519         if (kvm_is_error_hva(addr))
1520                 return -1;
1521
1522         if (entry < nr_pages)
1523                 return 0;
1524
1525         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1526 }
1527 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1528
1529 static struct page *kvm_pfn_to_page(pfn_t pfn)
1530 {
1531         if (is_error_noslot_pfn(pfn))
1532                 return KVM_ERR_PTR_BAD_PAGE;
1533
1534         if (kvm_is_reserved_pfn(pfn)) {
1535                 WARN_ON(1);
1536                 return KVM_ERR_PTR_BAD_PAGE;
1537         }
1538
1539         return pfn_to_page(pfn);
1540 }
1541
1542 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1543 {
1544         pfn_t pfn;
1545
1546         pfn = gfn_to_pfn(kvm, gfn);
1547
1548         return kvm_pfn_to_page(pfn);
1549 }
1550 EXPORT_SYMBOL_GPL(gfn_to_page);
1551
1552 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1553 {
1554         pfn_t pfn;
1555
1556         pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1557
1558         return kvm_pfn_to_page(pfn);
1559 }
1560 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1561
1562 void kvm_release_page_clean(struct page *page)
1563 {
1564         WARN_ON(is_error_page(page));
1565
1566         kvm_release_pfn_clean(page_to_pfn(page));
1567 }
1568 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1569
1570 void kvm_release_pfn_clean(pfn_t pfn)
1571 {
1572         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1573                 put_page(pfn_to_page(pfn));
1574 }
1575 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1576
1577 void kvm_release_page_dirty(struct page *page)
1578 {
1579         WARN_ON(is_error_page(page));
1580
1581         kvm_release_pfn_dirty(page_to_pfn(page));
1582 }
1583 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1584
1585 static void kvm_release_pfn_dirty(pfn_t pfn)
1586 {
1587         kvm_set_pfn_dirty(pfn);
1588         kvm_release_pfn_clean(pfn);
1589 }
1590
1591 void kvm_set_pfn_dirty(pfn_t pfn)
1592 {
1593         if (!kvm_is_reserved_pfn(pfn)) {
1594                 struct page *page = pfn_to_page(pfn);
1595
1596                 if (!PageReserved(page))
1597                         SetPageDirty(page);
1598         }
1599 }
1600 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1601
1602 void kvm_set_pfn_accessed(pfn_t pfn)
1603 {
1604         if (!kvm_is_reserved_pfn(pfn))
1605                 mark_page_accessed(pfn_to_page(pfn));
1606 }
1607 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1608
1609 void kvm_get_pfn(pfn_t pfn)
1610 {
1611         if (!kvm_is_reserved_pfn(pfn))
1612                 get_page(pfn_to_page(pfn));
1613 }
1614 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1615
1616 static int next_segment(unsigned long len, int offset)
1617 {
1618         if (len > PAGE_SIZE - offset)
1619                 return PAGE_SIZE - offset;
1620         else
1621                 return len;
1622 }
1623
1624 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1625                                  void *data, int offset, int len)
1626 {
1627         int r;
1628         unsigned long addr;
1629
1630         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1631         if (kvm_is_error_hva(addr))
1632                 return -EFAULT;
1633         r = __copy_from_user(data, (void __user *)addr + offset, len);
1634         if (r)
1635                 return -EFAULT;
1636         return 0;
1637 }
1638
1639 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1640                         int len)
1641 {
1642         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1643
1644         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1645 }
1646 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1647
1648 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1649                              int offset, int len)
1650 {
1651         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1652
1653         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1654 }
1655 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1656
1657 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1658 {
1659         gfn_t gfn = gpa >> PAGE_SHIFT;
1660         int seg;
1661         int offset = offset_in_page(gpa);
1662         int ret;
1663
1664         while ((seg = next_segment(len, offset)) != 0) {
1665                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1666                 if (ret < 0)
1667                         return ret;
1668                 offset = 0;
1669                 len -= seg;
1670                 data += seg;
1671                 ++gfn;
1672         }
1673         return 0;
1674 }
1675 EXPORT_SYMBOL_GPL(kvm_read_guest);
1676
1677 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1678 {
1679         gfn_t gfn = gpa >> PAGE_SHIFT;
1680         int seg;
1681         int offset = offset_in_page(gpa);
1682         int ret;
1683
1684         while ((seg = next_segment(len, offset)) != 0) {
1685                 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1686                 if (ret < 0)
1687                         return ret;
1688                 offset = 0;
1689                 len -= seg;
1690                 data += seg;
1691                 ++gfn;
1692         }
1693         return 0;
1694 }
1695 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1696
1697 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1698                                    void *data, int offset, unsigned long len)
1699 {
1700         int r;
1701         unsigned long addr;
1702
1703         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1704         if (kvm_is_error_hva(addr))
1705                 return -EFAULT;
1706         pagefault_disable();
1707         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1708         pagefault_enable();
1709         if (r)
1710                 return -EFAULT;
1711         return 0;
1712 }
1713
1714 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1715                           unsigned long len)
1716 {
1717         gfn_t gfn = gpa >> PAGE_SHIFT;
1718         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1719         int offset = offset_in_page(gpa);
1720
1721         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1722 }
1723 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1724
1725 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1726                                void *data, unsigned long len)
1727 {
1728         gfn_t gfn = gpa >> PAGE_SHIFT;
1729         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1730         int offset = offset_in_page(gpa);
1731
1732         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1733 }
1734 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1735
1736 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1737                                   const void *data, int offset, int len)
1738 {
1739         int r;
1740         unsigned long addr;
1741
1742         addr = gfn_to_hva_memslot(memslot, gfn);
1743         if (kvm_is_error_hva(addr))
1744                 return -EFAULT;
1745         r = __copy_to_user((void __user *)addr + offset, data, len);
1746         if (r)
1747                 return -EFAULT;
1748         mark_page_dirty_in_slot(memslot, gfn);
1749         return 0;
1750 }
1751
1752 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1753                          const void *data, int offset, int len)
1754 {
1755         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1756
1757         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1758 }
1759 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1760
1761 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1762                               const void *data, int offset, int len)
1763 {
1764         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1765
1766         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1767 }
1768 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1769
1770 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1771                     unsigned long len)
1772 {
1773         gfn_t gfn = gpa >> PAGE_SHIFT;
1774         int seg;
1775         int offset = offset_in_page(gpa);
1776         int ret;
1777
1778         while ((seg = next_segment(len, offset)) != 0) {
1779                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1780                 if (ret < 0)
1781                         return ret;
1782                 offset = 0;
1783                 len -= seg;
1784                 data += seg;
1785                 ++gfn;
1786         }
1787         return 0;
1788 }
1789 EXPORT_SYMBOL_GPL(kvm_write_guest);
1790
1791 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1792                          unsigned long len)
1793 {
1794         gfn_t gfn = gpa >> PAGE_SHIFT;
1795         int seg;
1796         int offset = offset_in_page(gpa);
1797         int ret;
1798
1799         while ((seg = next_segment(len, offset)) != 0) {
1800                 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1801                 if (ret < 0)
1802                         return ret;
1803                 offset = 0;
1804                 len -= seg;
1805                 data += seg;
1806                 ++gfn;
1807         }
1808         return 0;
1809 }
1810 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1811
1812 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1813                               gpa_t gpa, unsigned long len)
1814 {
1815         struct kvm_memslots *slots = kvm_memslots(kvm);
1816         int offset = offset_in_page(gpa);
1817         gfn_t start_gfn = gpa >> PAGE_SHIFT;
1818         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1819         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1820         gfn_t nr_pages_avail;
1821
1822         ghc->gpa = gpa;
1823         ghc->generation = slots->generation;
1824         ghc->len = len;
1825         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1826         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1827         if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1828                 ghc->hva += offset;
1829         } else {
1830                 /*
1831                  * If the requested region crosses two memslots, we still
1832                  * verify that the entire region is valid here.
1833                  */
1834                 while (start_gfn <= end_gfn) {
1835                         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1836                         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1837                                                    &nr_pages_avail);
1838                         if (kvm_is_error_hva(ghc->hva))
1839                                 return -EFAULT;
1840                         start_gfn += nr_pages_avail;
1841                 }
1842                 /* Use the slow path for cross page reads and writes. */
1843                 ghc->memslot = NULL;
1844         }
1845         return 0;
1846 }
1847 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1848
1849 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1850                            void *data, unsigned long len)
1851 {
1852         struct kvm_memslots *slots = kvm_memslots(kvm);
1853         int r;
1854
1855         BUG_ON(len > ghc->len);
1856
1857         if (slots->generation != ghc->generation)
1858                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1859
1860         if (unlikely(!ghc->memslot))
1861                 return kvm_write_guest(kvm, ghc->gpa, data, len);
1862
1863         if (kvm_is_error_hva(ghc->hva))
1864                 return -EFAULT;
1865
1866         r = __copy_to_user((void __user *)ghc->hva, data, len);
1867         if (r)
1868                 return -EFAULT;
1869         mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1870
1871         return 0;
1872 }
1873 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1874
1875 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1876                            void *data, unsigned long len)
1877 {
1878         struct kvm_memslots *slots = kvm_memslots(kvm);
1879         int r;
1880
1881         BUG_ON(len > ghc->len);
1882
1883         if (slots->generation != ghc->generation)
1884                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1885
1886         if (unlikely(!ghc->memslot))
1887                 return kvm_read_guest(kvm, ghc->gpa, data, len);
1888
1889         if (kvm_is_error_hva(ghc->hva))
1890                 return -EFAULT;
1891
1892         r = __copy_from_user(data, (void __user *)ghc->hva, len);
1893         if (r)
1894                 return -EFAULT;
1895
1896         return 0;
1897 }
1898 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1899
1900 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1901 {
1902         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
1903
1904         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1905 }
1906 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1907
1908 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1909 {
1910         gfn_t gfn = gpa >> PAGE_SHIFT;
1911         int seg;
1912         int offset = offset_in_page(gpa);
1913         int ret;
1914
1915         while ((seg = next_segment(len, offset)) != 0) {
1916                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1917                 if (ret < 0)
1918                         return ret;
1919                 offset = 0;
1920                 len -= seg;
1921                 ++gfn;
1922         }
1923         return 0;
1924 }
1925 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1926
1927 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
1928                                     gfn_t gfn)
1929 {
1930         if (memslot && memslot->dirty_bitmap) {
1931                 unsigned long rel_gfn = gfn - memslot->base_gfn;
1932
1933                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
1934         }
1935 }
1936
1937 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1938 {
1939         struct kvm_memory_slot *memslot;
1940
1941         memslot = gfn_to_memslot(kvm, gfn);
1942         mark_page_dirty_in_slot(memslot, gfn);
1943 }
1944 EXPORT_SYMBOL_GPL(mark_page_dirty);
1945
1946 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
1947 {
1948         struct kvm_memory_slot *memslot;
1949
1950         memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1951         mark_page_dirty_in_slot(memslot, gfn);
1952 }
1953 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
1954
1955 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
1956 {
1957         int old, val;
1958
1959         old = val = vcpu->halt_poll_ns;
1960         /* 10us base */
1961         if (val == 0 && halt_poll_ns_grow)
1962                 val = 10000;
1963         else
1964                 val *= halt_poll_ns_grow;
1965
1966         if (val > halt_poll_ns)
1967                 val = halt_poll_ns;
1968
1969         vcpu->halt_poll_ns = val;
1970         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
1971 }
1972
1973 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
1974 {
1975         int old, val;
1976
1977         old = val = vcpu->halt_poll_ns;
1978         if (halt_poll_ns_shrink == 0)
1979                 val = 0;
1980         else
1981                 val /= halt_poll_ns_shrink;
1982
1983         vcpu->halt_poll_ns = val;
1984         trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
1985 }
1986
1987 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
1988 {
1989         if (kvm_arch_vcpu_runnable(vcpu)) {
1990                 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1991                 return -EINTR;
1992         }
1993         if (kvm_cpu_has_pending_timer(vcpu))
1994                 return -EINTR;
1995         if (signal_pending(current))
1996                 return -EINTR;
1997
1998         return 0;
1999 }
2000
2001 /*
2002  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2003  */
2004 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2005 {
2006         ktime_t start, cur;
2007         DECLARE_SWAITQUEUE(wait);
2008         bool waited = false;
2009         u64 block_ns;
2010
2011         start = cur = ktime_get();
2012         if (vcpu->halt_poll_ns) {
2013                 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2014
2015                 ++vcpu->stat.halt_attempted_poll;
2016                 do {
2017                         /*
2018                          * This sets KVM_REQ_UNHALT if an interrupt
2019                          * arrives.
2020                          */
2021                         if (kvm_vcpu_check_block(vcpu) < 0) {
2022                                 ++vcpu->stat.halt_successful_poll;
2023                                 goto out;
2024                         }
2025                         cur = ktime_get();
2026                 } while (single_task_running() && ktime_before(cur, stop));
2027         }
2028
2029         kvm_arch_vcpu_blocking(vcpu);
2030
2031         for (;;) {
2032                 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2033
2034                 if (kvm_vcpu_check_block(vcpu) < 0)
2035                         break;
2036
2037                 waited = true;
2038                 schedule();
2039         }
2040
2041         finish_swait(&vcpu->wq, &wait);
2042         cur = ktime_get();
2043
2044         kvm_arch_vcpu_unblocking(vcpu);
2045 out:
2046         block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2047
2048         if (halt_poll_ns) {
2049                 if (block_ns <= vcpu->halt_poll_ns)
2050                         ;
2051                 /* we had a long block, shrink polling */
2052                 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2053                         shrink_halt_poll_ns(vcpu);
2054                 /* we had a short halt and our poll time is too small */
2055                 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2056                         block_ns < halt_poll_ns)
2057                         grow_halt_poll_ns(vcpu);
2058         } else
2059                 vcpu->halt_poll_ns = 0;
2060
2061         trace_kvm_vcpu_wakeup(block_ns, waited);
2062 }
2063 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2064
2065 #ifndef CONFIG_S390
2066 /*
2067  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2068  */
2069 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2070 {
2071         int me;
2072         int cpu = vcpu->cpu;
2073         struct swait_queue_head *wqp;
2074
2075         wqp = kvm_arch_vcpu_wq(vcpu);
2076         if (swait_active(wqp)) {
2077                 swake_up(wqp);
2078                 ++vcpu->stat.halt_wakeup;
2079         }
2080
2081         me = get_cpu();
2082         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2083                 if (kvm_arch_vcpu_should_kick(vcpu))
2084                         smp_send_reschedule(cpu);
2085         put_cpu();
2086 }
2087 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2088 #endif /* !CONFIG_S390 */
2089
2090 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2091 {
2092         struct pid *pid;
2093         struct task_struct *task = NULL;
2094         int ret = 0;
2095
2096         rcu_read_lock();
2097         pid = rcu_dereference(target->pid);
2098         if (pid)
2099                 task = get_pid_task(pid, PIDTYPE_PID);
2100         rcu_read_unlock();
2101         if (!task)
2102                 return ret;
2103         ret = yield_to(task, 1);
2104         put_task_struct(task);
2105
2106         return ret;
2107 }
2108 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2109
2110 /*
2111  * Helper that checks whether a VCPU is eligible for directed yield.
2112  * Most eligible candidate to yield is decided by following heuristics:
2113  *
2114  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2115  *  (preempted lock holder), indicated by @in_spin_loop.
2116  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2117  *
2118  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2119  *  chance last time (mostly it has become eligible now since we have probably
2120  *  yielded to lockholder in last iteration. This is done by toggling
2121  *  @dy_eligible each time a VCPU checked for eligibility.)
2122  *
2123  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2124  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2125  *  burning. Giving priority for a potential lock-holder increases lock
2126  *  progress.
2127  *
2128  *  Since algorithm is based on heuristics, accessing another VCPU data without
2129  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2130  *  and continue with next VCPU and so on.
2131  */
2132 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2133 {
2134 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2135         bool eligible;
2136
2137         eligible = !vcpu->spin_loop.in_spin_loop ||
2138                     vcpu->spin_loop.dy_eligible;
2139
2140         if (vcpu->spin_loop.in_spin_loop)
2141                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2142
2143         return eligible;
2144 #else
2145         return true;
2146 #endif
2147 }
2148
2149 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
2150 {
2151         struct kvm *kvm = me->kvm;
2152         struct kvm_vcpu *vcpu;
2153         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2154         int yielded = 0;
2155         int try = 3;
2156         int pass;
2157         int i;
2158
2159         kvm_vcpu_set_in_spin_loop(me, true);
2160         /*
2161          * We boost the priority of a VCPU that is runnable but not
2162          * currently running, because it got preempted by something
2163          * else and called schedule in __vcpu_run.  Hopefully that
2164          * VCPU is holding the lock that we need and will release it.
2165          * We approximate round-robin by starting at the last boosted VCPU.
2166          */
2167         for (pass = 0; pass < 2 && !yielded && try; pass++) {
2168                 kvm_for_each_vcpu(i, vcpu, kvm) {
2169                         if (!pass && i <= last_boosted_vcpu) {
2170                                 i = last_boosted_vcpu;
2171                                 continue;
2172                         } else if (pass && i > last_boosted_vcpu)
2173                                 break;
2174                         if (!ACCESS_ONCE(vcpu->preempted))
2175                                 continue;
2176                         if (vcpu == me)
2177                                 continue;
2178                         if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2179                                 continue;
2180                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2181                                 continue;
2182
2183                         yielded = kvm_vcpu_yield_to(vcpu);
2184                         if (yielded > 0) {
2185                                 kvm->last_boosted_vcpu = i;
2186                                 break;
2187                         } else if (yielded < 0) {
2188                                 try--;
2189                                 if (!try)
2190                                         break;
2191                         }
2192                 }
2193         }
2194         kvm_vcpu_set_in_spin_loop(me, false);
2195
2196         /* Ensure vcpu is not eligible during next spinloop */
2197         kvm_vcpu_set_dy_eligible(me, false);
2198 }
2199 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2200
2201 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2202 {
2203         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2204         struct page *page;
2205
2206         if (vmf->pgoff == 0)
2207                 page = virt_to_page(vcpu->run);
2208 #ifdef CONFIG_X86
2209         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2210                 page = virt_to_page(vcpu->arch.pio_data);
2211 #endif
2212 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2213         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2214                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2215 #endif
2216         else
2217                 return kvm_arch_vcpu_fault(vcpu, vmf);
2218         get_page(page);
2219         vmf->page = page;
2220         return 0;
2221 }
2222
2223 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2224         .fault = kvm_vcpu_fault,
2225 };
2226
2227 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2228 {
2229         vma->vm_ops = &kvm_vcpu_vm_ops;
2230         return 0;
2231 }
2232
2233 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2234 {
2235         struct kvm_vcpu *vcpu = filp->private_data;
2236
2237         kvm_put_kvm(vcpu->kvm);
2238         return 0;
2239 }
2240
2241 static struct file_operations kvm_vcpu_fops = {
2242         .release        = kvm_vcpu_release,
2243         .unlocked_ioctl = kvm_vcpu_ioctl,
2244 #ifdef CONFIG_KVM_COMPAT
2245         .compat_ioctl   = kvm_vcpu_compat_ioctl,
2246 #endif
2247         .mmap           = kvm_vcpu_mmap,
2248         .llseek         = noop_llseek,
2249 };
2250
2251 /*
2252  * Allocates an inode for the vcpu.
2253  */
2254 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2255 {
2256         return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2257 }
2258
2259 /*
2260  * Creates some virtual cpus.  Good luck creating more than one.
2261  */
2262 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2263 {
2264         int r;
2265         struct kvm_vcpu *vcpu, *v;
2266
2267         if (id >= KVM_MAX_VCPUS)
2268                 return -EINVAL;
2269
2270         vcpu = kvm_arch_vcpu_create(kvm, id);
2271         if (IS_ERR(vcpu))
2272                 return PTR_ERR(vcpu);
2273
2274         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2275
2276         r = kvm_arch_vcpu_setup(vcpu);
2277         if (r)
2278                 goto vcpu_destroy;
2279
2280         mutex_lock(&kvm->lock);
2281         if (!kvm_vcpu_compatible(vcpu)) {
2282                 r = -EINVAL;
2283                 goto unlock_vcpu_destroy;
2284         }
2285         if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
2286                 r = -EINVAL;
2287                 goto unlock_vcpu_destroy;
2288         }
2289
2290         kvm_for_each_vcpu(r, v, kvm)
2291                 if (v->vcpu_id == id) {
2292                         r = -EEXIST;
2293                         goto unlock_vcpu_destroy;
2294                 }
2295
2296         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2297
2298         /* Now it's all set up, let userspace reach it */
2299         kvm_get_kvm(kvm);
2300         r = create_vcpu_fd(vcpu);
2301         if (r < 0) {
2302                 kvm_put_kvm(kvm);
2303                 goto unlock_vcpu_destroy;
2304         }
2305
2306         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2307
2308         /*
2309          * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2310          * before kvm->online_vcpu's incremented value.
2311          */
2312         smp_wmb();
2313         atomic_inc(&kvm->online_vcpus);
2314
2315         mutex_unlock(&kvm->lock);
2316         kvm_arch_vcpu_postcreate(vcpu);
2317         return r;
2318
2319 unlock_vcpu_destroy:
2320         mutex_unlock(&kvm->lock);
2321 vcpu_destroy:
2322         kvm_arch_vcpu_destroy(vcpu);
2323         return r;
2324 }
2325
2326 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2327 {
2328         if (sigset) {
2329                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2330                 vcpu->sigset_active = 1;
2331                 vcpu->sigset = *sigset;
2332         } else
2333                 vcpu->sigset_active = 0;
2334         return 0;
2335 }
2336
2337 static long kvm_vcpu_ioctl(struct file *filp,
2338                            unsigned int ioctl, unsigned long arg)
2339 {
2340         struct kvm_vcpu *vcpu = filp->private_data;
2341         void __user *argp = (void __user *)arg;
2342         int r;
2343         struct kvm_fpu *fpu = NULL;
2344         struct kvm_sregs *kvm_sregs = NULL;
2345
2346         if (vcpu->kvm->mm != current->mm)
2347                 return -EIO;
2348
2349         if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2350                 return -EINVAL;
2351
2352 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2353         /*
2354          * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
2355          * so vcpu_load() would break it.
2356          */
2357         if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2358                 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2359 #endif
2360
2361
2362         r = vcpu_load(vcpu);
2363         if (r)
2364                 return r;
2365         switch (ioctl) {
2366         case KVM_RUN:
2367                 r = -EINVAL;
2368                 if (arg)
2369                         goto out;
2370                 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
2371                         /* The thread running this VCPU changed. */
2372                         struct pid *oldpid = vcpu->pid;
2373                         struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2374
2375                         rcu_assign_pointer(vcpu->pid, newpid);
2376                         if (oldpid)
2377                                 synchronize_rcu();
2378                         put_pid(oldpid);
2379                 }
2380                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2381                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2382                 break;
2383         case KVM_GET_REGS: {
2384                 struct kvm_regs *kvm_regs;
2385
2386                 r = -ENOMEM;
2387                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2388                 if (!kvm_regs)
2389                         goto out;
2390                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2391                 if (r)
2392                         goto out_free1;
2393                 r = -EFAULT;
2394                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2395                         goto out_free1;
2396                 r = 0;
2397 out_free1:
2398                 kfree(kvm_regs);
2399                 break;
2400         }
2401         case KVM_SET_REGS: {
2402                 struct kvm_regs *kvm_regs;
2403
2404                 r = -ENOMEM;
2405                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2406                 if (IS_ERR(kvm_regs)) {
2407                         r = PTR_ERR(kvm_regs);
2408                         goto out;
2409                 }
2410                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2411                 kfree(kvm_regs);
2412                 break;
2413         }
2414         case KVM_GET_SREGS: {
2415                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2416                 r = -ENOMEM;
2417                 if (!kvm_sregs)
2418                         goto out;
2419                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2420                 if (r)
2421                         goto out;
2422                 r = -EFAULT;
2423                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2424                         goto out;
2425                 r = 0;
2426                 break;
2427         }
2428         case KVM_SET_SREGS: {
2429                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2430                 if (IS_ERR(kvm_sregs)) {
2431                         r = PTR_ERR(kvm_sregs);
2432                         kvm_sregs = NULL;
2433                         goto out;
2434                 }
2435                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2436                 break;
2437         }
2438         case KVM_GET_MP_STATE: {
2439                 struct kvm_mp_state mp_state;
2440
2441                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2442                 if (r)
2443                         goto out;
2444                 r = -EFAULT;
2445                 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2446                         goto out;
2447                 r = 0;
2448                 break;
2449         }
2450         case KVM_SET_MP_STATE: {
2451                 struct kvm_mp_state mp_state;
2452
2453                 r = -EFAULT;
2454                 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2455                         goto out;
2456                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2457                 break;
2458         }
2459         case KVM_TRANSLATE: {
2460                 struct kvm_translation tr;
2461
2462                 r = -EFAULT;
2463                 if (copy_from_user(&tr, argp, sizeof(tr)))
2464                         goto out;
2465                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2466                 if (r)
2467                         goto out;
2468                 r = -EFAULT;
2469                 if (copy_to_user(argp, &tr, sizeof(tr)))
2470                         goto out;
2471                 r = 0;
2472                 break;
2473         }
2474         case KVM_SET_GUEST_DEBUG: {
2475                 struct kvm_guest_debug dbg;
2476
2477                 r = -EFAULT;
2478                 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2479                         goto out;
2480                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2481                 break;
2482         }
2483         case KVM_SET_SIGNAL_MASK: {
2484                 struct kvm_signal_mask __user *sigmask_arg = argp;
2485                 struct kvm_signal_mask kvm_sigmask;
2486                 sigset_t sigset, *p;
2487
2488                 p = NULL;
2489                 if (argp) {
2490                         r = -EFAULT;
2491                         if (copy_from_user(&kvm_sigmask, argp,
2492                                            sizeof(kvm_sigmask)))
2493                                 goto out;
2494                         r = -EINVAL;
2495                         if (kvm_sigmask.len != sizeof(sigset))
2496                                 goto out;
2497                         r = -EFAULT;
2498                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2499                                            sizeof(sigset)))
2500                                 goto out;
2501                         p = &sigset;
2502                 }
2503                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2504                 break;
2505         }
2506         case KVM_GET_FPU: {
2507                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2508                 r = -ENOMEM;
2509                 if (!fpu)
2510                         goto out;
2511                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2512                 if (r)
2513                         goto out;
2514                 r = -EFAULT;
2515                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2516                         goto out;
2517                 r = 0;
2518                 break;
2519         }
2520         case KVM_SET_FPU: {
2521                 fpu = memdup_user(argp, sizeof(*fpu));
2522                 if (IS_ERR(fpu)) {
2523                         r = PTR_ERR(fpu);
2524                         fpu = NULL;
2525                         goto out;
2526                 }
2527                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2528                 break;
2529         }
2530         default:
2531                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2532         }
2533 out:
2534         vcpu_put(vcpu);
2535         kfree(fpu);
2536         kfree(kvm_sregs);
2537         return r;
2538 }
2539
2540 #ifdef CONFIG_KVM_COMPAT
2541 static long kvm_vcpu_compat_ioctl(struct file *filp,
2542                                   unsigned int ioctl, unsigned long arg)
2543 {
2544         struct kvm_vcpu *vcpu = filp->private_data;
2545         void __user *argp = compat_ptr(arg);
2546         int r;
2547
2548         if (vcpu->kvm->mm != current->mm)
2549                 return -EIO;
2550
2551         switch (ioctl) {
2552         case KVM_SET_SIGNAL_MASK: {
2553                 struct kvm_signal_mask __user *sigmask_arg = argp;
2554                 struct kvm_signal_mask kvm_sigmask;
2555                 compat_sigset_t csigset;
2556                 sigset_t sigset;
2557
2558                 if (argp) {
2559                         r = -EFAULT;
2560                         if (copy_from_user(&kvm_sigmask, argp,
2561                                            sizeof(kvm_sigmask)))
2562                                 goto out;
2563                         r = -EINVAL;
2564                         if (kvm_sigmask.len != sizeof(csigset))
2565                                 goto out;
2566                         r = -EFAULT;
2567                         if (copy_from_user(&csigset, sigmask_arg->sigset,
2568                                            sizeof(csigset)))
2569                                 goto out;
2570                         sigset_from_compat(&sigset, &csigset);
2571                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2572                 } else
2573                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2574                 break;
2575         }
2576         default:
2577                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2578         }
2579
2580 out:
2581         return r;
2582 }
2583 #endif
2584
2585 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2586                                  int (*accessor)(struct kvm_device *dev,
2587                                                  struct kvm_device_attr *attr),
2588                                  unsigned long arg)
2589 {
2590         struct kvm_device_attr attr;
2591
2592         if (!accessor)
2593                 return -EPERM;
2594
2595         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2596                 return -EFAULT;
2597
2598         return accessor(dev, &attr);
2599 }
2600
2601 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2602                              unsigned long arg)
2603 {
2604         struct kvm_device *dev = filp->private_data;
2605
2606         switch (ioctl) {
2607         case KVM_SET_DEVICE_ATTR:
2608                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2609         case KVM_GET_DEVICE_ATTR:
2610                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2611         case KVM_HAS_DEVICE_ATTR:
2612                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2613         default:
2614                 if (dev->ops->ioctl)
2615                         return dev->ops->ioctl(dev, ioctl, arg);
2616
2617                 return -ENOTTY;
2618         }
2619 }
2620
2621 static int kvm_device_release(struct inode *inode, struct file *filp)
2622 {
2623         struct kvm_device *dev = filp->private_data;
2624         struct kvm *kvm = dev->kvm;
2625
2626         kvm_put_kvm(kvm);
2627         return 0;
2628 }
2629
2630 static const struct file_operations kvm_device_fops = {
2631         .unlocked_ioctl = kvm_device_ioctl,
2632 #ifdef CONFIG_KVM_COMPAT
2633         .compat_ioctl = kvm_device_ioctl,
2634 #endif
2635         .release = kvm_device_release,
2636 };
2637
2638 struct kvm_device *kvm_device_from_filp(struct file *filp)
2639 {
2640         if (filp->f_op != &kvm_device_fops)
2641                 return NULL;
2642
2643         return filp->private_data;
2644 }
2645
2646 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2647 #ifdef CONFIG_KVM_MPIC
2648         [KVM_DEV_TYPE_FSL_MPIC_20]      = &kvm_mpic_ops,
2649         [KVM_DEV_TYPE_FSL_MPIC_42]      = &kvm_mpic_ops,
2650 #endif
2651
2652 #ifdef CONFIG_KVM_XICS
2653         [KVM_DEV_TYPE_XICS]             = &kvm_xics_ops,
2654 #endif
2655 };
2656
2657 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2658 {
2659         if (type >= ARRAY_SIZE(kvm_device_ops_table))
2660                 return -ENOSPC;
2661
2662         if (kvm_device_ops_table[type] != NULL)
2663                 return -EEXIST;
2664
2665         kvm_device_ops_table[type] = ops;
2666         return 0;
2667 }
2668
2669 void kvm_unregister_device_ops(u32 type)
2670 {
2671         if (kvm_device_ops_table[type] != NULL)
2672                 kvm_device_ops_table[type] = NULL;
2673 }
2674
2675 static int kvm_ioctl_create_device(struct kvm *kvm,
2676                                    struct kvm_create_device *cd)
2677 {
2678         struct kvm_device_ops *ops = NULL;
2679         struct kvm_device *dev;
2680         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2681         int ret;
2682
2683         if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2684                 return -ENODEV;
2685
2686         ops = kvm_device_ops_table[cd->type];
2687         if (ops == NULL)
2688                 return -ENODEV;
2689
2690         if (test)
2691                 return 0;
2692
2693         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2694         if (!dev)
2695                 return -ENOMEM;
2696
2697         dev->ops = ops;
2698         dev->kvm = kvm;
2699
2700         ret = ops->create(dev, cd->type);
2701         if (ret < 0) {
2702                 kfree(dev);
2703                 return ret;
2704         }
2705
2706         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2707         if (ret < 0) {
2708                 ops->destroy(dev);
2709                 return ret;
2710         }
2711
2712         list_add(&dev->vm_node, &kvm->devices);
2713         kvm_get_kvm(kvm);
2714         cd->fd = ret;
2715         return 0;
2716 }
2717
2718 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2719 {
2720         switch (arg) {
2721         case KVM_CAP_USER_MEMORY:
2722         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2723         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2724         case KVM_CAP_INTERNAL_ERROR_DATA:
2725 #ifdef CONFIG_HAVE_KVM_MSI
2726         case KVM_CAP_SIGNAL_MSI:
2727 #endif
2728 #ifdef CONFIG_HAVE_KVM_IRQFD
2729         case KVM_CAP_IRQFD:
2730         case KVM_CAP_IRQFD_RESAMPLE:
2731 #endif
2732         case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2733         case KVM_CAP_CHECK_EXTENSION_VM:
2734                 return 1;
2735 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2736         case KVM_CAP_IRQ_ROUTING:
2737                 return KVM_MAX_IRQ_ROUTES;
2738 #endif
2739 #if KVM_ADDRESS_SPACE_NUM > 1
2740         case KVM_CAP_MULTI_ADDRESS_SPACE:
2741                 return KVM_ADDRESS_SPACE_NUM;
2742 #endif
2743         default:
2744                 break;
2745         }
2746         return kvm_vm_ioctl_check_extension(kvm, arg);
2747 }
2748
2749 static long kvm_vm_ioctl(struct file *filp,
2750                            unsigned int ioctl, unsigned long arg)
2751 {
2752         struct kvm *kvm = filp->private_data;
2753         void __user *argp = (void __user *)arg;
2754         int r;
2755
2756         if (kvm->mm != current->mm)
2757                 return -EIO;
2758         switch (ioctl) {
2759         case KVM_CREATE_VCPU:
2760                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2761                 break;
2762         case KVM_SET_USER_MEMORY_REGION: {
2763                 struct kvm_userspace_memory_region kvm_userspace_mem;
2764
2765                 r = -EFAULT;
2766                 if (copy_from_user(&kvm_userspace_mem, argp,
2767                                                 sizeof(kvm_userspace_mem)))
2768                         goto out;
2769
2770                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2771                 break;
2772         }
2773         case KVM_GET_DIRTY_LOG: {
2774                 struct kvm_dirty_log log;
2775
2776                 r = -EFAULT;
2777                 if (copy_from_user(&log, argp, sizeof(log)))
2778                         goto out;
2779                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2780                 break;
2781         }
2782 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2783         case KVM_REGISTER_COALESCED_MMIO: {
2784                 struct kvm_coalesced_mmio_zone zone;
2785
2786                 r = -EFAULT;
2787                 if (copy_from_user(&zone, argp, sizeof(zone)))
2788                         goto out;
2789                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2790                 break;
2791         }
2792         case KVM_UNREGISTER_COALESCED_MMIO: {
2793                 struct kvm_coalesced_mmio_zone zone;
2794
2795                 r = -EFAULT;
2796                 if (copy_from_user(&zone, argp, sizeof(zone)))
2797                         goto out;
2798                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2799                 break;
2800         }
2801 #endif
2802         case KVM_IRQFD: {
2803                 struct kvm_irqfd data;
2804
2805                 r = -EFAULT;
2806                 if (copy_from_user(&data, argp, sizeof(data)))
2807                         goto out;
2808                 r = kvm_irqfd(kvm, &data);
2809                 break;
2810         }
2811         case KVM_IOEVENTFD: {
2812                 struct kvm_ioeventfd data;
2813
2814                 r = -EFAULT;
2815                 if (copy_from_user(&data, argp, sizeof(data)))
2816                         goto out;
2817                 r = kvm_ioeventfd(kvm, &data);
2818                 break;
2819         }
2820 #ifdef CONFIG_HAVE_KVM_MSI
2821         case KVM_SIGNAL_MSI: {
2822                 struct kvm_msi msi;
2823
2824                 r = -EFAULT;
2825                 if (copy_from_user(&msi, argp, sizeof(msi)))
2826                         goto out;
2827                 r = kvm_send_userspace_msi(kvm, &msi);
2828                 break;
2829         }
2830 #endif
2831 #ifdef __KVM_HAVE_IRQ_LINE
2832         case KVM_IRQ_LINE_STATUS:
2833         case KVM_IRQ_LINE: {
2834                 struct kvm_irq_level irq_event;
2835
2836                 r = -EFAULT;
2837                 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
2838                         goto out;
2839
2840                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
2841                                         ioctl == KVM_IRQ_LINE_STATUS);
2842                 if (r)
2843                         goto out;
2844
2845                 r = -EFAULT;
2846                 if (ioctl == KVM_IRQ_LINE_STATUS) {
2847                         if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
2848                                 goto out;
2849                 }
2850
2851                 r = 0;
2852                 break;
2853         }
2854 #endif
2855 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2856         case KVM_SET_GSI_ROUTING: {
2857                 struct kvm_irq_routing routing;
2858                 struct kvm_irq_routing __user *urouting;
2859                 struct kvm_irq_routing_entry *entries;
2860
2861                 r = -EFAULT;
2862                 if (copy_from_user(&routing, argp, sizeof(routing)))
2863                         goto out;
2864                 r = -EINVAL;
2865                 if (routing.nr > KVM_MAX_IRQ_ROUTES)
2866                         goto out;
2867                 if (routing.flags)
2868                         goto out;
2869                 r = -ENOMEM;
2870                 entries = vmalloc(routing.nr * sizeof(*entries));
2871                 if (!entries)
2872                         goto out;
2873                 r = -EFAULT;
2874                 urouting = argp;
2875                 if (copy_from_user(entries, urouting->entries,
2876                                    routing.nr * sizeof(*entries)))
2877                         goto out_free_irq_routing;
2878                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
2879                                         routing.flags);
2880 out_free_irq_routing:
2881                 vfree(entries);
2882                 break;
2883         }
2884 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
2885         case KVM_CREATE_DEVICE: {
2886                 struct kvm_create_device cd;
2887
2888                 r = -EFAULT;
2889                 if (copy_from_user(&cd, argp, sizeof(cd)))
2890                         goto out;
2891
2892                 r = kvm_ioctl_create_device(kvm, &cd);
2893                 if (r)
2894                         goto out;
2895
2896                 r = -EFAULT;
2897                 if (copy_to_user(argp, &cd, sizeof(cd)))
2898                         goto out;
2899
2900                 r = 0;
2901                 break;
2902         }
2903         case KVM_CHECK_EXTENSION:
2904                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
2905                 break;
2906         default:
2907                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2908         }
2909 out:
2910         return r;
2911 }
2912
2913 #ifdef CONFIG_KVM_COMPAT
2914 struct compat_kvm_dirty_log {
2915         __u32 slot;
2916         __u32 padding1;
2917         union {
2918                 compat_uptr_t dirty_bitmap; /* one bit per page */
2919                 __u64 padding2;
2920         };
2921 };
2922
2923 static long kvm_vm_compat_ioctl(struct file *filp,
2924                            unsigned int ioctl, unsigned long arg)
2925 {
2926         struct kvm *kvm = filp->private_data;
2927         int r;
2928
2929         if (kvm->mm != current->mm)
2930                 return -EIO;
2931         switch (ioctl) {
2932         case KVM_GET_DIRTY_LOG: {
2933                 struct compat_kvm_dirty_log compat_log;
2934                 struct kvm_dirty_log log;
2935
2936                 r = -EFAULT;
2937                 if (copy_from_user(&compat_log, (void __user *)arg,
2938                                    sizeof(compat_log)))
2939                         goto out;
2940                 log.slot         = compat_log.slot;
2941                 log.padding1     = compat_log.padding1;
2942                 log.padding2     = compat_log.padding2;
2943                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2944
2945                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2946                 break;
2947         }
2948         default:
2949                 r = kvm_vm_ioctl(filp, ioctl, arg);
2950         }
2951
2952 out:
2953         return r;
2954 }
2955 #endif
2956
2957 static struct file_operations kvm_vm_fops = {
2958         .release        = kvm_vm_release,
2959         .unlocked_ioctl = kvm_vm_ioctl,
2960 #ifdef CONFIG_KVM_COMPAT
2961         .compat_ioctl   = kvm_vm_compat_ioctl,
2962 #endif
2963         .llseek         = noop_llseek,
2964 };
2965
2966 static int kvm_dev_ioctl_create_vm(unsigned long type)
2967 {
2968         int r;
2969         struct kvm *kvm;
2970
2971         kvm = kvm_create_vm(type);
2972         if (IS_ERR(kvm))
2973                 return PTR_ERR(kvm);
2974 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2975         r = kvm_coalesced_mmio_init(kvm);
2976         if (r < 0) {
2977                 kvm_put_kvm(kvm);
2978                 return r;
2979         }
2980 #endif
2981         r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2982         if (r < 0)
2983                 kvm_put_kvm(kvm);
2984
2985         return r;
2986 }
2987
2988 static long kvm_dev_ioctl(struct file *filp,
2989                           unsigned int ioctl, unsigned long arg)
2990 {
2991         long r = -EINVAL;
2992
2993         switch (ioctl) {
2994         case KVM_GET_API_VERSION:
2995                 if (arg)
2996                         goto out;
2997                 r = KVM_API_VERSION;
2998                 break;
2999         case KVM_CREATE_VM:
3000                 r = kvm_dev_ioctl_create_vm(arg);
3001                 break;
3002         case KVM_CHECK_EXTENSION:
3003                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3004                 break;
3005         case KVM_GET_VCPU_MMAP_SIZE:
3006                 if (arg)
3007                         goto out;
3008                 r = PAGE_SIZE;     /* struct kvm_run */
3009 #ifdef CONFIG_X86
3010                 r += PAGE_SIZE;    /* pio data page */
3011 #endif
3012 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3013                 r += PAGE_SIZE;    /* coalesced mmio ring page */
3014 #endif
3015                 break;
3016         case KVM_TRACE_ENABLE:
3017         case KVM_TRACE_PAUSE:
3018         case KVM_TRACE_DISABLE:
3019                 r = -EOPNOTSUPP;
3020                 break;
3021         default:
3022                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3023         }
3024 out:
3025         return r;
3026 }
3027
3028 static struct file_operations kvm_chardev_ops = {
3029         .unlocked_ioctl = kvm_dev_ioctl,
3030         .compat_ioctl   = kvm_dev_ioctl,
3031         .llseek         = noop_llseek,
3032 };
3033
3034 static struct miscdevice kvm_dev = {
3035         KVM_MINOR,
3036         "kvm",
3037         &kvm_chardev_ops,
3038 };
3039
3040 static void hardware_enable_nolock(void *junk)
3041 {
3042         int cpu = raw_smp_processor_id();
3043         int r;
3044
3045         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3046                 return;
3047
3048         cpumask_set_cpu(cpu, cpus_hardware_enabled);
3049
3050         r = kvm_arch_hardware_enable();
3051
3052         if (r) {
3053                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3054                 atomic_inc(&hardware_enable_failed);
3055                 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3056         }
3057 }
3058
3059 static void hardware_enable(void)
3060 {
3061         raw_spin_lock(&kvm_count_lock);
3062         if (kvm_usage_count)
3063                 hardware_enable_nolock(NULL);
3064         raw_spin_unlock(&kvm_count_lock);
3065 }
3066
3067 static void hardware_disable_nolock(void *junk)
3068 {
3069         int cpu = raw_smp_processor_id();
3070
3071         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3072                 return;
3073         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3074         kvm_arch_hardware_disable();
3075 }
3076
3077 static void hardware_disable(void)
3078 {
3079         raw_spin_lock(&kvm_count_lock);
3080         if (kvm_usage_count)
3081                 hardware_disable_nolock(NULL);
3082         raw_spin_unlock(&kvm_count_lock);
3083 }
3084
3085 static void hardware_disable_all_nolock(void)
3086 {
3087         BUG_ON(!kvm_usage_count);
3088
3089         kvm_usage_count--;
3090         if (!kvm_usage_count)
3091                 on_each_cpu(hardware_disable_nolock, NULL, 1);
3092 }
3093
3094 static void hardware_disable_all(void)
3095 {
3096         raw_spin_lock(&kvm_count_lock);
3097         hardware_disable_all_nolock();
3098         raw_spin_unlock(&kvm_count_lock);
3099 }
3100
3101 static int hardware_enable_all(void)
3102 {
3103         int r = 0;
3104
3105         raw_spin_lock(&kvm_count_lock);
3106
3107         kvm_usage_count++;
3108         if (kvm_usage_count == 1) {
3109                 atomic_set(&hardware_enable_failed, 0);
3110                 on_each_cpu(hardware_enable_nolock, NULL, 1);
3111
3112                 if (atomic_read(&hardware_enable_failed)) {
3113                         hardware_disable_all_nolock();
3114                         r = -EBUSY;
3115                 }
3116         }
3117
3118         raw_spin_unlock(&kvm_count_lock);
3119
3120         return r;
3121 }
3122
3123 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
3124                            void *v)
3125 {
3126         val &= ~CPU_TASKS_FROZEN;
3127         switch (val) {
3128         case CPU_DYING:
3129                 hardware_disable();
3130                 break;
3131         case CPU_STARTING:
3132                 hardware_enable();
3133                 break;
3134         }
3135         return NOTIFY_OK;
3136 }
3137
3138 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3139                       void *v)
3140 {
3141         /*
3142          * Some (well, at least mine) BIOSes hang on reboot if
3143          * in vmx root mode.
3144          *
3145          * And Intel TXT required VMX off for all cpu when system shutdown.
3146          */
3147         pr_info("kvm: exiting hardware virtualization\n");
3148         kvm_rebooting = true;
3149         on_each_cpu(hardware_disable_nolock, NULL, 1);
3150         return NOTIFY_OK;
3151 }
3152
3153 static struct notifier_block kvm_reboot_notifier = {
3154         .notifier_call = kvm_reboot,
3155         .priority = 0,
3156 };
3157
3158 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3159 {
3160         int i;
3161
3162         for (i = 0; i < bus->dev_count; i++) {
3163                 struct kvm_io_device *pos = bus->range[i].dev;
3164
3165                 kvm_iodevice_destructor(pos);
3166         }
3167         kfree(bus);
3168 }
3169
3170 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3171                                  const struct kvm_io_range *r2)
3172 {
3173         gpa_t addr1 = r1->addr;
3174         gpa_t addr2 = r2->addr;
3175
3176         if (addr1 < addr2)
3177                 return -1;
3178
3179         /* If r2->len == 0, match the exact address.  If r2->len != 0,
3180          * accept any overlapping write.  Any order is acceptable for
3181          * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3182          * we process all of them.
3183          */
3184         if (r2->len) {
3185                 addr1 += r1->len;
3186                 addr2 += r2->len;
3187         }
3188
3189         if (addr1 > addr2)
3190                 return 1;
3191
3192         return 0;
3193 }
3194
3195 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3196 {
3197         return kvm_io_bus_cmp(p1, p2);
3198 }
3199
3200 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
3201                           gpa_t addr, int len)
3202 {
3203         bus->range[bus->dev_count++] = (struct kvm_io_range) {
3204                 .addr = addr,
3205                 .len = len,
3206                 .dev = dev,
3207         };
3208
3209         sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
3210                 kvm_io_bus_sort_cmp, NULL);
3211
3212         return 0;
3213 }
3214
3215 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3216                              gpa_t addr, int len)
3217 {
3218         struct kvm_io_range *range, key;
3219         int off;
3220
3221         key = (struct kvm_io_range) {
3222                 .addr = addr,
3223                 .len = len,
3224         };
3225
3226         range = bsearch(&key, bus->range, bus->dev_count,
3227                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3228         if (range == NULL)
3229                 return -ENOENT;
3230
3231         off = range - bus->range;
3232
3233         while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3234                 off--;
3235
3236         return off;
3237 }
3238
3239 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3240                               struct kvm_io_range *range, const void *val)
3241 {
3242         int idx;
3243
3244         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3245         if (idx < 0)
3246                 return -EOPNOTSUPP;
3247
3248         while (idx < bus->dev_count &&
3249                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3250                 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3251                                         range->len, val))
3252                         return idx;
3253                 idx++;
3254         }
3255
3256         return -EOPNOTSUPP;
3257 }
3258
3259 /* kvm_io_bus_write - called under kvm->slots_lock */
3260 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3261                      int len, const void *val)
3262 {
3263         struct kvm_io_bus *bus;
3264         struct kvm_io_range range;
3265         int r;
3266
3267         range = (struct kvm_io_range) {
3268                 .addr = addr,
3269                 .len = len,
3270         };
3271
3272         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3273         r = __kvm_io_bus_write(vcpu, bus, &range, val);
3274         return r < 0 ? r : 0;
3275 }
3276
3277 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3278 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3279                             gpa_t addr, int len, const void *val, long cookie)
3280 {
3281         struct kvm_io_bus *bus;
3282         struct kvm_io_range range;
3283
3284         range = (struct kvm_io_range) {
3285                 .addr = addr,
3286                 .len = len,
3287         };
3288
3289         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3290
3291         /* First try the device referenced by cookie. */
3292         if ((cookie >= 0) && (cookie < bus->dev_count) &&
3293             (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3294                 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3295                                         val))
3296                         return cookie;
3297
3298         /*
3299          * cookie contained garbage; fall back to search and return the
3300          * correct cookie value.
3301          */
3302         return __kvm_io_bus_write(vcpu, bus, &range, val);
3303 }
3304
3305 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3306                              struct kvm_io_range *range, void *val)
3307 {
3308         int idx;
3309
3310         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3311         if (idx < 0)
3312                 return -EOPNOTSUPP;
3313
3314         while (idx < bus->dev_count &&
3315                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3316                 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3317                                        range->len, val))
3318                         return idx;
3319                 idx++;
3320         }
3321
3322         return -EOPNOTSUPP;
3323 }
3324 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3325
3326 /* kvm_io_bus_read - called under kvm->slots_lock */
3327 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3328                     int len, void *val)
3329 {
3330         struct kvm_io_bus *bus;
3331         struct kvm_io_range range;
3332         int r;
3333
3334         range = (struct kvm_io_range) {
3335                 .addr = addr,
3336                 .len = len,
3337         };
3338
3339         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3340         r = __kvm_io_bus_read(vcpu, bus, &range, val);
3341         return r < 0 ? r : 0;
3342 }
3343
3344
3345 /* Caller must hold slots_lock. */
3346 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3347                             int len, struct kvm_io_device *dev)
3348 {
3349         struct kvm_io_bus *new_bus, *bus;
3350
3351         bus = kvm->buses[bus_idx];
3352         /* exclude ioeventfd which is limited by maximum fd */
3353         if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3354                 return -ENOSPC;
3355
3356         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3357                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3358         if (!new_bus)
3359                 return -ENOMEM;
3360         memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
3361                sizeof(struct kvm_io_range)));
3362         kvm_io_bus_insert_dev(new_bus, dev, addr, len);
3363         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3364         synchronize_srcu_expedited(&kvm->srcu);
3365         kfree(bus);
3366
3367         return 0;
3368 }
3369
3370 /* Caller must hold slots_lock. */
3371 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3372                               struct kvm_io_device *dev)
3373 {
3374         int i, r;
3375         struct kvm_io_bus *new_bus, *bus;
3376
3377         bus = kvm->buses[bus_idx];
3378         r = -ENOENT;
3379         for (i = 0; i < bus->dev_count; i++)
3380                 if (bus->range[i].dev == dev) {
3381                         r = 0;
3382                         break;
3383                 }
3384
3385         if (r)
3386                 return r;
3387
3388         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3389                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3390         if (!new_bus)
3391                 return -ENOMEM;
3392
3393         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3394         new_bus->dev_count--;
3395         memcpy(new_bus->range + i, bus->range + i + 1,
3396                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3397
3398         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3399         synchronize_srcu_expedited(&kvm->srcu);
3400         kfree(bus);
3401         return r;
3402 }
3403
3404 static struct notifier_block kvm_cpu_notifier = {
3405         .notifier_call = kvm_cpu_hotplug,
3406 };
3407
3408 static int vm_stat_get(void *_offset, u64 *val)
3409 {
3410         unsigned offset = (long)_offset;
3411         struct kvm *kvm;
3412
3413         *val = 0;
3414         spin_lock(&kvm_lock);
3415         list_for_each_entry(kvm, &vm_list, vm_list)
3416                 *val += *(u32 *)((void *)kvm + offset);
3417         spin_unlock(&kvm_lock);
3418         return 0;
3419 }
3420
3421 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
3422
3423 static int vcpu_stat_get(void *_offset, u64 *val)
3424 {
3425         unsigned offset = (long)_offset;
3426         struct kvm *kvm;
3427         struct kvm_vcpu *vcpu;
3428         int i;
3429
3430         *val = 0;
3431         spin_lock(&kvm_lock);
3432         list_for_each_entry(kvm, &vm_list, vm_list)
3433                 kvm_for_each_vcpu(i, vcpu, kvm)
3434                         *val += *(u32 *)((void *)vcpu + offset);
3435
3436         spin_unlock(&kvm_lock);
3437         return 0;
3438 }
3439
3440 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
3441
3442 static const struct file_operations *stat_fops[] = {
3443         [KVM_STAT_VCPU] = &vcpu_stat_fops,
3444         [KVM_STAT_VM]   = &vm_stat_fops,
3445 };
3446
3447 static int kvm_init_debug(void)
3448 {
3449         int r = -EEXIST;
3450         struct kvm_stats_debugfs_item *p;
3451
3452         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3453         if (kvm_debugfs_dir == NULL)
3454                 goto out;
3455
3456         for (p = debugfs_entries; p->name; ++p) {
3457                 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
3458                                                 (void *)(long)p->offset,
3459                                                 stat_fops[p->kind]);
3460                 if (p->dentry == NULL)
3461                         goto out_dir;
3462         }
3463
3464         return 0;
3465
3466 out_dir:
3467         debugfs_remove_recursive(kvm_debugfs_dir);
3468 out:
3469         return r;
3470 }
3471
3472 static void kvm_exit_debug(void)
3473 {
3474         struct kvm_stats_debugfs_item *p;
3475
3476         for (p = debugfs_entries; p->name; ++p)
3477                 debugfs_remove(p->dentry);
3478         debugfs_remove(kvm_debugfs_dir);
3479 }
3480
3481 static int kvm_suspend(void)
3482 {
3483         if (kvm_usage_count)
3484                 hardware_disable_nolock(NULL);
3485         return 0;
3486 }
3487
3488 static void kvm_resume(void)
3489 {
3490         if (kvm_usage_count) {
3491                 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3492                 hardware_enable_nolock(NULL);
3493         }
3494 }
3495
3496 static struct syscore_ops kvm_syscore_ops = {
3497         .suspend = kvm_suspend,
3498         .resume = kvm_resume,
3499 };
3500
3501 static inline
3502 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3503 {
3504         return container_of(pn, struct kvm_vcpu, preempt_notifier);
3505 }
3506
3507 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3508 {
3509         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3510
3511         if (vcpu->preempted)
3512                 vcpu->preempted = false;
3513
3514         kvm_arch_sched_in(vcpu, cpu);
3515
3516         kvm_arch_vcpu_load(vcpu, cpu);
3517 }
3518
3519 static void kvm_sched_out(struct preempt_notifier *pn,
3520                           struct task_struct *next)
3521 {
3522         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3523
3524         if (current->state == TASK_RUNNING)
3525                 vcpu->preempted = true;
3526         kvm_arch_vcpu_put(vcpu);
3527 }
3528
3529 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3530                   struct module *module)
3531 {
3532         int r;
3533         int cpu;
3534
3535         r = kvm_arch_init(opaque);
3536         if (r)
3537                 goto out_fail;
3538
3539         /*
3540          * kvm_arch_init makes sure there's at most one caller
3541          * for architectures that support multiple implementations,
3542          * like intel and amd on x86.
3543          * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
3544          * conflicts in case kvm is already setup for another implementation.
3545          */
3546         r = kvm_irqfd_init();
3547         if (r)
3548                 goto out_irqfd;
3549
3550         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3551                 r = -ENOMEM;
3552                 goto out_free_0;
3553         }
3554
3555         r = kvm_arch_hardware_setup();
3556         if (r < 0)
3557                 goto out_free_0a;
3558
3559         for_each_online_cpu(cpu) {
3560                 smp_call_function_single(cpu,
3561                                 kvm_arch_check_processor_compat,
3562                                 &r, 1);
3563                 if (r < 0)
3564                         goto out_free_1;
3565         }
3566
3567         r = register_cpu_notifier(&kvm_cpu_notifier);
3568         if (r)
3569                 goto out_free_2;
3570         register_reboot_notifier(&kvm_reboot_notifier);
3571
3572         /* A kmem cache lets us meet the alignment requirements of fx_save. */
3573         if (!vcpu_align)
3574                 vcpu_align = __alignof__(struct kvm_vcpu);
3575         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
3576                                            0, NULL);
3577         if (!kvm_vcpu_cache) {
3578                 r = -ENOMEM;
3579                 goto out_free_3;
3580         }
3581
3582         r = kvm_async_pf_init();
3583         if (r)
3584                 goto out_free;
3585
3586         kvm_chardev_ops.owner = module;
3587         kvm_vm_fops.owner = module;
3588         kvm_vcpu_fops.owner = module;
3589
3590         r = misc_register(&kvm_dev);
3591         if (r) {
3592                 pr_err("kvm: misc device register failed\n");
3593                 goto out_unreg;
3594         }
3595
3596         register_syscore_ops(&kvm_syscore_ops);
3597
3598         kvm_preempt_ops.sched_in = kvm_sched_in;
3599         kvm_preempt_ops.sched_out = kvm_sched_out;
3600
3601         r = kvm_init_debug();
3602         if (r) {
3603                 pr_err("kvm: create debugfs files failed\n");
3604                 goto out_undebugfs;
3605         }
3606
3607         r = kvm_vfio_ops_init();
3608         WARN_ON(r);
3609
3610         return 0;
3611
3612 out_undebugfs:
3613         unregister_syscore_ops(&kvm_syscore_ops);
3614         misc_deregister(&kvm_dev);
3615 out_unreg:
3616         kvm_async_pf_deinit();
3617 out_free:
3618         kmem_cache_destroy(kvm_vcpu_cache);
3619 out_free_3:
3620         unregister_reboot_notifier(&kvm_reboot_notifier);
3621         unregister_cpu_notifier(&kvm_cpu_notifier);
3622 out_free_2:
3623 out_free_1:
3624         kvm_arch_hardware_unsetup();
3625 out_free_0a:
3626         free_cpumask_var(cpus_hardware_enabled);
3627 out_free_0:
3628         kvm_irqfd_exit();
3629 out_irqfd:
3630         kvm_arch_exit();
3631 out_fail:
3632         return r;
3633 }
3634 EXPORT_SYMBOL_GPL(kvm_init);
3635
3636 void kvm_exit(void)
3637 {
3638         kvm_exit_debug();
3639         misc_deregister(&kvm_dev);
3640         kmem_cache_destroy(kvm_vcpu_cache);
3641         kvm_async_pf_deinit();
3642         unregister_syscore_ops(&kvm_syscore_ops);
3643         unregister_reboot_notifier(&kvm_reboot_notifier);
3644         unregister_cpu_notifier(&kvm_cpu_notifier);
3645         on_each_cpu(hardware_disable_nolock, NULL, 1);
3646         kvm_arch_hardware_unsetup();
3647         kvm_arch_exit();
3648         kvm_irqfd_exit();
3649         free_cpumask_var(cpus_hardware_enabled);
3650         kvm_vfio_ops_exit();
3651 }
3652 EXPORT_SYMBOL_GPL(kvm_exit);