kernel/virt/kvm/kvm_main.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *
  14  * This work is licensed under the terms of the GNU GPL, version 2.  See
  15  * the COPYING file in the top-level directory.
  16  *
  17  */
  18
  19 #include <kvm/iodev.h>
  20
  21 #include <linux/kvm_host.h>
  22 #include <linux/kvm.h>
  23 #include <linux/module.h>
  24 #include <linux/errno.h>
  25 #include <linux/percpu.h>
  26 #include <linux/mm.h>
  27 #include <linux/miscdevice.h>
  28 #include <linux/vmalloc.h>
  29 #include <linux/reboot.h>
  30 #include <linux/debugfs.h>
  31 #include <linux/highmem.h>
  32 #include <linux/file.h>
  33 #include <linux/syscore_ops.h>
  34 #include <linux/cpu.h>
  35 #include <linux/sched.h>
  36 #include <linux/cpumask.h>
  37 #include <linux/smp.h>
  38 #include <linux/anon_inodes.h>
  39 #include <linux/profile.h>
  40 #include <linux/kvm_para.h>
  41 #include <linux/pagemap.h>
  42 #include <linux/mman.h>
  43 #include <linux/swap.h>
  44 #include <linux/bitops.h>
  45 #include <linux/spinlock.h>
  46 #include <linux/compat.h>
  47 #include <linux/srcu.h>
  48 #include <linux/hugetlb.h>
  49 #include <linux/slab.h>
  50 #include <linux/sort.h>
  51 #include <linux/bsearch.h>
  52
  53 #include <asm/processor.h>
  54 #include <asm/io.h>
  55 #include <asm/ioctl.h>
  56 #include <asm/uaccess.h>
  57 #include <asm/pgtable.h>
  58
  59 #include "coalesced_mmio.h"
  60 #include "async_pf.h"
  61 #include "vfio.h"
  62
  63 #define CREATE_TRACE_POINTS
  64 #include <trace/events/kvm.h>
  65
  66 MODULE_AUTHOR("Qumranet");
  67 MODULE_LICENSE("GPL");
  68
  69 /* Architectures should define their poll value according to the halt latency */
  70 static unsigned int halt_poll_ns = KVM_HALT_POLL_NS_DEFAULT;
  71 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
  72
  73 /* Default doubles per-vcpu halt_poll_ns. */
  74 static unsigned int halt_poll_ns_grow = 2;
  75 module_param(halt_poll_ns_grow, int, S_IRUGO);
  76
  77 /* Default resets per-vcpu halt_poll_ns . */
  78 static unsigned int halt_poll_ns_shrink;
  79 module_param(halt_poll_ns_shrink, int, S_IRUGO);
  80
  81 /*
  82  * Ordering of locks:
  83  *
  84  *      kvm->lock --> kvm->slots_lock --> kvm->irq_lock
  85  */
  86
  87 DEFINE_SPINLOCK(kvm_lock);
  88 static DEFINE_RAW_SPINLOCK(kvm_count_lock);
  89 LIST_HEAD(vm_list);
  90
  91 static cpumask_var_t cpus_hardware_enabled;
  92 static int kvm_usage_count;
  93 static atomic_t hardware_enable_failed;
  94
  95 struct kmem_cache *kvm_vcpu_cache;
  96 EXPORT_SYMBOL_GPL(kvm_vcpu_cache);
  97
  98 static __read_mostly struct preempt_ops kvm_preempt_ops;
  99
 100 struct dentry *kvm_debugfs_dir;
 101 EXPORT_SYMBOL_GPL(kvm_debugfs_dir);
 102
 103 static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl,
 104                            unsigned long arg);
 105 #ifdef CONFIG_KVM_COMPAT
 106 static long kvm_vcpu_compat_ioctl(struct file *file, unsigned int ioctl,
 107                                   unsigned long arg);
 108 #endif
 109 static int hardware_enable_all(void);
 110 static void hardware_disable_all(void);
 111
 112 static void kvm_io_bus_destroy(struct kvm_io_bus *bus);
 113
 114 static void kvm_release_pfn_dirty(pfn_t pfn);
 115 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot, gfn_t gfn);
 116
 117 __visible bool kvm_rebooting;
 118 EXPORT_SYMBOL_GPL(kvm_rebooting);
 119
 120 static bool largepages_enabled = true;
 121
 122 bool kvm_is_reserved_pfn(pfn_t pfn)
 123 {
 124         if (pfn_valid(pfn))
 125                 return PageReserved(pfn_to_page(pfn));
 126
 127         return true;
 128 }
 129
 130 /*
 131  * Switches to specified vcpu, until a matching vcpu_put()
 132  */
 133 int vcpu_load(struct kvm_vcpu *vcpu)
 134 {
 135         int cpu;
 136
 137         if (mutex_lock_killable(&vcpu->mutex))
 138                 return -EINTR;
 139         cpu = get_cpu();
 140         preempt_notifier_register(&vcpu->preempt_notifier);
 141         kvm_arch_vcpu_load(vcpu, cpu);
 142         put_cpu();
 143         return 0;
 144 }
 145
 146 void vcpu_put(struct kvm_vcpu *vcpu)
 147 {
 148         preempt_disable();
 149         kvm_arch_vcpu_put(vcpu);
 150         preempt_notifier_unregister(&vcpu->preempt_notifier);
 151         preempt_enable();
 152         mutex_unlock(&vcpu->mutex);
 153 }
 154
 155 static void ack_flush(void *_completed)
 156 {
 157 }
 158
 159 bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
 160 {
 161         int i, cpu, me;
 162         cpumask_var_t cpus;
 163         bool called = true;
 164         struct kvm_vcpu *vcpu;
 165
 166         zalloc_cpumask_var(&cpus, GFP_ATOMIC);
 167
 168         me = get_cpu();
 169         kvm_for_each_vcpu(i, vcpu, kvm) {
 170                 kvm_make_request(req, vcpu);
 171                 cpu = vcpu->cpu;
 172
 173                 /* Set ->requests bit before we read ->mode */
 174                 smp_mb();
 175
 176                 if (cpus != NULL && cpu != -1 && cpu != me &&
 177                       kvm_vcpu_exiting_guest_mode(vcpu) != OUTSIDE_GUEST_MODE)
 178                         cpumask_set_cpu(cpu, cpus);
 179         }
 180         if (unlikely(cpus == NULL))
 181                 smp_call_function_many(cpu_online_mask, ack_flush, NULL, 1);
 182         else if (!cpumask_empty(cpus))
 183                 smp_call_function_many(cpus, ack_flush, NULL, 1);
 184         else
 185                 called = false;
 186         put_cpu();
 187         free_cpumask_var(cpus);
 188         return called;
 189 }
 190
 191 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
 192 void kvm_flush_remote_tlbs(struct kvm *kvm)
 193 {
 194         long dirty_count = kvm->tlbs_dirty;
 195
 196         smp_mb();
 197         if (kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH))
 198                 ++kvm->stat.remote_tlb_flush;
 199         cmpxchg(&kvm->tlbs_dirty, dirty_count, 0);
 200 }
 201 EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs);
 202 #endif
 203
 204 void kvm_reload_remote_mmus(struct kvm *kvm)
 205 {
 206         kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_RELOAD);
 207 }
 208
 209 void kvm_make_mclock_inprogress_request(struct kvm *kvm)
 210 {
 211         kvm_make_all_cpus_request(kvm, KVM_REQ_MCLOCK_INPROGRESS);
 212 }
 213
 214 void kvm_make_scan_ioapic_request(struct kvm *kvm)
 215 {
 216         kvm_make_all_cpus_request(kvm, KVM_REQ_SCAN_IOAPIC);
 217 }
 218
 219 int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 220 {
 221         struct page *page;
 222         int r;
 223
 224         mutex_init(&vcpu->mutex);
 225         vcpu->cpu = -1;
 226         vcpu->kvm = kvm;
 227         vcpu->vcpu_id = id;
 228         vcpu->pid = NULL;
 229         init_swait_queue_head(&vcpu->wq);
 230         kvm_async_pf_vcpu_init(vcpu);
 231
 232         vcpu->pre_pcpu = -1;
 233         INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
 234
 235         page = alloc_page(GFP_KERNEL | __GFP_ZERO);
 236         if (!page) {
 237                 r = -ENOMEM;
 238                 goto fail;
 239         }
 240         vcpu->run = page_address(page);
 241
 242         kvm_vcpu_set_in_spin_loop(vcpu, false);
 243         kvm_vcpu_set_dy_eligible(vcpu, false);
 244         vcpu->preempted = false;
 245
 246         r = kvm_arch_vcpu_init(vcpu);
 247         if (r < 0)
 248                 goto fail_free_run;
 249         return 0;
 250
 251 fail_free_run:
 252         free_page((unsigned long)vcpu->run);
 253 fail:
 254         return r;
 255 }
 256 EXPORT_SYMBOL_GPL(kvm_vcpu_init);
 257
 258 void kvm_vcpu_uninit(struct kvm_vcpu *vcpu)
 259 {
 260         put_pid(vcpu->pid);
 261         kvm_arch_vcpu_uninit(vcpu);
 262         free_page((unsigned long)vcpu->run);
 263 }
 264 EXPORT_SYMBOL_GPL(kvm_vcpu_uninit);
 265
 266 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 267 static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn)
 268 {
 269         return container_of(mn, struct kvm, mmu_notifier);
 270 }
 271
 272 static void kvm_mmu_notifier_invalidate_page(struct mmu_notifier *mn,
 273                                              struct mm_struct *mm,
 274                                              unsigned long address)
 275 {
 276         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 277         int need_tlb_flush, idx;
 278
 279         /*
 280          * When ->invalidate_page runs, the linux pte has been zapped
 281          * already but the page is still allocated until
 282          * ->invalidate_page returns. So if we increase the sequence
 283          * here the kvm page fault will notice if the spte can't be
 284          * established because the page is going to be freed. If
 285          * instead the kvm page fault establishes the spte before
 286          * ->invalidate_page runs, kvm_unmap_hva will release it
 287          * before returning.
 288          *
 289          * The sequence increase only need to be seen at spin_unlock
 290          * time, and not at spin_lock time.
 291          *
 292          * Increasing the sequence after the spin_unlock would be
 293          * unsafe because the kvm page fault could then establish the
 294          * pte after kvm_unmap_hva returned, without noticing the page
 295          * is going to be freed.
 296          */
 297         idx = srcu_read_lock(&kvm->srcu);
 298         spin_lock(&kvm->mmu_lock);
 299
 300         kvm->mmu_notifier_seq++;
 301         need_tlb_flush = kvm_unmap_hva(kvm, address) | kvm->tlbs_dirty;
 302         /* we've to flush the tlb before the pages can be freed */
 303         if (need_tlb_flush)
 304                 kvm_flush_remote_tlbs(kvm);
 305
 306         spin_unlock(&kvm->mmu_lock);
 307
 308         kvm_arch_mmu_notifier_invalidate_page(kvm, address);
 309
 310         srcu_read_unlock(&kvm->srcu, idx);
 311 }
 312
 313 static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn,
 314                                         struct mm_struct *mm,
 315                                         unsigned long address,
 316                                         pte_t pte)
 317 {
 318         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 319         int idx;
 320
 321         idx = srcu_read_lock(&kvm->srcu);
 322         spin_lock(&kvm->mmu_lock);
 323         kvm->mmu_notifier_seq++;
 324         kvm_set_spte_hva(kvm, address, pte);
 325         spin_unlock(&kvm->mmu_lock);
 326         srcu_read_unlock(&kvm->srcu, idx);
 327 }
 328
 329 static void kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 330                                                     struct mm_struct *mm,
 331                                                     unsigned long start,
 332                                                     unsigned long end)
 333 {
 334         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 335         int need_tlb_flush = 0, idx;
 336
 337         idx = srcu_read_lock(&kvm->srcu);
 338         spin_lock(&kvm->mmu_lock);
 339         /*
 340          * The count increase must become visible at unlock time as no
 341          * spte can be established without taking the mmu_lock and
 342          * count is also read inside the mmu_lock critical section.
 343          */
 344         kvm->mmu_notifier_count++;
 345         need_tlb_flush = kvm_unmap_hva_range(kvm, start, end);
 346         need_tlb_flush |= kvm->tlbs_dirty;
 347         /* we've to flush the tlb before the pages can be freed */
 348         if (need_tlb_flush)
 349                 kvm_flush_remote_tlbs(kvm);
 350
 351         spin_unlock(&kvm->mmu_lock);
 352         srcu_read_unlock(&kvm->srcu, idx);
 353 }
 354
 355 static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 356                                                   struct mm_struct *mm,
 357                                                   unsigned long start,
 358                                                   unsigned long end)
 359 {
 360         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 361
 362         spin_lock(&kvm->mmu_lock);
 363         /*
 364          * This sequence increase will notify the kvm page fault that
 365          * the page that is going to be mapped in the spte could have
 366          * been freed.
 367          */
 368         kvm->mmu_notifier_seq++;
 369         smp_wmb();
 370         /*
 371          * The above sequence increase must be visible before the
 372          * below count decrease, which is ensured by the smp_wmb above
 373          * in conjunction with the smp_rmb in mmu_notifier_retry().
 374          */
 375         kvm->mmu_notifier_count--;
 376         spin_unlock(&kvm->mmu_lock);
 377
 378         BUG_ON(kvm->mmu_notifier_count < 0);
 379 }
 380
 381 static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
 382                                               struct mm_struct *mm,
 383                                               unsigned long start,
 384                                               unsigned long end)
 385 {
 386         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 387         int young, idx;
 388
 389         idx = srcu_read_lock(&kvm->srcu);
 390         spin_lock(&kvm->mmu_lock);
 391
 392         young = kvm_age_hva(kvm, start, end);
 393         if (young)
 394                 kvm_flush_remote_tlbs(kvm);
 395
 396         spin_unlock(&kvm->mmu_lock);
 397         srcu_read_unlock(&kvm->srcu, idx);
 398
 399         return young;
 400 }
 401
 402 static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
 403                                         struct mm_struct *mm,
 404                                         unsigned long start,
 405                                         unsigned long end)
 406 {
 407         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 408         int young, idx;
 409
 410         idx = srcu_read_lock(&kvm->srcu);
 411         spin_lock(&kvm->mmu_lock);
 412         /*
 413          * Even though we do not flush TLB, this will still adversely
 414          * affect performance on pre-Haswell Intel EPT, where there is
 415          * no EPT Access Bit to clear so that we have to tear down EPT
 416          * tables instead. If we find this unacceptable, we can always
 417          * add a parameter to kvm_age_hva so that it effectively doesn't
 418          * do anything on clear_young.
 419          *
 420          * Also note that currently we never issue secondary TLB flushes
 421          * from clear_young, leaving this job up to the regular system
 422          * cadence. If we find this inaccurate, we might come up with a
 423          * more sophisticated heuristic later.
 424          */
 425         young = kvm_age_hva(kvm, start, end);
 426         spin_unlock(&kvm->mmu_lock);
 427         srcu_read_unlock(&kvm->srcu, idx);
 428
 429         return young;
 430 }
 431
 432 static int kvm_mmu_notifier_test_young(struct mmu_notifier *mn,
 433                                        struct mm_struct *mm,
 434                                        unsigned long address)
 435 {
 436         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 437         int young, idx;
 438
 439         idx = srcu_read_lock(&kvm->srcu);
 440         spin_lock(&kvm->mmu_lock);
 441         young = kvm_test_age_hva(kvm, address);
 442         spin_unlock(&kvm->mmu_lock);
 443         srcu_read_unlock(&kvm->srcu, idx);
 444
 445         return young;
 446 }
 447
 448 static void kvm_mmu_notifier_release(struct mmu_notifier *mn,
 449                                      struct mm_struct *mm)
 450 {
 451         struct kvm *kvm = mmu_notifier_to_kvm(mn);
 452         int idx;
 453
 454         idx = srcu_read_lock(&kvm->srcu);
 455         kvm_arch_flush_shadow_all(kvm);
 456         srcu_read_unlock(&kvm->srcu, idx);
 457 }
 458
 459 static const struct mmu_notifier_ops kvm_mmu_notifier_ops = {
 460         .invalidate_page        = kvm_mmu_notifier_invalidate_page,
 461         .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start,
 462         .invalidate_range_end   = kvm_mmu_notifier_invalidate_range_end,
 463         .clear_flush_young      = kvm_mmu_notifier_clear_flush_young,
 464         .clear_young            = kvm_mmu_notifier_clear_young,
 465         .test_young             = kvm_mmu_notifier_test_young,
 466         .change_pte             = kvm_mmu_notifier_change_pte,
 467         .release                = kvm_mmu_notifier_release,
 468 };
 469
 470 static int kvm_init_mmu_notifier(struct kvm *kvm)
 471 {
 472         kvm->mmu_notifier.ops = &kvm_mmu_notifier_ops;
 473         return mmu_notifier_register(&kvm->mmu_notifier, current->mm);
 474 }
 475
 476 #else  /* !(CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER) */
 477
 478 static int kvm_init_mmu_notifier(struct kvm *kvm)
 479 {
 480         return 0;
 481 }
 482
 483 #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */
 484
 485 static struct kvm_memslots *kvm_alloc_memslots(void)
 486 {
 487         int i;
 488         struct kvm_memslots *slots;
 489
 490         slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
 491         if (!slots)
 492                 return NULL;
 493
 494         /*
 495          * Init kvm generation close to the maximum to easily test the
 496          * code of handling generation number wrap-around.
 497          */
 498         slots->generation = -150;
 499         for (i = 0; i < KVM_MEM_SLOTS_NUM; i++)
 500                 slots->id_to_index[i] = slots->memslots[i].id = i;
 501
 502         return slots;
 503 }
 504
 505 static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
 506 {
 507         if (!memslot->dirty_bitmap)
 508                 return;
 509
 510         kvfree(memslot->dirty_bitmap);
 511         memslot->dirty_bitmap = NULL;
 512 }
 513
 514 /*
 515  * Free any memory in @free but not in @dont.
 516  */
 517 static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *free,
 518                               struct kvm_memory_slot *dont)
 519 {
 520         if (!dont || free->dirty_bitmap != dont->dirty_bitmap)
 521                 kvm_destroy_dirty_bitmap(free);
 522
 523         kvm_arch_free_memslot(kvm, free, dont);
 524
 525         free->npages = 0;
 526 }
 527
 528 static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
 529 {
 530         struct kvm_memory_slot *memslot;
 531
 532         if (!slots)
 533                 return;
 534
 535         kvm_for_each_memslot(memslot, slots)
 536                 kvm_free_memslot(kvm, memslot, NULL);
 537
 538         kvfree(slots);
 539 }
 540
 541 static struct kvm *kvm_create_vm(unsigned long type)
 542 {
 543         int r, i;
 544         struct kvm *kvm = kvm_arch_alloc_vm();
 545
 546         if (!kvm)
 547                 return ERR_PTR(-ENOMEM);
 548
 549         r = kvm_arch_init_vm(kvm, type);
 550         if (r)
 551                 goto out_err_no_disable;
 552
 553         r = hardware_enable_all();
 554         if (r)
 555                 goto out_err_no_disable;
 556
 557 #ifdef CONFIG_HAVE_KVM_IRQFD
 558         INIT_HLIST_HEAD(&kvm->irq_ack_notifier_list);
 559 #endif
 560
 561         BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX);
 562
 563         r = -ENOMEM;
 564         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
 565                 kvm->memslots[i] = kvm_alloc_memslots();
 566                 if (!kvm->memslots[i])
 567                         goto out_err_no_srcu;
 568         }
 569
 570         if (init_srcu_struct(&kvm->srcu))
 571                 goto out_err_no_srcu;
 572         if (init_srcu_struct(&kvm->irq_srcu))
 573                 goto out_err_no_irq_srcu;
 574         for (i = 0; i < KVM_NR_BUSES; i++) {
 575                 kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
 576                                         GFP_KERNEL);
 577                 if (!kvm->buses[i])
 578                         goto out_err;
 579         }
 580
 581         spin_lock_init(&kvm->mmu_lock);
 582         kvm->mm = current->mm;
 583         atomic_inc(&kvm->mm->mm_count);
 584         kvm_eventfd_init(kvm);
 585         mutex_init(&kvm->lock);
 586         mutex_init(&kvm->irq_lock);
 587         mutex_init(&kvm->slots_lock);
 588         atomic_set(&kvm->users_count, 1);
 589         INIT_LIST_HEAD(&kvm->devices);
 590
 591         r = kvm_init_mmu_notifier(kvm);
 592         if (r)
 593                 goto out_err;
 594
 595         spin_lock(&kvm_lock);
 596         list_add(&kvm->vm_list, &vm_list);
 597         spin_unlock(&kvm_lock);
 598
 599         preempt_notifier_inc();
 600
 601         return kvm;
 602
 603 out_err:
 604         cleanup_srcu_struct(&kvm->irq_srcu);
 605 out_err_no_irq_srcu:
 606         cleanup_srcu_struct(&kvm->srcu);
 607 out_err_no_srcu:
 608         hardware_disable_all();
 609 out_err_no_disable:
 610         for (i = 0; i < KVM_NR_BUSES; i++)
 611                 kfree(kvm->buses[i]);
 612         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
 613                 kvm_free_memslots(kvm, kvm->memslots[i]);
 614         kvm_arch_free_vm(kvm);
 615         return ERR_PTR(r);
 616 }
 617
 618 /*
 619  * Avoid using vmalloc for a small buffer.
 620  * Should not be used when the size is statically known.
 621  */
 622 void *kvm_kvzalloc(unsigned long size)
 623 {
 624         if (size > PAGE_SIZE)
 625                 return vzalloc(size);
 626         else
 627                 return kzalloc(size, GFP_KERNEL);
 628 }
 629
 630 static void kvm_destroy_devices(struct kvm *kvm)
 631 {
 632         struct list_head *node, *tmp;
 633
 634         list_for_each_safe(node, tmp, &kvm->devices) {
 635                 struct kvm_device *dev =
 636                         list_entry(node, struct kvm_device, vm_node);
 637
 638                 list_del(node);
 639                 dev->ops->destroy(dev);
 640         }
 641 }
 642
 643 static void kvm_destroy_vm(struct kvm *kvm)
 644 {
 645         int i;
 646         struct mm_struct *mm = kvm->mm;
 647
 648         kvm_arch_sync_events(kvm);
 649         spin_lock(&kvm_lock);
 650         list_del(&kvm->vm_list);
 651         spin_unlock(&kvm_lock);
 652         kvm_free_irq_routing(kvm);
 653         for (i = 0; i < KVM_NR_BUSES; i++)
 654                 kvm_io_bus_destroy(kvm->buses[i]);
 655         kvm_coalesced_mmio_free(kvm);
 656 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 657         mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
 658 #else
 659         kvm_arch_flush_shadow_all(kvm);
 660 #endif
 661         kvm_arch_destroy_vm(kvm);
 662         kvm_destroy_devices(kvm);
 663         for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
 664                 kvm_free_memslots(kvm, kvm->memslots[i]);
 665         cleanup_srcu_struct(&kvm->irq_srcu);
 666         cleanup_srcu_struct(&kvm->srcu);
 667         kvm_arch_free_vm(kvm);
 668         preempt_notifier_dec();
 669         hardware_disable_all();
 670         mmdrop(mm);
 671 }
 672
 673 void kvm_get_kvm(struct kvm *kvm)
 674 {
 675         atomic_inc(&kvm->users_count);
 676 }
 677 EXPORT_SYMBOL_GPL(kvm_get_kvm);
 678
 679 void kvm_put_kvm(struct kvm *kvm)
 680 {
 681         if (atomic_dec_and_test(&kvm->users_count))
 682                 kvm_destroy_vm(kvm);
 683 }
 684 EXPORT_SYMBOL_GPL(kvm_put_kvm);
 685
 686
 687 static int kvm_vm_release(struct inode *inode, struct file *filp)
 688 {
 689         struct kvm *kvm = filp->private_data;
 690
 691         kvm_irqfd_release(kvm);
 692
 693         kvm_put_kvm(kvm);
 694         return 0;
 695 }
 696
 697 /*
 698  * Allocation size is twice as large as the actual dirty bitmap size.
 699  * See x86's kvm_vm_ioctl_get_dirty_log() why this is needed.
 700  */
 701 static int kvm_create_dirty_bitmap(struct kvm_memory_slot *memslot)
 702 {
 703         unsigned long dirty_bytes = 2 * kvm_dirty_bitmap_bytes(memslot);
 704
 705         memslot->dirty_bitmap = kvm_kvzalloc(dirty_bytes);
 706         if (!memslot->dirty_bitmap)
 707                 return -ENOMEM;
 708
 709         return 0;
 710 }
 711
 712 /*
 713  * Insert memslot and re-sort memslots based on their GFN,
 714  * so binary search could be used to lookup GFN.
 715  * Sorting algorithm takes advantage of having initially
 716  * sorted array and known changed memslot position.
 717  */
 718 static void update_memslots(struct kvm_memslots *slots,
 719                             struct kvm_memory_slot *new)
 720 {
 721         int id = new->id;
 722         int i = slots->id_to_index[id];
 723         struct kvm_memory_slot *mslots = slots->memslots;
 724
 725         WARN_ON(mslots[i].id != id);
 726         if (!new->npages) {
 727                 WARN_ON(!mslots[i].npages);
 728                 if (mslots[i].npages)
 729                         slots->used_slots--;
 730         } else {
 731                 if (!mslots[i].npages)
 732                         slots->used_slots++;
 733         }
 734
 735         while (i < KVM_MEM_SLOTS_NUM - 1 &&
 736                new->base_gfn <= mslots[i + 1].base_gfn) {
 737                 if (!mslots[i + 1].npages)
 738                         break;
 739                 mslots[i] = mslots[i + 1];
 740                 slots->id_to_index[mslots[i].id] = i;
 741                 i++;
 742         }
 743
 744         /*
 745          * The ">=" is needed when creating a slot with base_gfn == 0,
 746          * so that it moves before all those with base_gfn == npages == 0.
 747          *
 748          * On the other hand, if new->npages is zero, the above loop has
 749          * already left i pointing to the beginning of the empty part of
 750          * mslots, and the ">=" would move the hole backwards in this
 751          * case---which is wrong.  So skip the loop when deleting a slot.
 752          */
 753         if (new->npages) {
 754                 while (i > 0 &&
 755                        new->base_gfn >= mslots[i - 1].base_gfn) {
 756                         mslots[i] = mslots[i - 1];
 757                         slots->id_to_index[mslots[i].id] = i;
 758                         i--;
 759                 }
 760         } else
 761                 WARN_ON_ONCE(i != slots->used_slots);
 762
 763         mslots[i] = *new;
 764         slots->id_to_index[mslots[i].id] = i;
 765 }
 766
 767 static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
 768 {
 769         u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
 770
 771 #ifdef __KVM_HAVE_READONLY_MEM
 772         valid_flags |= KVM_MEM_READONLY;
 773 #endif
 774
 775         if (mem->flags & ~valid_flags)
 776                 return -EINVAL;
 777
 778         return 0;
 779 }
 780
 781 static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
 782                 int as_id, struct kvm_memslots *slots)
 783 {
 784         struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
 785
 786         /*
 787          * Set the low bit in the generation, which disables SPTE caching
 788          * until the end of synchronize_srcu_expedited.
 789          */
 790         WARN_ON(old_memslots->generation & 1);
 791         slots->generation = old_memslots->generation + 1;
 792
 793         rcu_assign_pointer(kvm->memslots[as_id], slots);
 794         synchronize_srcu_expedited(&kvm->srcu);
 795
 796         /*
 797          * Increment the new memslot generation a second time. This prevents
 798          * vm exits that race with memslot updates from caching a memslot
 799          * generation that will (potentially) be valid forever.
 800          */
 801         slots->generation++;
 802
 803         kvm_arch_memslots_updated(kvm, slots);
 804
 805         return old_memslots;
 806 }
 807
 808 /*
 809  * Allocate some memory and give it an address in the guest physical address
 810  * space.
 811  *
 812  * Discontiguous memory is allowed, mostly for framebuffers.
 813  *
 814  * Must be called holding kvm->slots_lock for write.
 815  */
 816 int __kvm_set_memory_region(struct kvm *kvm,
 817                             const struct kvm_userspace_memory_region *mem)
 818 {
 819         int r;
 820         gfn_t base_gfn;
 821         unsigned long npages;
 822         struct kvm_memory_slot *slot;
 823         struct kvm_memory_slot old, new;
 824         struct kvm_memslots *slots = NULL, *old_memslots;
 825         int as_id, id;
 826         enum kvm_mr_change change;
 827
 828         r = check_memory_region_flags(mem);
 829         if (r)
 830                 goto out;
 831
 832         r = -EINVAL;
 833         as_id = mem->slot >> 16;
 834         id = (u16)mem->slot;
 835
 836         /* General sanity checks */
 837         if (mem->memory_size & (PAGE_SIZE - 1))
 838                 goto out;
 839         if (mem->guest_phys_addr & (PAGE_SIZE - 1))
 840                 goto out;
 841         /* We can read the guest memory with __xxx_user() later on. */
 842         if ((id < KVM_USER_MEM_SLOTS) &&
 843             ((mem->userspace_addr & (PAGE_SIZE - 1)) ||
 844              !access_ok(VERIFY_WRITE,
 845                         (void __user *)(unsigned long)mem->userspace_addr,
 846                         mem->memory_size)))
 847                 goto out;
 848         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_MEM_SLOTS_NUM)
 849                 goto out;
 850         if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 851                 goto out;
 852
 853         slot = id_to_memslot(__kvm_memslots(kvm, as_id), id);
 854         base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 855         npages = mem->memory_size >> PAGE_SHIFT;
 856
 857         if (npages > KVM_MEM_MAX_NR_PAGES)
 858                 goto out;
 859
 860         new = old = *slot;
 861
 862         new.id = id;
 863         new.base_gfn = base_gfn;
 864         new.npages = npages;
 865         new.flags = mem->flags;
 866
 867         if (npages) {
 868                 if (!old.npages)
 869                         change = KVM_MR_CREATE;
 870                 else { /* Modify an existing slot. */
 871                         if ((mem->userspace_addr != old.userspace_addr) ||
 872                             (npages != old.npages) ||
 873                             ((new.flags ^ old.flags) & KVM_MEM_READONLY))
 874                                 goto out;
 875
 876                         if (base_gfn != old.base_gfn)
 877                                 change = KVM_MR_MOVE;
 878                         else if (new.flags != old.flags)
 879                                 change = KVM_MR_FLAGS_ONLY;
 880                         else { /* Nothing to change. */
 881                                 r = 0;
 882                                 goto out;
 883                         }
 884                 }
 885         } else {
 886                 if (!old.npages)
 887                         goto out;
 888
 889                 change = KVM_MR_DELETE;
 890                 new.base_gfn = 0;
 891                 new.flags = 0;
 892         }
 893
 894         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 895                 /* Check for overlaps */
 896                 r = -EEXIST;
 897                 kvm_for_each_memslot(slot, __kvm_memslots(kvm, as_id)) {
 898                         if ((slot->id >= KVM_USER_MEM_SLOTS) ||
 899                             (slot->id == id))
 900                                 continue;
 901                         if (!((base_gfn + npages <= slot->base_gfn) ||
 902                               (base_gfn >= slot->base_gfn + slot->npages)))
 903                                 goto out;
 904                 }
 905         }
 906
 907         /* Free page dirty bitmap if unneeded */
 908         if (!(new.flags & KVM_MEM_LOG_DIRTY_PAGES))
 909                 new.dirty_bitmap = NULL;
 910
 911         r = -ENOMEM;
 912         if (change == KVM_MR_CREATE) {
 913                 new.userspace_addr = mem->userspace_addr;
 914
 915                 if (kvm_arch_create_memslot(kvm, &new, npages))
 916                         goto out_free;
 917         }
 918
 919         /* Allocate page dirty bitmap if needed */
 920         if ((new.flags & KVM_MEM_LOG_DIRTY_PAGES) && !new.dirty_bitmap) {
 921                 if (kvm_create_dirty_bitmap(&new) < 0)
 922                         goto out_free;
 923         }
 924
 925         slots = kvm_kvzalloc(sizeof(struct kvm_memslots));
 926         if (!slots)
 927                 goto out_free;
 928         memcpy(slots, __kvm_memslots(kvm, as_id), sizeof(struct kvm_memslots));
 929
 930         if ((change == KVM_MR_DELETE) || (change == KVM_MR_MOVE)) {
 931                 slot = id_to_memslot(slots, id);
 932                 slot->flags |= KVM_MEMSLOT_INVALID;
 933
 934                 old_memslots = install_new_memslots(kvm, as_id, slots);
 935
 936                 /* slot was deleted or moved, clear iommu mapping */
 937                 kvm_iommu_unmap_pages(kvm, &old);
 938                 /* From this point no new shadow pages pointing to a deleted,
 939                  * or moved, memslot will be created.
 940                  *
 941                  * validation of sp->gfn happens in:
 942                  *      - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
 943                  *      - kvm_is_visible_gfn (mmu_check_roots)
 944                  */
 945                 kvm_arch_flush_shadow_memslot(kvm, slot);
 946
 947                 /*
 948                  * We can re-use the old_memslots from above, the only difference
 949                  * from the currently installed memslots is the invalid flag.  This
 950                  * will get overwritten by update_memslots anyway.
 951                  */
 952                 slots = old_memslots;
 953         }
 954
 955         r = kvm_arch_prepare_memory_region(kvm, &new, mem, change);
 956         if (r)
 957                 goto out_slots;
 958
 959         /* actual memory is freed via old in kvm_free_memslot below */
 960         if (change == KVM_MR_DELETE) {
 961                 new.dirty_bitmap = NULL;
 962                 memset(&new.arch, 0, sizeof(new.arch));
 963         }
 964
 965         update_memslots(slots, &new);
 966         old_memslots = install_new_memslots(kvm, as_id, slots);
 967
 968         kvm_arch_commit_memory_region(kvm, mem, &old, &new, change);
 969
 970         kvm_free_memslot(kvm, &old, &new);
 971         kvfree(old_memslots);
 972
 973         /*
 974          * IOMMU mapping:  New slots need to be mapped.  Old slots need to be
 975          * un-mapped and re-mapped if their base changes.  Since base change
 976          * unmapping is handled above with slot deletion, mapping alone is
 977          * needed here.  Anything else the iommu might care about for existing
 978          * slots (size changes, userspace addr changes and read-only flag
 979          * changes) is disallowed above, so any other attribute changes getting
 980          * here can be skipped.
 981          */
 982         if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
 983                 r = kvm_iommu_map_pages(kvm, &new);
 984                 return r;
 985         }
 986
 987         return 0;
 988
 989 out_slots:
 990         kvfree(slots);
 991 out_free:
 992         kvm_free_memslot(kvm, &new, &old);
 993 out:
 994         return r;
 995 }
 996 EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
 997
 998 int kvm_set_memory_region(struct kvm *kvm,
 999                           const struct kvm_userspace_memory_region *mem)
1000 {
1001         int r;
1002
1003         mutex_lock(&kvm->slots_lock);
1004         r = __kvm_set_memory_region(kvm, mem);
1005         mutex_unlock(&kvm->slots_lock);
1006         return r;
1007 }
1008 EXPORT_SYMBOL_GPL(kvm_set_memory_region);
1009
1010 static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
1011                                           struct kvm_userspace_memory_region *mem)
1012 {
1013         if ((u16)mem->slot >= KVM_USER_MEM_SLOTS)
1014                 return -EINVAL;
1015
1016         return kvm_set_memory_region(kvm, mem);
1017 }
1018
1019 int kvm_get_dirty_log(struct kvm *kvm,
1020                         struct kvm_dirty_log *log, int *is_dirty)
1021 {
1022         struct kvm_memslots *slots;
1023         struct kvm_memory_slot *memslot;
1024         int r, i, as_id, id;
1025         unsigned long n;
1026         unsigned long any = 0;
1027
1028         r = -EINVAL;
1029         as_id = log->slot >> 16;
1030         id = (u16)log->slot;
1031         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1032                 goto out;
1033
1034         slots = __kvm_memslots(kvm, as_id);
1035         memslot = id_to_memslot(slots, id);
1036         r = -ENOENT;
1037         if (!memslot->dirty_bitmap)
1038                 goto out;
1039
1040         n = kvm_dirty_bitmap_bytes(memslot);
1041
1042         for (i = 0; !any && i < n/sizeof(long); ++i)
1043                 any = memslot->dirty_bitmap[i];
1044
1045         r = -EFAULT;
1046         if (copy_to_user(log->dirty_bitmap, memslot->dirty_bitmap, n))
1047                 goto out;
1048
1049         if (any)
1050                 *is_dirty = 1;
1051
1052         r = 0;
1053 out:
1054         return r;
1055 }
1056 EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
1057
1058 #ifdef CONFIG_KVM_GENERIC_DIRTYLOG_READ_PROTECT
1059 /**
1060  * kvm_get_dirty_log_protect - get a snapshot of dirty pages, and if any pages
1061  *      are dirty write protect them for next write.
1062  * @kvm:        pointer to kvm instance
1063  * @log:        slot id and address to which we copy the log
1064  * @is_dirty:   flag set if any page is dirty
1065  *
1066  * We need to keep it in mind that VCPU threads can write to the bitmap
1067  * concurrently. So, to avoid losing track of dirty pages we keep the
1068  * following order:
1069  *
1070  *    1. Take a snapshot of the bit and clear it if needed.
1071  *    2. Write protect the corresponding page.
1072  *    3. Copy the snapshot to the userspace.
1073  *    4. Upon return caller flushes TLB's if needed.
1074  *
1075  * Between 2 and 4, the guest may write to the page using the remaining TLB
1076  * entry.  This is not a problem because the page is reported dirty using
1077  * the snapshot taken before and step 4 ensures that writes done after
1078  * exiting to userspace will be logged for the next call.
1079  *
1080  */
1081 int kvm_get_dirty_log_protect(struct kvm *kvm,
1082                         struct kvm_dirty_log *log, bool *is_dirty)
1083 {
1084         struct kvm_memslots *slots;
1085         struct kvm_memory_slot *memslot;
1086         int r, i, as_id, id;
1087         unsigned long n;
1088         unsigned long *dirty_bitmap;
1089         unsigned long *dirty_bitmap_buffer;
1090
1091         r = -EINVAL;
1092         as_id = log->slot >> 16;
1093         id = (u16)log->slot;
1094         if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
1095                 goto out;
1096
1097         slots = __kvm_memslots(kvm, as_id);
1098         memslot = id_to_memslot(slots, id);
1099
1100         dirty_bitmap = memslot->dirty_bitmap;
1101         r = -ENOENT;
1102         if (!dirty_bitmap)
1103                 goto out;
1104
1105         n = kvm_dirty_bitmap_bytes(memslot);
1106
1107         dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
1108         memset(dirty_bitmap_buffer, 0, n);
1109
1110         spin_lock(&kvm->mmu_lock);
1111         *is_dirty = false;
1112         for (i = 0; i < n / sizeof(long); i++) {
1113                 unsigned long mask;
1114                 gfn_t offset;
1115
1116                 if (!dirty_bitmap[i])
1117                         continue;
1118
1119                 *is_dirty = true;
1120
1121                 mask = xchg(&dirty_bitmap[i], 0);
1122                 dirty_bitmap_buffer[i] = mask;
1123
1124                 if (mask) {
1125                         offset = i * BITS_PER_LONG;
1126                         kvm_arch_mmu_enable_log_dirty_pt_masked(kvm, memslot,
1127                                                                 offset, mask);
1128                 }
1129         }
1130
1131         spin_unlock(&kvm->mmu_lock);
1132
1133         r = -EFAULT;
1134         if (copy_to_user(log->dirty_bitmap, dirty_bitmap_buffer, n))
1135                 goto out;
1136
1137         r = 0;
1138 out:
1139         return r;
1140 }
1141 EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
1142 #endif
1143
1144 bool kvm_largepages_enabled(void)
1145 {
1146         return largepages_enabled;
1147 }
1148
1149 void kvm_disable_largepages(void)
1150 {
1151         largepages_enabled = false;
1152 }
1153 EXPORT_SYMBOL_GPL(kvm_disable_largepages);
1154
1155 struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
1156 {
1157         return __gfn_to_memslot(kvm_memslots(kvm), gfn);
1158 }
1159 EXPORT_SYMBOL_GPL(gfn_to_memslot);
1160
1161 struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
1162 {
1163         return __gfn_to_memslot(kvm_vcpu_memslots(vcpu), gfn);
1164 }
1165
1166 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
1167 {
1168         struct kvm_memory_slot *memslot = gfn_to_memslot(kvm, gfn);
1169
1170         if (!memslot || memslot->id >= KVM_USER_MEM_SLOTS ||
1171               memslot->flags & KVM_MEMSLOT_INVALID)
1172                 return 0;
1173
1174         return 1;
1175 }
1176 EXPORT_SYMBOL_GPL(kvm_is_visible_gfn);
1177
1178 unsigned long kvm_host_page_size(struct kvm *kvm, gfn_t gfn)
1179 {
1180         struct vm_area_struct *vma;
1181         unsigned long addr, size;
1182
1183         size = PAGE_SIZE;
1184
1185         addr = gfn_to_hva(kvm, gfn);
1186         if (kvm_is_error_hva(addr))
1187                 return PAGE_SIZE;
1188
1189         down_read(&current->mm->mmap_sem);
1190         vma = find_vma(current->mm, addr);
1191         if (!vma)
1192                 goto out;
1193
1194         size = vma_kernel_pagesize(vma);
1195
1196 out:
1197         up_read(&current->mm->mmap_sem);
1198
1199         return size;
1200 }
1201
1202 static bool memslot_is_readonly(struct kvm_memory_slot *slot)
1203 {
1204         return slot->flags & KVM_MEM_READONLY;
1205 }
1206
1207 static unsigned long __gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1208                                        gfn_t *nr_pages, bool write)
1209 {
1210         if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
1211                 return KVM_HVA_ERR_BAD;
1212
1213         if (memslot_is_readonly(slot) && write)
1214                 return KVM_HVA_ERR_RO_BAD;
1215
1216         if (nr_pages)
1217                 *nr_pages = slot->npages - (gfn - slot->base_gfn);
1218
1219         return __gfn_to_hva_memslot(slot, gfn);
1220 }
1221
1222 static unsigned long gfn_to_hva_many(struct kvm_memory_slot *slot, gfn_t gfn,
1223                                      gfn_t *nr_pages)
1224 {
1225         return __gfn_to_hva_many(slot, gfn, nr_pages, true);
1226 }
1227
1228 unsigned long gfn_to_hva_memslot(struct kvm_memory_slot *slot,
1229                                         gfn_t gfn)
1230 {
1231         return gfn_to_hva_many(slot, gfn, NULL);
1232 }
1233 EXPORT_SYMBOL_GPL(gfn_to_hva_memslot);
1234
1235 unsigned long gfn_to_hva(struct kvm *kvm, gfn_t gfn)
1236 {
1237         return gfn_to_hva_many(gfn_to_memslot(kvm, gfn), gfn, NULL);
1238 }
1239 EXPORT_SYMBOL_GPL(gfn_to_hva);
1240
1241 unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn)
1242 {
1243         return gfn_to_hva_many(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn, NULL);
1244 }
1245 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_hva);
1246
1247 /*
1248  * If writable is set to false, the hva returned by this function is only
1249  * allowed to be read.
1250  */
1251 unsigned long gfn_to_hva_memslot_prot(struct kvm_memory_slot *slot,
1252                                       gfn_t gfn, bool *writable)
1253 {
1254         unsigned long hva = __gfn_to_hva_many(slot, gfn, NULL, false);
1255
1256         if (!kvm_is_error_hva(hva) && writable)
1257                 *writable = !memslot_is_readonly(slot);
1258
1259         return hva;
1260 }
1261
1262 unsigned long gfn_to_hva_prot(struct kvm *kvm, gfn_t gfn, bool *writable)
1263 {
1264         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1265
1266         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1267 }
1268
1269 unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable)
1270 {
1271         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1272
1273         return gfn_to_hva_memslot_prot(slot, gfn, writable);
1274 }
1275
1276 static int get_user_page_nowait(struct task_struct *tsk, struct mm_struct *mm,
1277         unsigned long start, int write, struct page **page)
1278 {
1279         int flags = FOLL_TOUCH | FOLL_NOWAIT | FOLL_HWPOISON | FOLL_GET;
1280
1281         if (write)
1282                 flags |= FOLL_WRITE;
1283
1284         return __get_user_pages(tsk, mm, start, 1, flags, page, NULL, NULL);
1285 }
1286
1287 static inline int check_user_page_hwpoison(unsigned long addr)
1288 {
1289         int rc, flags = FOLL_TOUCH | FOLL_HWPOISON | FOLL_WRITE;
1290
1291         rc = __get_user_pages(current, current->mm, addr, 1,
1292                               flags, NULL, NULL, NULL);
1293         return rc == -EHWPOISON;
1294 }
1295
1296 /*
1297  * The atomic path to get the writable pfn which will be stored in @pfn,
1298  * true indicates success, otherwise false is returned.
1299  */
1300 static bool hva_to_pfn_fast(unsigned long addr, bool atomic, bool *async,
1301                             bool write_fault, bool *writable, pfn_t *pfn)
1302 {
1303         struct page *page[1];
1304         int npages;
1305
1306         if (!(async || atomic))
1307                 return false;
1308
1309         /*
1310          * Fast pin a writable pfn only if it is a write fault request
1311          * or the caller allows to map a writable pfn for a read fault
1312          * request.
1313          */
1314         if (!(write_fault || writable))
1315                 return false;
1316
1317         npages = __get_user_pages_fast(addr, 1, 1, page);
1318         if (npages == 1) {
1319                 *pfn = page_to_pfn(page[0]);
1320
1321                 if (writable)
1322                         *writable = true;
1323                 return true;
1324         }
1325
1326         return false;
1327 }
1328
1329 /*
1330  * The slow path to get the pfn of the specified host virtual address,
1331  * 1 indicates success, -errno is returned if error is detected.
1332  */
1333 static int hva_to_pfn_slow(unsigned long addr, bool *async, bool write_fault,
1334                            bool *writable, pfn_t *pfn)
1335 {
1336         struct page *page[1];
1337         int npages = 0;
1338
1339         might_sleep();
1340
1341         if (writable)
1342                 *writable = write_fault;
1343
1344         if (async) {
1345                 down_read(&current->mm->mmap_sem);
1346                 npages = get_user_page_nowait(current, current->mm,
1347                                               addr, write_fault, page);
1348                 up_read(&current->mm->mmap_sem);
1349         } else
1350                 npages = __get_user_pages_unlocked(current, current->mm, addr, 1,
1351                                                    write_fault, 0, page,
1352                                                    FOLL_TOUCH|FOLL_HWPOISON);
1353         if (npages != 1)
1354                 return npages;
1355
1356         /* map read fault as writable if possible */
1357         if (unlikely(!write_fault) && writable) {
1358                 struct page *wpage[1];
1359
1360                 npages = __get_user_pages_fast(addr, 1, 1, wpage);
1361                 if (npages == 1) {
1362                         *writable = true;
1363                         put_page(page[0]);
1364                         page[0] = wpage[0];
1365                 }
1366
1367                 npages = 1;
1368         }
1369         *pfn = page_to_pfn(page[0]);
1370         return npages;
1371 }
1372
1373 static bool vma_is_valid(struct vm_area_struct *vma, bool write_fault)
1374 {
1375         if (unlikely(!(vma->vm_flags & VM_READ)))
1376                 return false;
1377
1378         if (write_fault && (unlikely(!(vma->vm_flags & VM_WRITE))))
1379                 return false;
1380
1381         return true;
1382 }
1383
1384 /*
1385  * Pin guest page in memory and return its pfn.
1386  * @addr: host virtual address which maps memory to the guest
1387  * @atomic: whether this function can sleep
1388  * @async: whether this function need to wait IO complete if the
1389  *         host page is not in the memory
1390  * @write_fault: whether we should get a writable host page
1391  * @writable: whether it allows to map a writable host page for !@write_fault
1392  *
1393  * The function will map a writable host page for these two cases:
1394  * 1): @write_fault = true
1395  * 2): @write_fault = false && @writable, @writable will tell the caller
1396  *     whether the mapping is writable.
1397  */
1398 static pfn_t hva_to_pfn(unsigned long addr, bool atomic, bool *async,
1399                         bool write_fault, bool *writable)
1400 {
1401         struct vm_area_struct *vma;
1402         pfn_t pfn = 0;
1403         int npages;
1404
1405         /* we can do it either atomically or asynchronously, not both */
1406         BUG_ON(atomic && async);
1407
1408         if (hva_to_pfn_fast(addr, atomic, async, write_fault, writable, &pfn))
1409                 return pfn;
1410
1411         if (atomic)
1412                 return KVM_PFN_ERR_FAULT;
1413
1414         npages = hva_to_pfn_slow(addr, async, write_fault, writable, &pfn);
1415         if (npages == 1)
1416                 return pfn;
1417
1418         down_read(&current->mm->mmap_sem);
1419         if (npages == -EHWPOISON ||
1420               (!async && check_user_page_hwpoison(addr))) {
1421                 pfn = KVM_PFN_ERR_HWPOISON;
1422                 goto exit;
1423         }
1424
1425         vma = find_vma_intersection(current->mm, addr, addr + 1);
1426
1427         if (vma == NULL)
1428                 pfn = KVM_PFN_ERR_FAULT;
1429         else if ((vma->vm_flags & VM_PFNMAP)) {
1430                 pfn = ((addr - vma->vm_start) >> PAGE_SHIFT) +
1431                         vma->vm_pgoff;
1432                 BUG_ON(!kvm_is_reserved_pfn(pfn));
1433         } else {
1434                 if (async && vma_is_valid(vma, write_fault))
1435                         *async = true;
1436                 pfn = KVM_PFN_ERR_FAULT;
1437         }
1438 exit:
1439         up_read(&current->mm->mmap_sem);
1440         return pfn;
1441 }
1442
1443 pfn_t __gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn, bool atomic,
1444                            bool *async, bool write_fault, bool *writable)
1445 {
1446         unsigned long addr = __gfn_to_hva_many(slot, gfn, NULL, write_fault);
1447
1448         if (addr == KVM_HVA_ERR_RO_BAD)
1449                 return KVM_PFN_ERR_RO_FAULT;
1450
1451         if (kvm_is_error_hva(addr))
1452                 return KVM_PFN_NOSLOT;
1453
1454         /* Do not map writable pfn in the readonly memslot. */
1455         if (writable && memslot_is_readonly(slot)) {
1456                 *writable = false;
1457                 writable = NULL;
1458         }
1459
1460         return hva_to_pfn(addr, atomic, async, write_fault,
1461                           writable);
1462 }
1463 EXPORT_SYMBOL_GPL(__gfn_to_pfn_memslot);
1464
1465 pfn_t gfn_to_pfn_prot(struct kvm *kvm, gfn_t gfn, bool write_fault,
1466                       bool *writable)
1467 {
1468         return __gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn, false, NULL,
1469                                     write_fault, writable);
1470 }
1471 EXPORT_SYMBOL_GPL(gfn_to_pfn_prot);
1472
1473 pfn_t gfn_to_pfn_memslot(struct kvm_memory_slot *slot, gfn_t gfn)
1474 {
1475         return __gfn_to_pfn_memslot(slot, gfn, false, NULL, true, NULL);
1476 }
1477 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot);
1478
1479 pfn_t gfn_to_pfn_memslot_atomic(struct kvm_memory_slot *slot, gfn_t gfn)
1480 {
1481         return __gfn_to_pfn_memslot(slot, gfn, true, NULL, true, NULL);
1482 }
1483 EXPORT_SYMBOL_GPL(gfn_to_pfn_memslot_atomic);
1484
1485 pfn_t gfn_to_pfn_atomic(struct kvm *kvm, gfn_t gfn)
1486 {
1487         return gfn_to_pfn_memslot_atomic(gfn_to_memslot(kvm, gfn), gfn);
1488 }
1489 EXPORT_SYMBOL_GPL(gfn_to_pfn_atomic);
1490
1491 pfn_t kvm_vcpu_gfn_to_pfn_atomic(struct kvm_vcpu *vcpu, gfn_t gfn)
1492 {
1493         return gfn_to_pfn_memslot_atomic(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1494 }
1495 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn_atomic);
1496
1497 pfn_t gfn_to_pfn(struct kvm *kvm, gfn_t gfn)
1498 {
1499         return gfn_to_pfn_memslot(gfn_to_memslot(kvm, gfn), gfn);
1500 }
1501 EXPORT_SYMBOL_GPL(gfn_to_pfn);
1502
1503 pfn_t kvm_vcpu_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn)
1504 {
1505         return gfn_to_pfn_memslot(kvm_vcpu_gfn_to_memslot(vcpu, gfn), gfn);
1506 }
1507 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_pfn);
1508
1509 int gfn_to_page_many_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1510                             struct page **pages, int nr_pages)
1511 {
1512         unsigned long addr;
1513         gfn_t entry;
1514
1515         addr = gfn_to_hva_many(slot, gfn, &entry);
1516         if (kvm_is_error_hva(addr))
1517                 return -1;
1518
1519         if (entry < nr_pages)
1520                 return 0;
1521
1522         return __get_user_pages_fast(addr, nr_pages, 1, pages);
1523 }
1524 EXPORT_SYMBOL_GPL(gfn_to_page_many_atomic);
1525
1526 static struct page *kvm_pfn_to_page(pfn_t pfn)
1527 {
1528         if (is_error_noslot_pfn(pfn))
1529                 return KVM_ERR_PTR_BAD_PAGE;
1530
1531         if (kvm_is_reserved_pfn(pfn)) {
1532                 WARN_ON(1);
1533                 return KVM_ERR_PTR_BAD_PAGE;
1534         }
1535
1536         return pfn_to_page(pfn);
1537 }
1538
1539 struct page *gfn_to_page(struct kvm *kvm, gfn_t gfn)
1540 {
1541         pfn_t pfn;
1542
1543         pfn = gfn_to_pfn(kvm, gfn);
1544
1545         return kvm_pfn_to_page(pfn);
1546 }
1547 EXPORT_SYMBOL_GPL(gfn_to_page);
1548
1549 struct page *kvm_vcpu_gfn_to_page(struct kvm_vcpu *vcpu, gfn_t gfn)
1550 {
1551         pfn_t pfn;
1552
1553         pfn = kvm_vcpu_gfn_to_pfn(vcpu, gfn);
1554
1555         return kvm_pfn_to_page(pfn);
1556 }
1557 EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_page);
1558
1559 void kvm_release_page_clean(struct page *page)
1560 {
1561         WARN_ON(is_error_page(page));
1562
1563         kvm_release_pfn_clean(page_to_pfn(page));
1564 }
1565 EXPORT_SYMBOL_GPL(kvm_release_page_clean);
1566
1567 void kvm_release_pfn_clean(pfn_t pfn)
1568 {
1569         if (!is_error_noslot_pfn(pfn) && !kvm_is_reserved_pfn(pfn))
1570                 put_page(pfn_to_page(pfn));
1571 }
1572 EXPORT_SYMBOL_GPL(kvm_release_pfn_clean);
1573
1574 void kvm_release_page_dirty(struct page *page)
1575 {
1576         WARN_ON(is_error_page(page));
1577
1578         kvm_release_pfn_dirty(page_to_pfn(page));
1579 }
1580 EXPORT_SYMBOL_GPL(kvm_release_page_dirty);
1581
1582 static void kvm_release_pfn_dirty(pfn_t pfn)
1583 {
1584         kvm_set_pfn_dirty(pfn);
1585         kvm_release_pfn_clean(pfn);
1586 }
1587
1588 void kvm_set_pfn_dirty(pfn_t pfn)
1589 {
1590         if (!kvm_is_reserved_pfn(pfn)) {
1591                 struct page *page = pfn_to_page(pfn);
1592
1593                 if (!PageReserved(page))
1594                         SetPageDirty(page);
1595         }
1596 }
1597 EXPORT_SYMBOL_GPL(kvm_set_pfn_dirty);
1598
1599 void kvm_set_pfn_accessed(pfn_t pfn)
1600 {
1601         if (!kvm_is_reserved_pfn(pfn))
1602                 mark_page_accessed(pfn_to_page(pfn));
1603 }
1604 EXPORT_SYMBOL_GPL(kvm_set_pfn_accessed);
1605
1606 void kvm_get_pfn(pfn_t pfn)
1607 {
1608         if (!kvm_is_reserved_pfn(pfn))
1609                 get_page(pfn_to_page(pfn));
1610 }
1611 EXPORT_SYMBOL_GPL(kvm_get_pfn);
1612
1613 static int next_segment(unsigned long len, int offset)
1614 {
1615         if (len > PAGE_SIZE - offset)
1616                 return PAGE_SIZE - offset;
1617         else
1618                 return len;
1619 }
1620
1621 static int __kvm_read_guest_page(struct kvm_memory_slot *slot, gfn_t gfn,
1622                                  void *data, int offset, int len)
1623 {
1624         int r;
1625         unsigned long addr;
1626
1627         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1628         if (kvm_is_error_hva(addr))
1629                 return -EFAULT;
1630         r = __copy_from_user(data, (void __user *)addr + offset, len);
1631         if (r)
1632                 return -EFAULT;
1633         return 0;
1634 }
1635
1636 int kvm_read_guest_page(struct kvm *kvm, gfn_t gfn, void *data, int offset,
1637                         int len)
1638 {
1639         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1640
1641         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1642 }
1643 EXPORT_SYMBOL_GPL(kvm_read_guest_page);
1644
1645 int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data,
1646                              int offset, int len)
1647 {
1648         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1649
1650         return __kvm_read_guest_page(slot, gfn, data, offset, len);
1651 }
1652 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_page);
1653
1654 int kvm_read_guest(struct kvm *kvm, gpa_t gpa, void *data, unsigned long len)
1655 {
1656         gfn_t gfn = gpa >> PAGE_SHIFT;
1657         int seg;
1658         int offset = offset_in_page(gpa);
1659         int ret;
1660
1661         while ((seg = next_segment(len, offset)) != 0) {
1662                 ret = kvm_read_guest_page(kvm, gfn, data, offset, seg);
1663                 if (ret < 0)
1664                         return ret;
1665                 offset = 0;
1666                 len -= seg;
1667                 data += seg;
1668                 ++gfn;
1669         }
1670         return 0;
1671 }
1672 EXPORT_SYMBOL_GPL(kvm_read_guest);
1673
1674 int kvm_vcpu_read_guest(struct kvm_vcpu *vcpu, gpa_t gpa, void *data, unsigned long len)
1675 {
1676         gfn_t gfn = gpa >> PAGE_SHIFT;
1677         int seg;
1678         int offset = offset_in_page(gpa);
1679         int ret;
1680
1681         while ((seg = next_segment(len, offset)) != 0) {
1682                 ret = kvm_vcpu_read_guest_page(vcpu, gfn, data, offset, seg);
1683                 if (ret < 0)
1684                         return ret;
1685                 offset = 0;
1686                 len -= seg;
1687                 data += seg;
1688                 ++gfn;
1689         }
1690         return 0;
1691 }
1692 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest);
1693
1694 static int __kvm_read_guest_atomic(struct kvm_memory_slot *slot, gfn_t gfn,
1695                                    void *data, int offset, unsigned long len)
1696 {
1697         int r;
1698         unsigned long addr;
1699
1700         addr = gfn_to_hva_memslot_prot(slot, gfn, NULL);
1701         if (kvm_is_error_hva(addr))
1702                 return -EFAULT;
1703         pagefault_disable();
1704         r = __copy_from_user_inatomic(data, (void __user *)addr + offset, len);
1705         pagefault_enable();
1706         if (r)
1707                 return -EFAULT;
1708         return 0;
1709 }
1710
1711 int kvm_read_guest_atomic(struct kvm *kvm, gpa_t gpa, void *data,
1712                           unsigned long len)
1713 {
1714         gfn_t gfn = gpa >> PAGE_SHIFT;
1715         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1716         int offset = offset_in_page(gpa);
1717
1718         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1719 }
1720 EXPORT_SYMBOL_GPL(kvm_read_guest_atomic);
1721
1722 int kvm_vcpu_read_guest_atomic(struct kvm_vcpu *vcpu, gpa_t gpa,
1723                                void *data, unsigned long len)
1724 {
1725         gfn_t gfn = gpa >> PAGE_SHIFT;
1726         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1727         int offset = offset_in_page(gpa);
1728
1729         return __kvm_read_guest_atomic(slot, gfn, data, offset, len);
1730 }
1731 EXPORT_SYMBOL_GPL(kvm_vcpu_read_guest_atomic);
1732
1733 static int __kvm_write_guest_page(struct kvm_memory_slot *memslot, gfn_t gfn,
1734                                   const void *data, int offset, int len)
1735 {
1736         int r;
1737         unsigned long addr;
1738
1739         addr = gfn_to_hva_memslot(memslot, gfn);
1740         if (kvm_is_error_hva(addr))
1741                 return -EFAULT;
1742         r = __copy_to_user((void __user *)addr + offset, data, len);
1743         if (r)
1744                 return -EFAULT;
1745         mark_page_dirty_in_slot(memslot, gfn);
1746         return 0;
1747 }
1748
1749 int kvm_write_guest_page(struct kvm *kvm, gfn_t gfn,
1750                          const void *data, int offset, int len)
1751 {
1752         struct kvm_memory_slot *slot = gfn_to_memslot(kvm, gfn);
1753
1754         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1755 }
1756 EXPORT_SYMBOL_GPL(kvm_write_guest_page);
1757
1758 int kvm_vcpu_write_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn,
1759                               const void *data, int offset, int len)
1760 {
1761         struct kvm_memory_slot *slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1762
1763         return __kvm_write_guest_page(slot, gfn, data, offset, len);
1764 }
1765 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest_page);
1766
1767 int kvm_write_guest(struct kvm *kvm, gpa_t gpa, const void *data,
1768                     unsigned long len)
1769 {
1770         gfn_t gfn = gpa >> PAGE_SHIFT;
1771         int seg;
1772         int offset = offset_in_page(gpa);
1773         int ret;
1774
1775         while ((seg = next_segment(len, offset)) != 0) {
1776                 ret = kvm_write_guest_page(kvm, gfn, data, offset, seg);
1777                 if (ret < 0)
1778                         return ret;
1779                 offset = 0;
1780                 len -= seg;
1781                 data += seg;
1782                 ++gfn;
1783         }
1784         return 0;
1785 }
1786 EXPORT_SYMBOL_GPL(kvm_write_guest);
1787
1788 int kvm_vcpu_write_guest(struct kvm_vcpu *vcpu, gpa_t gpa, const void *data,
1789                          unsigned long len)
1790 {
1791         gfn_t gfn = gpa >> PAGE_SHIFT;
1792         int seg;
1793         int offset = offset_in_page(gpa);
1794         int ret;
1795
1796         while ((seg = next_segment(len, offset)) != 0) {
1797                 ret = kvm_vcpu_write_guest_page(vcpu, gfn, data, offset, seg);
1798                 if (ret < 0)
1799                         return ret;
1800                 offset = 0;
1801                 len -= seg;
1802                 data += seg;
1803                 ++gfn;
1804         }
1805         return 0;
1806 }
1807 EXPORT_SYMBOL_GPL(kvm_vcpu_write_guest);
1808
1809 int kvm_gfn_to_hva_cache_init(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1810                               gpa_t gpa, unsigned long len)
1811 {
1812         struct kvm_memslots *slots = kvm_memslots(kvm);
1813         int offset = offset_in_page(gpa);
1814         gfn_t start_gfn = gpa >> PAGE_SHIFT;
1815         gfn_t end_gfn = (gpa + len - 1) >> PAGE_SHIFT;
1816         gfn_t nr_pages_needed = end_gfn - start_gfn + 1;
1817         gfn_t nr_pages_avail;
1818
1819         ghc->gpa = gpa;
1820         ghc->generation = slots->generation;
1821         ghc->len = len;
1822         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1823         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn, NULL);
1824         if (!kvm_is_error_hva(ghc->hva) && nr_pages_needed <= 1) {
1825                 ghc->hva += offset;
1826         } else {
1827                 /*
1828                  * If the requested region crosses two memslots, we still
1829                  * verify that the entire region is valid here.
1830                  */
1831                 while (start_gfn <= end_gfn) {
1832                         ghc->memslot = gfn_to_memslot(kvm, start_gfn);
1833                         ghc->hva = gfn_to_hva_many(ghc->memslot, start_gfn,
1834                                                    &nr_pages_avail);
1835                         if (kvm_is_error_hva(ghc->hva))
1836                                 return -EFAULT;
1837                         start_gfn += nr_pages_avail;
1838                 }
1839                 /* Use the slow path for cross page reads and writes. */
1840                 ghc->memslot = NULL;
1841         }
1842         return 0;
1843 }
1844 EXPORT_SYMBOL_GPL(kvm_gfn_to_hva_cache_init);
1845
1846 int kvm_write_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1847                            void *data, unsigned long len)
1848 {
1849         struct kvm_memslots *slots = kvm_memslots(kvm);
1850         int r;
1851
1852         BUG_ON(len > ghc->len);
1853
1854         if (slots->generation != ghc->generation)
1855                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1856
1857         if (unlikely(!ghc->memslot))
1858                 return kvm_write_guest(kvm, ghc->gpa, data, len);
1859
1860         if (kvm_is_error_hva(ghc->hva))
1861                 return -EFAULT;
1862
1863         r = __copy_to_user((void __user *)ghc->hva, data, len);
1864         if (r)
1865                 return -EFAULT;
1866         mark_page_dirty_in_slot(ghc->memslot, ghc->gpa >> PAGE_SHIFT);
1867
1868         return 0;
1869 }
1870 EXPORT_SYMBOL_GPL(kvm_write_guest_cached);
1871
1872 int kvm_read_guest_cached(struct kvm *kvm, struct gfn_to_hva_cache *ghc,
1873                            void *data, unsigned long len)
1874 {
1875         struct kvm_memslots *slots = kvm_memslots(kvm);
1876         int r;
1877
1878         BUG_ON(len > ghc->len);
1879
1880         if (slots->generation != ghc->generation)
1881                 kvm_gfn_to_hva_cache_init(kvm, ghc, ghc->gpa, ghc->len);
1882
1883         if (unlikely(!ghc->memslot))
1884                 return kvm_read_guest(kvm, ghc->gpa, data, len);
1885
1886         if (kvm_is_error_hva(ghc->hva))
1887                 return -EFAULT;
1888
1889         r = __copy_from_user(data, (void __user *)ghc->hva, len);
1890         if (r)
1891                 return -EFAULT;
1892
1893         return 0;
1894 }
1895 EXPORT_SYMBOL_GPL(kvm_read_guest_cached);
1896
1897 int kvm_clear_guest_page(struct kvm *kvm, gfn_t gfn, int offset, int len)
1898 {
1899         const void *zero_page = (const void *) __va(page_to_phys(ZERO_PAGE(0)));
1900
1901         return kvm_write_guest_page(kvm, gfn, zero_page, offset, len);
1902 }
1903 EXPORT_SYMBOL_GPL(kvm_clear_guest_page);
1904
1905 int kvm_clear_guest(struct kvm *kvm, gpa_t gpa, unsigned long len)
1906 {
1907         gfn_t gfn = gpa >> PAGE_SHIFT;
1908         int seg;
1909         int offset = offset_in_page(gpa);
1910         int ret;
1911
1912         while ((seg = next_segment(len, offset)) != 0) {
1913                 ret = kvm_clear_guest_page(kvm, gfn, offset, seg);
1914                 if (ret < 0)
1915                         return ret;
1916                 offset = 0;
1917                 len -= seg;
1918                 ++gfn;
1919         }
1920         return 0;
1921 }
1922 EXPORT_SYMBOL_GPL(kvm_clear_guest);
1923
1924 static void mark_page_dirty_in_slot(struct kvm_memory_slot *memslot,
1925                                     gfn_t gfn)
1926 {
1927         if (memslot && memslot->dirty_bitmap) {
1928                 unsigned long rel_gfn = gfn - memslot->base_gfn;
1929
1930                 set_bit_le(rel_gfn, memslot->dirty_bitmap);
1931         }
1932 }
1933
1934 void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
1935 {
1936         struct kvm_memory_slot *memslot;
1937
1938         memslot = gfn_to_memslot(kvm, gfn);
1939         mark_page_dirty_in_slot(memslot, gfn);
1940 }
1941 EXPORT_SYMBOL_GPL(mark_page_dirty);
1942
1943 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
1944 {
1945         struct kvm_memory_slot *memslot;
1946
1947         memslot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1948         mark_page_dirty_in_slot(memslot, gfn);
1949 }
1950 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
1951
1952 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
1953 {
1954         int old, val;
1955
1956         old = val = vcpu->halt_poll_ns;
1957         /* 10us base */
1958         if (val == 0 && halt_poll_ns_grow)
1959                 val = 10000;
1960         else
1961                 val *= halt_poll_ns_grow;
1962
1963         if (val > halt_poll_ns)
1964                 val = halt_poll_ns;
1965
1966         vcpu->halt_poll_ns = val;
1967         trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
1968 }
1969
1970 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
1971 {
1972         int old, val;
1973
1974         old = val = vcpu->halt_poll_ns;
1975         if (halt_poll_ns_shrink == 0)
1976                 val = 0;
1977         else
1978                 val /= halt_poll_ns_shrink;
1979
1980         vcpu->halt_poll_ns = val;
1981         trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
1982 }
1983
1984 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
1985 {
1986         if (kvm_arch_vcpu_runnable(vcpu)) {
1987                 kvm_make_request(KVM_REQ_UNHALT, vcpu);
1988                 return -EINTR;
1989         }
1990         if (kvm_cpu_has_pending_timer(vcpu))
1991                 return -EINTR;
1992         if (signal_pending(current))
1993                 return -EINTR;
1994
1995         return 0;
1996 }
1997
1998 /*
1999  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
2000  */
2001 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2002 {
2003         ktime_t start, cur;
2004         DECLARE_SWAITQUEUE(wait);
2005         bool waited = false;
2006         u64 block_ns;
2007
2008         start = cur = ktime_get();
2009         if (vcpu->halt_poll_ns) {
2010                 ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
2011
2012                 ++vcpu->stat.halt_attempted_poll;
2013                 do {
2014                         /*
2015                          * This sets KVM_REQ_UNHALT if an interrupt
2016                          * arrives.
2017                          */
2018                         if (kvm_vcpu_check_block(vcpu) < 0) {
2019                                 ++vcpu->stat.halt_successful_poll;
2020                                 goto out;
2021                         }
2022                         cur = ktime_get();
2023                 } while (single_task_running() && ktime_before(cur, stop));
2024         }
2025
2026         kvm_arch_vcpu_blocking(vcpu);
2027
2028         for (;;) {
2029                 prepare_to_swait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
2030
2031                 if (kvm_vcpu_check_block(vcpu) < 0)
2032                         break;
2033
2034                 waited = true;
2035                 schedule();
2036         }
2037
2038         finish_swait(&vcpu->wq, &wait);
2039         cur = ktime_get();
2040
2041         kvm_arch_vcpu_unblocking(vcpu);
2042 out:
2043         block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
2044
2045         if (halt_poll_ns) {
2046                 if (block_ns <= vcpu->halt_poll_ns)
2047                         ;
2048                 /* we had a long block, shrink polling */
2049                 else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
2050                         shrink_halt_poll_ns(vcpu);
2051                 /* we had a short halt and our poll time is too small */
2052                 else if (vcpu->halt_poll_ns < halt_poll_ns &&
2053                         block_ns < halt_poll_ns)
2054                         grow_halt_poll_ns(vcpu);
2055         } else
2056                 vcpu->halt_poll_ns = 0;
2057
2058         trace_kvm_vcpu_wakeup(block_ns, waited);
2059 }
2060 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
2061
2062 #ifndef CONFIG_S390
2063 /*
2064  * Kick a sleeping VCPU, or a guest VCPU in guest mode, into host kernel mode.
2065  */
2066 void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
2067 {
2068         int me;
2069         int cpu = vcpu->cpu;
2070         struct swait_queue_head *wqp;
2071
2072         wqp = kvm_arch_vcpu_wq(vcpu);
2073         if (swait_active(wqp)) {
2074                 swake_up(wqp);
2075                 ++vcpu->stat.halt_wakeup;
2076         }
2077
2078         me = get_cpu();
2079         if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
2080                 if (kvm_arch_vcpu_should_kick(vcpu))
2081                         smp_send_reschedule(cpu);
2082         put_cpu();
2083 }
2084 EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
2085 #endif /* !CONFIG_S390 */
2086
2087 int kvm_vcpu_yield_to(struct kvm_vcpu *target)
2088 {
2089         struct pid *pid;
2090         struct task_struct *task = NULL;
2091         int ret = 0;
2092
2093         rcu_read_lock();
2094         pid = rcu_dereference(target->pid);
2095         if (pid)
2096                 task = get_pid_task(pid, PIDTYPE_PID);
2097         rcu_read_unlock();
2098         if (!task)
2099                 return ret;
2100         ret = yield_to(task, 1);
2101         put_task_struct(task);
2102
2103         return ret;
2104 }
2105 EXPORT_SYMBOL_GPL(kvm_vcpu_yield_to);
2106
2107 /*
2108  * Helper that checks whether a VCPU is eligible for directed yield.
2109  * Most eligible candidate to yield is decided by following heuristics:
2110  *
2111  *  (a) VCPU which has not done pl-exit or cpu relax intercepted recently
2112  *  (preempted lock holder), indicated by @in_spin_loop.
2113  *  Set at the beiginning and cleared at the end of interception/PLE handler.
2114  *
2115  *  (b) VCPU which has done pl-exit/ cpu relax intercepted but did not get
2116  *  chance last time (mostly it has become eligible now since we have probably
2117  *  yielded to lockholder in last iteration. This is done by toggling
2118  *  @dy_eligible each time a VCPU checked for eligibility.)
2119  *
2120  *  Yielding to a recently pl-exited/cpu relax intercepted VCPU before yielding
2121  *  to preempted lock-holder could result in wrong VCPU selection and CPU
2122  *  burning. Giving priority for a potential lock-holder increases lock
2123  *  progress.
2124  *
2125  *  Since algorithm is based on heuristics, accessing another VCPU data without
2126  *  locking does not harm. It may result in trying to yield to  same VCPU, fail
2127  *  and continue with next VCPU and so on.
2128  */
2129 static bool kvm_vcpu_eligible_for_directed_yield(struct kvm_vcpu *vcpu)
2130 {
2131 #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT
2132         bool eligible;
2133
2134         eligible = !vcpu->spin_loop.in_spin_loop ||
2135                     vcpu->spin_loop.dy_eligible;
2136
2137         if (vcpu->spin_loop.in_spin_loop)
2138                 kvm_vcpu_set_dy_eligible(vcpu, !vcpu->spin_loop.dy_eligible);
2139
2140         return eligible;
2141 #else
2142         return true;
2143 #endif
2144 }
2145
2146 void kvm_vcpu_on_spin(struct kvm_vcpu *me)
2147 {
2148         struct kvm *kvm = me->kvm;
2149         struct kvm_vcpu *vcpu;
2150         int last_boosted_vcpu = me->kvm->last_boosted_vcpu;
2151         int yielded = 0;
2152         int try = 3;
2153         int pass;
2154         int i;
2155
2156         kvm_vcpu_set_in_spin_loop(me, true);
2157         /*
2158          * We boost the priority of a VCPU that is runnable but not
2159          * currently running, because it got preempted by something
2160          * else and called schedule in __vcpu_run.  Hopefully that
2161          * VCPU is holding the lock that we need and will release it.
2162          * We approximate round-robin by starting at the last boosted VCPU.
2163          */
2164         for (pass = 0; pass < 2 && !yielded && try; pass++) {
2165                 kvm_for_each_vcpu(i, vcpu, kvm) {
2166                         if (!pass && i <= last_boosted_vcpu) {
2167                                 i = last_boosted_vcpu;
2168                                 continue;
2169                         } else if (pass && i > last_boosted_vcpu)
2170                                 break;
2171                         if (!ACCESS_ONCE(vcpu->preempted))
2172                                 continue;
2173                         if (vcpu == me)
2174                                 continue;
2175                         if (swait_active(&vcpu->wq) && !kvm_arch_vcpu_runnable(vcpu))
2176                                 continue;
2177                         if (!kvm_vcpu_eligible_for_directed_yield(vcpu))
2178                                 continue;
2179
2180                         yielded = kvm_vcpu_yield_to(vcpu);
2181                         if (yielded > 0) {
2182                                 kvm->last_boosted_vcpu = i;
2183                                 break;
2184                         } else if (yielded < 0) {
2185                                 try--;
2186                                 if (!try)
2187                                         break;
2188                         }
2189                 }
2190         }
2191         kvm_vcpu_set_in_spin_loop(me, false);
2192
2193         /* Ensure vcpu is not eligible during next spinloop */
2194         kvm_vcpu_set_dy_eligible(me, false);
2195 }
2196 EXPORT_SYMBOL_GPL(kvm_vcpu_on_spin);
2197
2198 static int kvm_vcpu_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
2199 {
2200         struct kvm_vcpu *vcpu = vma->vm_file->private_data;
2201         struct page *page;
2202
2203         if (vmf->pgoff == 0)
2204                 page = virt_to_page(vcpu->run);
2205 #ifdef CONFIG_X86
2206         else if (vmf->pgoff == KVM_PIO_PAGE_OFFSET)
2207                 page = virt_to_page(vcpu->arch.pio_data);
2208 #endif
2209 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2210         else if (vmf->pgoff == KVM_COALESCED_MMIO_PAGE_OFFSET)
2211                 page = virt_to_page(vcpu->kvm->coalesced_mmio_ring);
2212 #endif
2213         else
2214                 return kvm_arch_vcpu_fault(vcpu, vmf);
2215         get_page(page);
2216         vmf->page = page;
2217         return 0;
2218 }
2219
2220 static const struct vm_operations_struct kvm_vcpu_vm_ops = {
2221         .fault = kvm_vcpu_fault,
2222 };
2223
2224 static int kvm_vcpu_mmap(struct file *file, struct vm_area_struct *vma)
2225 {
2226         vma->vm_ops = &kvm_vcpu_vm_ops;
2227         return 0;
2228 }
2229
2230 static int kvm_vcpu_release(struct inode *inode, struct file *filp)
2231 {
2232         struct kvm_vcpu *vcpu = filp->private_data;
2233
2234         kvm_put_kvm(vcpu->kvm);
2235         return 0;
2236 }
2237
2238 static struct file_operations kvm_vcpu_fops = {
2239         .release        = kvm_vcpu_release,
2240         .unlocked_ioctl = kvm_vcpu_ioctl,
2241 #ifdef CONFIG_KVM_COMPAT
2242         .compat_ioctl   = kvm_vcpu_compat_ioctl,
2243 #endif
2244         .mmap           = kvm_vcpu_mmap,
2245         .llseek         = noop_llseek,
2246 };
2247
2248 /*
2249  * Allocates an inode for the vcpu.
2250  */
2251 static int create_vcpu_fd(struct kvm_vcpu *vcpu)
2252 {
2253         return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
2254 }
2255
2256 /*
2257  * Creates some virtual cpus.  Good luck creating more than one.
2258  */
2259 static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
2260 {
2261         int r;
2262         struct kvm_vcpu *vcpu, *v;
2263
2264         if (id >= KVM_MAX_VCPUS)
2265                 return -EINVAL;
2266
2267         vcpu = kvm_arch_vcpu_create(kvm, id);
2268         if (IS_ERR(vcpu))
2269                 return PTR_ERR(vcpu);
2270
2271         preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
2272
2273         r = kvm_arch_vcpu_setup(vcpu);
2274         if (r)
2275                 goto vcpu_destroy;
2276
2277         mutex_lock(&kvm->lock);
2278         if (!kvm_vcpu_compatible(vcpu)) {
2279                 r = -EINVAL;
2280                 goto unlock_vcpu_destroy;
2281         }
2282         if (atomic_read(&kvm->online_vcpus) == KVM_MAX_VCPUS) {
2283                 r = -EINVAL;
2284                 goto unlock_vcpu_destroy;
2285         }
2286
2287         kvm_for_each_vcpu(r, v, kvm)
2288                 if (v->vcpu_id == id) {
2289                         r = -EEXIST;
2290                         goto unlock_vcpu_destroy;
2291                 }
2292
2293         BUG_ON(kvm->vcpus[atomic_read(&kvm->online_vcpus)]);
2294
2295         /* Now it's all set up, let userspace reach it */
2296         kvm_get_kvm(kvm);
2297         r = create_vcpu_fd(vcpu);
2298         if (r < 0) {
2299                 kvm_put_kvm(kvm);
2300                 goto unlock_vcpu_destroy;
2301         }
2302
2303         kvm->vcpus[atomic_read(&kvm->online_vcpus)] = vcpu;
2304
2305         /*
2306          * Pairs with smp_rmb() in kvm_get_vcpu.  Write kvm->vcpus
2307          * before kvm->online_vcpu's incremented value.
2308          */
2309         smp_wmb();
2310         atomic_inc(&kvm->online_vcpus);
2311
2312         mutex_unlock(&kvm->lock);
2313         kvm_arch_vcpu_postcreate(vcpu);
2314         return r;
2315
2316 unlock_vcpu_destroy:
2317         mutex_unlock(&kvm->lock);
2318 vcpu_destroy:
2319         kvm_arch_vcpu_destroy(vcpu);
2320         return r;
2321 }
2322
2323 static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset)
2324 {
2325         if (sigset) {
2326                 sigdelsetmask(sigset, sigmask(SIGKILL)|sigmask(SIGSTOP));
2327                 vcpu->sigset_active = 1;
2328                 vcpu->sigset = *sigset;
2329         } else
2330                 vcpu->sigset_active = 0;
2331         return 0;
2332 }
2333
2334 static long kvm_vcpu_ioctl(struct file *filp,
2335                            unsigned int ioctl, unsigned long arg)
2336 {
2337         struct kvm_vcpu *vcpu = filp->private_data;
2338         void __user *argp = (void __user *)arg;
2339         int r;
2340         struct kvm_fpu *fpu = NULL;
2341         struct kvm_sregs *kvm_sregs = NULL;
2342
2343         if (vcpu->kvm->mm != current->mm)
2344                 return -EIO;
2345
2346         if (unlikely(_IOC_TYPE(ioctl) != KVMIO))
2347                 return -EINVAL;
2348
2349 #if defined(CONFIG_S390) || defined(CONFIG_PPC) || defined(CONFIG_MIPS)
2350         /*
2351          * Special cases: vcpu ioctls that are asynchronous to vcpu execution,
2352          * so vcpu_load() would break it.
2353          */
2354         if (ioctl == KVM_S390_INTERRUPT || ioctl == KVM_S390_IRQ || ioctl == KVM_INTERRUPT)
2355                 return kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2356 #endif
2357
2358
2359         r = vcpu_load(vcpu);
2360         if (r)
2361                 return r;
2362         switch (ioctl) {
2363         case KVM_RUN:
2364                 r = -EINVAL;
2365                 if (arg)
2366                         goto out;
2367                 if (unlikely(vcpu->pid != current->pids[PIDTYPE_PID].pid)) {
2368                         /* The thread running this VCPU changed. */
2369                         struct pid *oldpid = vcpu->pid;
2370                         struct pid *newpid = get_task_pid(current, PIDTYPE_PID);
2371
2372                         rcu_assign_pointer(vcpu->pid, newpid);
2373                         if (oldpid)
2374                                 synchronize_rcu();
2375                         put_pid(oldpid);
2376                 }
2377                 r = kvm_arch_vcpu_ioctl_run(vcpu, vcpu->run);
2378                 trace_kvm_userspace_exit(vcpu->run->exit_reason, r);
2379                 break;
2380         case KVM_GET_REGS: {
2381                 struct kvm_regs *kvm_regs;
2382
2383                 r = -ENOMEM;
2384                 kvm_regs = kzalloc(sizeof(struct kvm_regs), GFP_KERNEL);
2385                 if (!kvm_regs)
2386                         goto out;
2387                 r = kvm_arch_vcpu_ioctl_get_regs(vcpu, kvm_regs);
2388                 if (r)
2389                         goto out_free1;
2390                 r = -EFAULT;
2391                 if (copy_to_user(argp, kvm_regs, sizeof(struct kvm_regs)))
2392                         goto out_free1;
2393                 r = 0;
2394 out_free1:
2395                 kfree(kvm_regs);
2396                 break;
2397         }
2398         case KVM_SET_REGS: {
2399                 struct kvm_regs *kvm_regs;
2400
2401                 r = -ENOMEM;
2402                 kvm_regs = memdup_user(argp, sizeof(*kvm_regs));
2403                 if (IS_ERR(kvm_regs)) {
2404                         r = PTR_ERR(kvm_regs);
2405                         goto out;
2406                 }
2407                 r = kvm_arch_vcpu_ioctl_set_regs(vcpu, kvm_regs);
2408                 kfree(kvm_regs);
2409                 break;
2410         }
2411         case KVM_GET_SREGS: {
2412                 kvm_sregs = kzalloc(sizeof(struct kvm_sregs), GFP_KERNEL);
2413                 r = -ENOMEM;
2414                 if (!kvm_sregs)
2415                         goto out;
2416                 r = kvm_arch_vcpu_ioctl_get_sregs(vcpu, kvm_sregs);
2417                 if (r)
2418                         goto out;
2419                 r = -EFAULT;
2420                 if (copy_to_user(argp, kvm_sregs, sizeof(struct kvm_sregs)))
2421                         goto out;
2422                 r = 0;
2423                 break;
2424         }
2425         case KVM_SET_SREGS: {
2426                 kvm_sregs = memdup_user(argp, sizeof(*kvm_sregs));
2427                 if (IS_ERR(kvm_sregs)) {
2428                         r = PTR_ERR(kvm_sregs);
2429                         kvm_sregs = NULL;
2430                         goto out;
2431                 }
2432                 r = kvm_arch_vcpu_ioctl_set_sregs(vcpu, kvm_sregs);
2433                 break;
2434         }
2435         case KVM_GET_MP_STATE: {
2436                 struct kvm_mp_state mp_state;
2437
2438                 r = kvm_arch_vcpu_ioctl_get_mpstate(vcpu, &mp_state);
2439                 if (r)
2440                         goto out;
2441                 r = -EFAULT;
2442                 if (copy_to_user(argp, &mp_state, sizeof(mp_state)))
2443                         goto out;
2444                 r = 0;
2445                 break;
2446         }
2447         case KVM_SET_MP_STATE: {
2448                 struct kvm_mp_state mp_state;
2449
2450                 r = -EFAULT;
2451                 if (copy_from_user(&mp_state, argp, sizeof(mp_state)))
2452                         goto out;
2453                 r = kvm_arch_vcpu_ioctl_set_mpstate(vcpu, &mp_state);
2454                 break;
2455         }
2456         case KVM_TRANSLATE: {
2457                 struct kvm_translation tr;
2458
2459                 r = -EFAULT;
2460                 if (copy_from_user(&tr, argp, sizeof(tr)))
2461                         goto out;
2462                 r = kvm_arch_vcpu_ioctl_translate(vcpu, &tr);
2463                 if (r)
2464                         goto out;
2465                 r = -EFAULT;
2466                 if (copy_to_user(argp, &tr, sizeof(tr)))
2467                         goto out;
2468                 r = 0;
2469                 break;
2470         }
2471         case KVM_SET_GUEST_DEBUG: {
2472                 struct kvm_guest_debug dbg;
2473
2474                 r = -EFAULT;
2475                 if (copy_from_user(&dbg, argp, sizeof(dbg)))
2476                         goto out;
2477                 r = kvm_arch_vcpu_ioctl_set_guest_debug(vcpu, &dbg);
2478                 break;
2479         }
2480         case KVM_SET_SIGNAL_MASK: {
2481                 struct kvm_signal_mask __user *sigmask_arg = argp;
2482                 struct kvm_signal_mask kvm_sigmask;
2483                 sigset_t sigset, *p;
2484
2485                 p = NULL;
2486                 if (argp) {
2487                         r = -EFAULT;
2488                         if (copy_from_user(&kvm_sigmask, argp,
2489                                            sizeof(kvm_sigmask)))
2490                                 goto out;
2491                         r = -EINVAL;
2492                         if (kvm_sigmask.len != sizeof(sigset))
2493                                 goto out;
2494                         r = -EFAULT;
2495                         if (copy_from_user(&sigset, sigmask_arg->sigset,
2496                                            sizeof(sigset)))
2497                                 goto out;
2498                         p = &sigset;
2499                 }
2500                 r = kvm_vcpu_ioctl_set_sigmask(vcpu, p);
2501                 break;
2502         }
2503         case KVM_GET_FPU: {
2504                 fpu = kzalloc(sizeof(struct kvm_fpu), GFP_KERNEL);
2505                 r = -ENOMEM;
2506                 if (!fpu)
2507                         goto out;
2508                 r = kvm_arch_vcpu_ioctl_get_fpu(vcpu, fpu);
2509                 if (r)
2510                         goto out;
2511                 r = -EFAULT;
2512                 if (copy_to_user(argp, fpu, sizeof(struct kvm_fpu)))
2513                         goto out;
2514                 r = 0;
2515                 break;
2516         }
2517         case KVM_SET_FPU: {
2518                 fpu = memdup_user(argp, sizeof(*fpu));
2519                 if (IS_ERR(fpu)) {
2520                         r = PTR_ERR(fpu);
2521                         fpu = NULL;
2522                         goto out;
2523                 }
2524                 r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu);
2525                 break;
2526         }
2527         default:
2528                 r = kvm_arch_vcpu_ioctl(filp, ioctl, arg);
2529         }
2530 out:
2531         vcpu_put(vcpu);
2532         kfree(fpu);
2533         kfree(kvm_sregs);
2534         return r;
2535 }
2536
2537 #ifdef CONFIG_KVM_COMPAT
2538 static long kvm_vcpu_compat_ioctl(struct file *filp,
2539                                   unsigned int ioctl, unsigned long arg)
2540 {
2541         struct kvm_vcpu *vcpu = filp->private_data;
2542         void __user *argp = compat_ptr(arg);
2543         int r;
2544
2545         if (vcpu->kvm->mm != current->mm)
2546                 return -EIO;
2547
2548         switch (ioctl) {
2549         case KVM_SET_SIGNAL_MASK: {
2550                 struct kvm_signal_mask __user *sigmask_arg = argp;
2551                 struct kvm_signal_mask kvm_sigmask;
2552                 compat_sigset_t csigset;
2553                 sigset_t sigset;
2554
2555                 if (argp) {
2556                         r = -EFAULT;
2557                         if (copy_from_user(&kvm_sigmask, argp,
2558                                            sizeof(kvm_sigmask)))
2559                                 goto out;
2560                         r = -EINVAL;
2561                         if (kvm_sigmask.len != sizeof(csigset))
2562                                 goto out;
2563                         r = -EFAULT;
2564                         if (copy_from_user(&csigset, sigmask_arg->sigset,
2565                                            sizeof(csigset)))
2566                                 goto out;
2567                         sigset_from_compat(&sigset, &csigset);
2568                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, &sigset);
2569                 } else
2570                         r = kvm_vcpu_ioctl_set_sigmask(vcpu, NULL);
2571                 break;
2572         }
2573         default:
2574                 r = kvm_vcpu_ioctl(filp, ioctl, arg);
2575         }
2576
2577 out:
2578         return r;
2579 }
2580 #endif
2581
2582 static int kvm_device_ioctl_attr(struct kvm_device *dev,
2583                                  int (*accessor)(struct kvm_device *dev,
2584                                                  struct kvm_device_attr *attr),
2585                                  unsigned long arg)
2586 {
2587         struct kvm_device_attr attr;
2588
2589         if (!accessor)
2590                 return -EPERM;
2591
2592         if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2593                 return -EFAULT;
2594
2595         return accessor(dev, &attr);
2596 }
2597
2598 static long kvm_device_ioctl(struct file *filp, unsigned int ioctl,
2599                              unsigned long arg)
2600 {
2601         struct kvm_device *dev = filp->private_data;
2602
2603         switch (ioctl) {
2604         case KVM_SET_DEVICE_ATTR:
2605                 return kvm_device_ioctl_attr(dev, dev->ops->set_attr, arg);
2606         case KVM_GET_DEVICE_ATTR:
2607                 return kvm_device_ioctl_attr(dev, dev->ops->get_attr, arg);
2608         case KVM_HAS_DEVICE_ATTR:
2609                 return kvm_device_ioctl_attr(dev, dev->ops->has_attr, arg);
2610         default:
2611                 if (dev->ops->ioctl)
2612                         return dev->ops->ioctl(dev, ioctl, arg);
2613
2614                 return -ENOTTY;
2615         }
2616 }
2617
2618 static int kvm_device_release(struct inode *inode, struct file *filp)
2619 {
2620         struct kvm_device *dev = filp->private_data;
2621         struct kvm *kvm = dev->kvm;
2622
2623         kvm_put_kvm(kvm);
2624         return 0;
2625 }
2626
2627 static const struct file_operations kvm_device_fops = {
2628         .unlocked_ioctl = kvm_device_ioctl,
2629 #ifdef CONFIG_KVM_COMPAT
2630         .compat_ioctl = kvm_device_ioctl,
2631 #endif
2632         .release = kvm_device_release,
2633 };
2634
2635 struct kvm_device *kvm_device_from_filp(struct file *filp)
2636 {
2637         if (filp->f_op != &kvm_device_fops)
2638                 return NULL;
2639
2640         return filp->private_data;
2641 }
2642
2643 static struct kvm_device_ops *kvm_device_ops_table[KVM_DEV_TYPE_MAX] = {
2644 #ifdef CONFIG_KVM_MPIC
2645         [KVM_DEV_TYPE_FSL_MPIC_20]      = &kvm_mpic_ops,
2646         [KVM_DEV_TYPE_FSL_MPIC_42]      = &kvm_mpic_ops,
2647 #endif
2648
2649 #ifdef CONFIG_KVM_XICS
2650         [KVM_DEV_TYPE_XICS]             = &kvm_xics_ops,
2651 #endif
2652 };
2653
2654 int kvm_register_device_ops(struct kvm_device_ops *ops, u32 type)
2655 {
2656         if (type >= ARRAY_SIZE(kvm_device_ops_table))
2657                 return -ENOSPC;
2658
2659         if (kvm_device_ops_table[type] != NULL)
2660                 return -EEXIST;
2661
2662         kvm_device_ops_table[type] = ops;
2663         return 0;
2664 }
2665
2666 void kvm_unregister_device_ops(u32 type)
2667 {
2668         if (kvm_device_ops_table[type] != NULL)
2669                 kvm_device_ops_table[type] = NULL;
2670 }
2671
2672 static int kvm_ioctl_create_device(struct kvm *kvm,
2673                                    struct kvm_create_device *cd)
2674 {
2675         struct kvm_device_ops *ops = NULL;
2676         struct kvm_device *dev;
2677         bool test = cd->flags & KVM_CREATE_DEVICE_TEST;
2678         int ret;
2679
2680         if (cd->type >= ARRAY_SIZE(kvm_device_ops_table))
2681                 return -ENODEV;
2682
2683         ops = kvm_device_ops_table[cd->type];
2684         if (ops == NULL)
2685                 return -ENODEV;
2686
2687         if (test)
2688                 return 0;
2689
2690         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
2691         if (!dev)
2692                 return -ENOMEM;
2693
2694         dev->ops = ops;
2695         dev->kvm = kvm;
2696
2697         ret = ops->create(dev, cd->type);
2698         if (ret < 0) {
2699                 kfree(dev);
2700                 return ret;
2701         }
2702
2703         ret = anon_inode_getfd(ops->name, &kvm_device_fops, dev, O_RDWR | O_CLOEXEC);
2704         if (ret < 0) {
2705                 ops->destroy(dev);
2706                 return ret;
2707         }
2708
2709         list_add(&dev->vm_node, &kvm->devices);
2710         kvm_get_kvm(kvm);
2711         cd->fd = ret;
2712         return 0;
2713 }
2714
2715 static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg)
2716 {
2717         switch (arg) {
2718         case KVM_CAP_USER_MEMORY:
2719         case KVM_CAP_DESTROY_MEMORY_REGION_WORKS:
2720         case KVM_CAP_JOIN_MEMORY_REGIONS_WORKS:
2721         case KVM_CAP_INTERNAL_ERROR_DATA:
2722 #ifdef CONFIG_HAVE_KVM_MSI
2723         case KVM_CAP_SIGNAL_MSI:
2724 #endif
2725 #ifdef CONFIG_HAVE_KVM_IRQFD
2726         case KVM_CAP_IRQFD:
2727         case KVM_CAP_IRQFD_RESAMPLE:
2728 #endif
2729         case KVM_CAP_IOEVENTFD_ANY_LENGTH:
2730         case KVM_CAP_CHECK_EXTENSION_VM:
2731                 return 1;
2732 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2733         case KVM_CAP_IRQ_ROUTING:
2734                 return KVM_MAX_IRQ_ROUTES;
2735 #endif
2736 #if KVM_ADDRESS_SPACE_NUM > 1
2737         case KVM_CAP_MULTI_ADDRESS_SPACE:
2738                 return KVM_ADDRESS_SPACE_NUM;
2739 #endif
2740         default:
2741                 break;
2742         }
2743         return kvm_vm_ioctl_check_extension(kvm, arg);
2744 }
2745
2746 static long kvm_vm_ioctl(struct file *filp,
2747                            unsigned int ioctl, unsigned long arg)
2748 {
2749         struct kvm *kvm = filp->private_data;
2750         void __user *argp = (void __user *)arg;
2751         int r;
2752
2753         if (kvm->mm != current->mm)
2754                 return -EIO;
2755         switch (ioctl) {
2756         case KVM_CREATE_VCPU:
2757                 r = kvm_vm_ioctl_create_vcpu(kvm, arg);
2758                 break;
2759         case KVM_SET_USER_MEMORY_REGION: {
2760                 struct kvm_userspace_memory_region kvm_userspace_mem;
2761
2762                 r = -EFAULT;
2763                 if (copy_from_user(&kvm_userspace_mem, argp,
2764                                                 sizeof(kvm_userspace_mem)))
2765                         goto out;
2766
2767                 r = kvm_vm_ioctl_set_memory_region(kvm, &kvm_userspace_mem);
2768                 break;
2769         }
2770         case KVM_GET_DIRTY_LOG: {
2771                 struct kvm_dirty_log log;
2772
2773                 r = -EFAULT;
2774                 if (copy_from_user(&log, argp, sizeof(log)))
2775                         goto out;
2776                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2777                 break;
2778         }
2779 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2780         case KVM_REGISTER_COALESCED_MMIO: {
2781                 struct kvm_coalesced_mmio_zone zone;
2782
2783                 r = -EFAULT;
2784                 if (copy_from_user(&zone, argp, sizeof(zone)))
2785                         goto out;
2786                 r = kvm_vm_ioctl_register_coalesced_mmio(kvm, &zone);
2787                 break;
2788         }
2789         case KVM_UNREGISTER_COALESCED_MMIO: {
2790                 struct kvm_coalesced_mmio_zone zone;
2791
2792                 r = -EFAULT;
2793                 if (copy_from_user(&zone, argp, sizeof(zone)))
2794                         goto out;
2795                 r = kvm_vm_ioctl_unregister_coalesced_mmio(kvm, &zone);
2796                 break;
2797         }
2798 #endif
2799         case KVM_IRQFD: {
2800                 struct kvm_irqfd data;
2801
2802                 r = -EFAULT;
2803                 if (copy_from_user(&data, argp, sizeof(data)))
2804                         goto out;
2805                 r = kvm_irqfd(kvm, &data);
2806                 break;
2807         }
2808         case KVM_IOEVENTFD: {
2809                 struct kvm_ioeventfd data;
2810
2811                 r = -EFAULT;
2812                 if (copy_from_user(&data, argp, sizeof(data)))
2813                         goto out;
2814                 r = kvm_ioeventfd(kvm, &data);
2815                 break;
2816         }
2817 #ifdef CONFIG_HAVE_KVM_MSI
2818         case KVM_SIGNAL_MSI: {
2819                 struct kvm_msi msi;
2820
2821                 r = -EFAULT;
2822                 if (copy_from_user(&msi, argp, sizeof(msi)))
2823                         goto out;
2824                 r = kvm_send_userspace_msi(kvm, &msi);
2825                 break;
2826         }
2827 #endif
2828 #ifdef __KVM_HAVE_IRQ_LINE
2829         case KVM_IRQ_LINE_STATUS:
2830         case KVM_IRQ_LINE: {
2831                 struct kvm_irq_level irq_event;
2832
2833                 r = -EFAULT;
2834                 if (copy_from_user(&irq_event, argp, sizeof(irq_event)))
2835                         goto out;
2836
2837                 r = kvm_vm_ioctl_irq_line(kvm, &irq_event,
2838                                         ioctl == KVM_IRQ_LINE_STATUS);
2839                 if (r)
2840                         goto out;
2841
2842                 r = -EFAULT;
2843                 if (ioctl == KVM_IRQ_LINE_STATUS) {
2844                         if (copy_to_user(argp, &irq_event, sizeof(irq_event)))
2845                                 goto out;
2846                 }
2847
2848                 r = 0;
2849                 break;
2850         }
2851 #endif
2852 #ifdef CONFIG_HAVE_KVM_IRQ_ROUTING
2853         case KVM_SET_GSI_ROUTING: {
2854                 struct kvm_irq_routing routing;
2855                 struct kvm_irq_routing __user *urouting;
2856                 struct kvm_irq_routing_entry *entries;
2857
2858                 r = -EFAULT;
2859                 if (copy_from_user(&routing, argp, sizeof(routing)))
2860                         goto out;
2861                 r = -EINVAL;
2862                 if (routing.nr >= KVM_MAX_IRQ_ROUTES)
2863                         goto out;
2864                 if (routing.flags)
2865                         goto out;
2866                 r = -ENOMEM;
2867                 entries = vmalloc(routing.nr * sizeof(*entries));
2868                 if (!entries)
2869                         goto out;
2870                 r = -EFAULT;
2871                 urouting = argp;
2872                 if (copy_from_user(entries, urouting->entries,
2873                                    routing.nr * sizeof(*entries)))
2874                         goto out_free_irq_routing;
2875                 r = kvm_set_irq_routing(kvm, entries, routing.nr,
2876                                         routing.flags);
2877 out_free_irq_routing:
2878                 vfree(entries);
2879                 break;
2880         }
2881 #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */
2882         case KVM_CREATE_DEVICE: {
2883                 struct kvm_create_device cd;
2884
2885                 r = -EFAULT;
2886                 if (copy_from_user(&cd, argp, sizeof(cd)))
2887                         goto out;
2888
2889                 r = kvm_ioctl_create_device(kvm, &cd);
2890                 if (r)
2891                         goto out;
2892
2893                 r = -EFAULT;
2894                 if (copy_to_user(argp, &cd, sizeof(cd)))
2895                         goto out;
2896
2897                 r = 0;
2898                 break;
2899         }
2900         case KVM_CHECK_EXTENSION:
2901                 r = kvm_vm_ioctl_check_extension_generic(kvm, arg);
2902                 break;
2903         default:
2904                 r = kvm_arch_vm_ioctl(filp, ioctl, arg);
2905         }
2906 out:
2907         return r;
2908 }
2909
2910 #ifdef CONFIG_KVM_COMPAT
2911 struct compat_kvm_dirty_log {
2912         __u32 slot;
2913         __u32 padding1;
2914         union {
2915                 compat_uptr_t dirty_bitmap; /* one bit per page */
2916                 __u64 padding2;
2917         };
2918 };
2919
2920 static long kvm_vm_compat_ioctl(struct file *filp,
2921                            unsigned int ioctl, unsigned long arg)
2922 {
2923         struct kvm *kvm = filp->private_data;
2924         int r;
2925
2926         if (kvm->mm != current->mm)
2927                 return -EIO;
2928         switch (ioctl) {
2929         case KVM_GET_DIRTY_LOG: {
2930                 struct compat_kvm_dirty_log compat_log;
2931                 struct kvm_dirty_log log;
2932
2933                 r = -EFAULT;
2934                 if (copy_from_user(&compat_log, (void __user *)arg,
2935                                    sizeof(compat_log)))
2936                         goto out;
2937                 log.slot         = compat_log.slot;
2938                 log.padding1     = compat_log.padding1;
2939                 log.padding2     = compat_log.padding2;
2940                 log.dirty_bitmap = compat_ptr(compat_log.dirty_bitmap);
2941
2942                 r = kvm_vm_ioctl_get_dirty_log(kvm, &log);
2943                 break;
2944         }
2945         default:
2946                 r = kvm_vm_ioctl(filp, ioctl, arg);
2947         }
2948
2949 out:
2950         return r;
2951 }
2952 #endif
2953
2954 static struct file_operations kvm_vm_fops = {
2955         .release        = kvm_vm_release,
2956         .unlocked_ioctl = kvm_vm_ioctl,
2957 #ifdef CONFIG_KVM_COMPAT
2958         .compat_ioctl   = kvm_vm_compat_ioctl,
2959 #endif
2960         .llseek         = noop_llseek,
2961 };
2962
2963 static int kvm_dev_ioctl_create_vm(unsigned long type)
2964 {
2965         int r;
2966         struct kvm *kvm;
2967
2968         kvm = kvm_create_vm(type);
2969         if (IS_ERR(kvm))
2970                 return PTR_ERR(kvm);
2971 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
2972         r = kvm_coalesced_mmio_init(kvm);
2973         if (r < 0) {
2974                 kvm_put_kvm(kvm);
2975                 return r;
2976         }
2977 #endif
2978         r = anon_inode_getfd("kvm-vm", &kvm_vm_fops, kvm, O_RDWR | O_CLOEXEC);
2979         if (r < 0)
2980                 kvm_put_kvm(kvm);
2981
2982         return r;
2983 }
2984
2985 static long kvm_dev_ioctl(struct file *filp,
2986                           unsigned int ioctl, unsigned long arg)
2987 {
2988         long r = -EINVAL;
2989
2990         switch (ioctl) {
2991         case KVM_GET_API_VERSION:
2992                 if (arg)
2993                         goto out;
2994                 r = KVM_API_VERSION;
2995                 break;
2996         case KVM_CREATE_VM:
2997                 r = kvm_dev_ioctl_create_vm(arg);
2998                 break;
2999         case KVM_CHECK_EXTENSION:
3000                 r = kvm_vm_ioctl_check_extension_generic(NULL, arg);
3001                 break;
3002         case KVM_GET_VCPU_MMAP_SIZE:
3003                 if (arg)
3004                         goto out;
3005                 r = PAGE_SIZE;     /* struct kvm_run */
3006 #ifdef CONFIG_X86
3007                 r += PAGE_SIZE;    /* pio data page */
3008 #endif
3009 #ifdef KVM_COALESCED_MMIO_PAGE_OFFSET
3010                 r += PAGE_SIZE;    /* coalesced mmio ring page */
3011 #endif
3012                 break;
3013         case KVM_TRACE_ENABLE:
3014         case KVM_TRACE_PAUSE:
3015         case KVM_TRACE_DISABLE:
3016                 r = -EOPNOTSUPP;
3017                 break;
3018         default:
3019                 return kvm_arch_dev_ioctl(filp, ioctl, arg);
3020         }
3021 out:
3022         return r;
3023 }
3024
3025 static struct file_operations kvm_chardev_ops = {
3026         .unlocked_ioctl = kvm_dev_ioctl,
3027         .compat_ioctl   = kvm_dev_ioctl,
3028         .llseek         = noop_llseek,
3029 };
3030
3031 static struct miscdevice kvm_dev = {
3032         KVM_MINOR,
3033         "kvm",
3034         &kvm_chardev_ops,
3035 };
3036
3037 static void hardware_enable_nolock(void *junk)
3038 {
3039         int cpu = raw_smp_processor_id();
3040         int r;
3041
3042         if (cpumask_test_cpu(cpu, cpus_hardware_enabled))
3043                 return;
3044
3045         cpumask_set_cpu(cpu, cpus_hardware_enabled);
3046
3047         r = kvm_arch_hardware_enable();
3048
3049         if (r) {
3050                 cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3051                 atomic_inc(&hardware_enable_failed);
3052                 pr_info("kvm: enabling virtualization on CPU%d failed\n", cpu);
3053         }
3054 }
3055
3056 static void hardware_enable(void)
3057 {
3058         raw_spin_lock(&kvm_count_lock);
3059         if (kvm_usage_count)
3060                 hardware_enable_nolock(NULL);
3061         raw_spin_unlock(&kvm_count_lock);
3062 }
3063
3064 static void hardware_disable_nolock(void *junk)
3065 {
3066         int cpu = raw_smp_processor_id();
3067
3068         if (!cpumask_test_cpu(cpu, cpus_hardware_enabled))
3069                 return;
3070         cpumask_clear_cpu(cpu, cpus_hardware_enabled);
3071         kvm_arch_hardware_disable();
3072 }
3073
3074 static void hardware_disable(void)
3075 {
3076         raw_spin_lock(&kvm_count_lock);
3077         if (kvm_usage_count)
3078                 hardware_disable_nolock(NULL);
3079         raw_spin_unlock(&kvm_count_lock);
3080 }
3081
3082 static void hardware_disable_all_nolock(void)
3083 {
3084         BUG_ON(!kvm_usage_count);
3085
3086         kvm_usage_count--;
3087         if (!kvm_usage_count)
3088                 on_each_cpu(hardware_disable_nolock, NULL, 1);
3089 }
3090
3091 static void hardware_disable_all(void)
3092 {
3093         raw_spin_lock(&kvm_count_lock);
3094         hardware_disable_all_nolock();
3095         raw_spin_unlock(&kvm_count_lock);
3096 }
3097
3098 static int hardware_enable_all(void)
3099 {
3100         int r = 0;
3101
3102         raw_spin_lock(&kvm_count_lock);
3103
3104         kvm_usage_count++;
3105         if (kvm_usage_count == 1) {
3106                 atomic_set(&hardware_enable_failed, 0);
3107                 on_each_cpu(hardware_enable_nolock, NULL, 1);
3108
3109                 if (atomic_read(&hardware_enable_failed)) {
3110                         hardware_disable_all_nolock();
3111                         r = -EBUSY;
3112                 }
3113         }
3114
3115         raw_spin_unlock(&kvm_count_lock);
3116
3117         return r;
3118 }
3119
3120 static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
3121                            void *v)
3122 {
3123         val &= ~CPU_TASKS_FROZEN;
3124         switch (val) {
3125         case CPU_DYING:
3126                 hardware_disable();
3127                 break;
3128         case CPU_STARTING:
3129                 hardware_enable();
3130                 break;
3131         }
3132         return NOTIFY_OK;
3133 }
3134
3135 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
3136                       void *v)
3137 {
3138         /*
3139          * Some (well, at least mine) BIOSes hang on reboot if
3140          * in vmx root mode.
3141          *
3142          * And Intel TXT required VMX off for all cpu when system shutdown.
3143          */
3144         pr_info("kvm: exiting hardware virtualization\n");
3145         kvm_rebooting = true;
3146         on_each_cpu(hardware_disable_nolock, NULL, 1);
3147         return NOTIFY_OK;
3148 }
3149
3150 static struct notifier_block kvm_reboot_notifier = {
3151         .notifier_call = kvm_reboot,
3152         .priority = 0,
3153 };
3154
3155 static void kvm_io_bus_destroy(struct kvm_io_bus *bus)
3156 {
3157         int i;
3158
3159         for (i = 0; i < bus->dev_count; i++) {
3160                 struct kvm_io_device *pos = bus->range[i].dev;
3161
3162                 kvm_iodevice_destructor(pos);
3163         }
3164         kfree(bus);
3165 }
3166
3167 static inline int kvm_io_bus_cmp(const struct kvm_io_range *r1,
3168                                  const struct kvm_io_range *r2)
3169 {
3170         gpa_t addr1 = r1->addr;
3171         gpa_t addr2 = r2->addr;
3172
3173         if (addr1 < addr2)
3174                 return -1;
3175
3176         /* If r2->len == 0, match the exact address.  If r2->len != 0,
3177          * accept any overlapping write.  Any order is acceptable for
3178          * overlapping ranges, because kvm_io_bus_get_first_dev ensures
3179          * we process all of them.
3180          */
3181         if (r2->len) {
3182                 addr1 += r1->len;
3183                 addr2 += r2->len;
3184         }
3185
3186         if (addr1 > addr2)
3187                 return 1;
3188
3189         return 0;
3190 }
3191
3192 static int kvm_io_bus_sort_cmp(const void *p1, const void *p2)
3193 {
3194         return kvm_io_bus_cmp(p1, p2);
3195 }
3196
3197 static int kvm_io_bus_insert_dev(struct kvm_io_bus *bus, struct kvm_io_device *dev,
3198                           gpa_t addr, int len)
3199 {
3200         bus->range[bus->dev_count++] = (struct kvm_io_range) {
3201                 .addr = addr,
3202                 .len = len,
3203                 .dev = dev,
3204         };
3205
3206         sort(bus->range, bus->dev_count, sizeof(struct kvm_io_range),
3207                 kvm_io_bus_sort_cmp, NULL);
3208
3209         return 0;
3210 }
3211
3212 static int kvm_io_bus_get_first_dev(struct kvm_io_bus *bus,
3213                              gpa_t addr, int len)
3214 {
3215         struct kvm_io_range *range, key;
3216         int off;
3217
3218         key = (struct kvm_io_range) {
3219                 .addr = addr,
3220                 .len = len,
3221         };
3222
3223         range = bsearch(&key, bus->range, bus->dev_count,
3224                         sizeof(struct kvm_io_range), kvm_io_bus_sort_cmp);
3225         if (range == NULL)
3226                 return -ENOENT;
3227
3228         off = range - bus->range;
3229
3230         while (off > 0 && kvm_io_bus_cmp(&key, &bus->range[off-1]) == 0)
3231                 off--;
3232
3233         return off;
3234 }
3235
3236 static int __kvm_io_bus_write(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3237                               struct kvm_io_range *range, const void *val)
3238 {
3239         int idx;
3240
3241         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3242         if (idx < 0)
3243                 return -EOPNOTSUPP;
3244
3245         while (idx < bus->dev_count &&
3246                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3247                 if (!kvm_iodevice_write(vcpu, bus->range[idx].dev, range->addr,
3248                                         range->len, val))
3249                         return idx;
3250                 idx++;
3251         }
3252
3253         return -EOPNOTSUPP;
3254 }
3255
3256 /* kvm_io_bus_write - called under kvm->slots_lock */
3257 int kvm_io_bus_write(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3258                      int len, const void *val)
3259 {
3260         struct kvm_io_bus *bus;
3261         struct kvm_io_range range;
3262         int r;
3263
3264         range = (struct kvm_io_range) {
3265                 .addr = addr,
3266                 .len = len,
3267         };
3268
3269         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3270         r = __kvm_io_bus_write(vcpu, bus, &range, val);
3271         return r < 0 ? r : 0;
3272 }
3273
3274 /* kvm_io_bus_write_cookie - called under kvm->slots_lock */
3275 int kvm_io_bus_write_cookie(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx,
3276                             gpa_t addr, int len, const void *val, long cookie)
3277 {
3278         struct kvm_io_bus *bus;
3279         struct kvm_io_range range;
3280
3281         range = (struct kvm_io_range) {
3282                 .addr = addr,
3283                 .len = len,
3284         };
3285
3286         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3287
3288         /* First try the device referenced by cookie. */
3289         if ((cookie >= 0) && (cookie < bus->dev_count) &&
3290             (kvm_io_bus_cmp(&range, &bus->range[cookie]) == 0))
3291                 if (!kvm_iodevice_write(vcpu, bus->range[cookie].dev, addr, len,
3292                                         val))
3293                         return cookie;
3294
3295         /*
3296          * cookie contained garbage; fall back to search and return the
3297          * correct cookie value.
3298          */
3299         return __kvm_io_bus_write(vcpu, bus, &range, val);
3300 }
3301
3302 static int __kvm_io_bus_read(struct kvm_vcpu *vcpu, struct kvm_io_bus *bus,
3303                              struct kvm_io_range *range, void *val)
3304 {
3305         int idx;
3306
3307         idx = kvm_io_bus_get_first_dev(bus, range->addr, range->len);
3308         if (idx < 0)
3309                 return -EOPNOTSUPP;
3310
3311         while (idx < bus->dev_count &&
3312                 kvm_io_bus_cmp(range, &bus->range[idx]) == 0) {
3313                 if (!kvm_iodevice_read(vcpu, bus->range[idx].dev, range->addr,
3314                                        range->len, val))
3315                         return idx;
3316                 idx++;
3317         }
3318
3319         return -EOPNOTSUPP;
3320 }
3321 EXPORT_SYMBOL_GPL(kvm_io_bus_write);
3322
3323 /* kvm_io_bus_read - called under kvm->slots_lock */
3324 int kvm_io_bus_read(struct kvm_vcpu *vcpu, enum kvm_bus bus_idx, gpa_t addr,
3325                     int len, void *val)
3326 {
3327         struct kvm_io_bus *bus;
3328         struct kvm_io_range range;
3329         int r;
3330
3331         range = (struct kvm_io_range) {
3332                 .addr = addr,
3333                 .len = len,
3334         };
3335
3336         bus = srcu_dereference(vcpu->kvm->buses[bus_idx], &vcpu->kvm->srcu);
3337         r = __kvm_io_bus_read(vcpu, bus, &range, val);
3338         return r < 0 ? r : 0;
3339 }
3340
3341
3342 /* Caller must hold slots_lock. */
3343 int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
3344                             int len, struct kvm_io_device *dev)
3345 {
3346         struct kvm_io_bus *new_bus, *bus;
3347
3348         bus = kvm->buses[bus_idx];
3349         /* exclude ioeventfd which is limited by maximum fd */
3350         if (bus->dev_count - bus->ioeventfd_count > NR_IOBUS_DEVS - 1)
3351                 return -ENOSPC;
3352
3353         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count + 1) *
3354                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3355         if (!new_bus)
3356                 return -ENOMEM;
3357         memcpy(new_bus, bus, sizeof(*bus) + (bus->dev_count *
3358                sizeof(struct kvm_io_range)));
3359         kvm_io_bus_insert_dev(new_bus, dev, addr, len);
3360         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3361         synchronize_srcu_expedited(&kvm->srcu);
3362         kfree(bus);
3363
3364         return 0;
3365 }
3366
3367 /* Caller must hold slots_lock. */
3368 int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
3369                               struct kvm_io_device *dev)
3370 {
3371         int i, r;
3372         struct kvm_io_bus *new_bus, *bus;
3373
3374         bus = kvm->buses[bus_idx];
3375         r = -ENOENT;
3376         for (i = 0; i < bus->dev_count; i++)
3377                 if (bus->range[i].dev == dev) {
3378                         r = 0;
3379                         break;
3380                 }
3381
3382         if (r)
3383                 return r;
3384
3385         new_bus = kmalloc(sizeof(*bus) + ((bus->dev_count - 1) *
3386                           sizeof(struct kvm_io_range)), GFP_KERNEL);
3387         if (!new_bus)
3388                 return -ENOMEM;
3389
3390         memcpy(new_bus, bus, sizeof(*bus) + i * sizeof(struct kvm_io_range));
3391         new_bus->dev_count--;
3392         memcpy(new_bus->range + i, bus->range + i + 1,
3393                (new_bus->dev_count - i) * sizeof(struct kvm_io_range));
3394
3395         rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
3396         synchronize_srcu_expedited(&kvm->srcu);
3397         kfree(bus);
3398         return r;
3399 }
3400
3401 static struct notifier_block kvm_cpu_notifier = {
3402         .notifier_call = kvm_cpu_hotplug,
3403 };
3404
3405 static int vm_stat_get(void *_offset, u64 *val)
3406 {
3407         unsigned offset = (long)_offset;
3408         struct kvm *kvm;
3409
3410         *val = 0;
3411         spin_lock(&kvm_lock);
3412         list_for_each_entry(kvm, &vm_list, vm_list)
3413                 *val += *(u32 *)((void *)kvm + offset);
3414         spin_unlock(&kvm_lock);
3415         return 0;
3416 }
3417
3418 DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, NULL, "%llu\n");
3419
3420 static int vcpu_stat_get(void *_offset, u64 *val)
3421 {
3422         unsigned offset = (long)_offset;
3423         struct kvm *kvm;
3424         struct kvm_vcpu *vcpu;
3425         int i;
3426
3427         *val = 0;
3428         spin_lock(&kvm_lock);
3429         list_for_each_entry(kvm, &vm_list, vm_list)
3430                 kvm_for_each_vcpu(i, vcpu, kvm)
3431                         *val += *(u32 *)((void *)vcpu + offset);
3432
3433         spin_unlock(&kvm_lock);
3434         return 0;
3435 }
3436
3437 DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, NULL, "%llu\n");
3438
3439 static const struct file_operations *stat_fops[] = {
3440         [KVM_STAT_VCPU] = &vcpu_stat_fops,
3441         [KVM_STAT_VM]   = &vm_stat_fops,
3442 };
3443
3444 static int kvm_init_debug(void)
3445 {
3446         int r = -EEXIST;
3447         struct kvm_stats_debugfs_item *p;
3448
3449         kvm_debugfs_dir = debugfs_create_dir("kvm", NULL);
3450         if (kvm_debugfs_dir == NULL)
3451                 goto out;
3452
3453         for (p = debugfs_entries; p->name; ++p) {
3454                 p->dentry = debugfs_create_file(p->name, 0444, kvm_debugfs_dir,
3455                                                 (void *)(long)p->offset,
3456                                                 stat_fops[p->kind]);
3457                 if (p->dentry == NULL)
3458                         goto out_dir;
3459         }
3460
3461         return 0;
3462
3463 out_dir:
3464         debugfs_remove_recursive(kvm_debugfs_dir);
3465 out:
3466         return r;
3467 }
3468
3469 static void kvm_exit_debug(void)
3470 {
3471         struct kvm_stats_debugfs_item *p;
3472
3473         for (p = debugfs_entries; p->name; ++p)
3474                 debugfs_remove(p->dentry);
3475         debugfs_remove(kvm_debugfs_dir);
3476 }
3477
3478 static int kvm_suspend(void)
3479 {
3480         if (kvm_usage_count)
3481                 hardware_disable_nolock(NULL);
3482         return 0;
3483 }
3484
3485 static void kvm_resume(void)
3486 {
3487         if (kvm_usage_count) {
3488                 WARN_ON(raw_spin_is_locked(&kvm_count_lock));
3489                 hardware_enable_nolock(NULL);
3490         }
3491 }
3492
3493 static struct syscore_ops kvm_syscore_ops = {
3494         .suspend = kvm_suspend,
3495         .resume = kvm_resume,
3496 };
3497
3498 static inline
3499 struct kvm_vcpu *preempt_notifier_to_vcpu(struct preempt_notifier *pn)
3500 {
3501         return container_of(pn, struct kvm_vcpu, preempt_notifier);
3502 }
3503
3504 static void kvm_sched_in(struct preempt_notifier *pn, int cpu)
3505 {
3506         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3507
3508         if (vcpu->preempted)
3509                 vcpu->preempted = false;
3510
3511         kvm_arch_sched_in(vcpu, cpu);
3512
3513         kvm_arch_vcpu_load(vcpu, cpu);
3514 }
3515
3516 static void kvm_sched_out(struct preempt_notifier *pn,
3517                           struct task_struct *next)
3518 {
3519         struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn);
3520
3521         if (current->state == TASK_RUNNING)
3522                 vcpu->preempted = true;
3523         kvm_arch_vcpu_put(vcpu);
3524 }
3525
3526 int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
3527                   struct module *module)
3528 {
3529         int r;
3530         int cpu;
3531
3532         r = kvm_arch_init(opaque);
3533         if (r)
3534                 goto out_fail;
3535
3536         /*
3537          * kvm_arch_init makes sure there's at most one caller
3538          * for architectures that support multiple implementations,
3539          * like intel and amd on x86.
3540          * kvm_arch_init must be called before kvm_irqfd_init to avoid creating
3541          * conflicts in case kvm is already setup for another implementation.
3542          */
3543         r = kvm_irqfd_init();
3544         if (r)
3545                 goto out_irqfd;
3546
3547         if (!zalloc_cpumask_var(&cpus_hardware_enabled, GFP_KERNEL)) {
3548                 r = -ENOMEM;
3549                 goto out_free_0;
3550         }
3551
3552         r = kvm_arch_hardware_setup();
3553         if (r < 0)
3554                 goto out_free_0a;
3555
3556         for_each_online_cpu(cpu) {
3557                 smp_call_function_single(cpu,
3558                                 kvm_arch_check_processor_compat,
3559                                 &r, 1);
3560                 if (r < 0)
3561                         goto out_free_1;
3562         }
3563
3564         r = register_cpu_notifier(&kvm_cpu_notifier);
3565         if (r)
3566                 goto out_free_2;
3567         register_reboot_notifier(&kvm_reboot_notifier);
3568
3569         /* A kmem cache lets us meet the alignment requirements of fx_save. */
3570         if (!vcpu_align)
3571                 vcpu_align = __alignof__(struct kvm_vcpu);
3572         kvm_vcpu_cache = kmem_cache_create("kvm_vcpu", vcpu_size, vcpu_align,
3573                                            0, NULL);
3574         if (!kvm_vcpu_cache) {
3575                 r = -ENOMEM;
3576                 goto out_free_3;
3577         }
3578
3579         r = kvm_async_pf_init();
3580         if (r)
3581                 goto out_free;
3582
3583         kvm_chardev_ops.owner = module;
3584         kvm_vm_fops.owner = module;
3585         kvm_vcpu_fops.owner = module;
3586
3587         r = misc_register(&kvm_dev);
3588         if (r) {
3589                 pr_err("kvm: misc device register failed\n");
3590                 goto out_unreg;
3591         }
3592
3593         register_syscore_ops(&kvm_syscore_ops);
3594
3595         kvm_preempt_ops.sched_in = kvm_sched_in;
3596         kvm_preempt_ops.sched_out = kvm_sched_out;
3597
3598         r = kvm_init_debug();
3599         if (r) {
3600                 pr_err("kvm: create debugfs files failed\n");
3601                 goto out_undebugfs;
3602         }
3603
3604         r = kvm_vfio_ops_init();
3605         WARN_ON(r);
3606
3607         return 0;
3608
3609 out_undebugfs:
3610         unregister_syscore_ops(&kvm_syscore_ops);
3611         misc_deregister(&kvm_dev);
3612 out_unreg:
3613         kvm_async_pf_deinit();
3614 out_free:
3615         kmem_cache_destroy(kvm_vcpu_cache);
3616 out_free_3:
3617         unregister_reboot_notifier(&kvm_reboot_notifier);
3618         unregister_cpu_notifier(&kvm_cpu_notifier);
3619 out_free_2:
3620 out_free_1:
3621         kvm_arch_hardware_unsetup();
3622 out_free_0a:
3623         free_cpumask_var(cpus_hardware_enabled);
3624 out_free_0:
3625         kvm_irqfd_exit();
3626 out_irqfd:
3627         kvm_arch_exit();
3628 out_fail:
3629         return r;
3630 }
3631 EXPORT_SYMBOL_GPL(kvm_init);
3632
3633 void kvm_exit(void)
3634 {
3635         kvm_exit_debug();
3636         misc_deregister(&kvm_dev);
3637         kmem_cache_destroy(kvm_vcpu_cache);
3638         kvm_async_pf_deinit();
3639         unregister_syscore_ops(&kvm_syscore_ops);
3640         unregister_reboot_notifier(&kvm_reboot_notifier);
3641         unregister_cpu_notifier(&kvm_cpu_notifier);
3642         on_each_cpu(hardware_disable_nolock, NULL, 1);
3643         kvm_arch_hardware_unsetup();
3644         kvm_arch_exit();
3645         kvm_irqfd_exit();
3646         free_cpumask_var(cpus_hardware_enabled);
3647         kvm_vfio_ops_exit();
3648 }
3649 EXPORT_SYMBOL_GPL(kvm_exit);