These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / arch / x86 / kernel / kvm.c

diff --git a/kernel/arch/x86/kernel/kvm.c b/kernel/arch/x86/kernel/kvm.c
index 9435620..8079508 100644 (file)
--- a/kernel/arch/x86/kernel/kvm.c
+++ b/kernel/arch/x86/kernel/kvm.c
@@ -36,6 +36,7 @@
 #include <linux/kprobes.h>
 #include <linux/debugfs.h>
 #include <linux/nmi.h>
+#include <linux/swait.h>
 #include <asm/timer.h>
 #include <asm/cpu.h>
 #include <asm/traps.h>
@@ -91,14 +92,14 @@ static void kvm_io_delay(void)
 
 struct kvm_task_sleep_node {
        struct hlist_node link;
-       wait_queue_head_t wq;
+       struct swait_queue_head wq;
        u32 token;
        int cpu;
        bool halted;
 };
 
 static struct kvm_task_sleep_head {
-       spinlock_t lock;
+       raw_spinlock_t lock;
        struct hlist_head list;
 } async_pf_sleepers[KVM_TASK_SLEEP_HASHSIZE];
 
@@ -122,17 +123,17 @@ void kvm_async_pf_task_wait(u32 token)
        u32 key = hash_32(token, KVM_TASK_SLEEP_HASHBITS);
        struct kvm_task_sleep_head *b = &async_pf_sleepers[key];
        struct kvm_task_sleep_node n, *e;
-       DEFINE_WAIT(wait);
+       DECLARE_SWAITQUEUE(wait);
 
        rcu_irq_enter();
 
-       spin_lock(&b->lock);
+       raw_spin_lock(&b->lock);
        e = _find_apf_task(b, token);
        if (e) {
                /* dummy entry exist -> wake up was delivered ahead of PF */
                hlist_del(&e->link);
                kfree(e);
-               spin_unlock(&b->lock);
+               raw_spin_unlock(&b->lock);
 
                rcu_irq_exit();
                return;
@@ -141,13 +142,13 @@ void kvm_async_pf_task_wait(u32 token)
        n.token = token;
        n.cpu = smp_processor_id();
        n.halted = is_idle_task(current) || preempt_count() > 1;
-       init_waitqueue_head(&n.wq);
+       init_swait_queue_head(&n.wq);
        hlist_add_head(&n.link, &b->list);
-       spin_unlock(&b->lock);
+       raw_spin_unlock(&b->lock);
 
        for (;;) {
                if (!n.halted)
-                       prepare_to_wait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
+                       prepare_to_swait(&n.wq, &wait, TASK_UNINTERRUPTIBLE);
                if (hlist_unhashed(&n.link))
                        break;
 
@@ -166,7 +167,7 @@ void kvm_async_pf_task_wait(u32 token)
                }
        }
        if (!n.halted)
-               finish_wait(&n.wq, &wait);
+               finish_swait(&n.wq, &wait);
 
        rcu_irq_exit();
        return;
@@ -178,8 +179,8 @@ static void apf_task_wake_one(struct kvm_task_sleep_node *n)
        hlist_del_init(&n->link);
        if (n->halted)
                smp_send_reschedule(n->cpu);
-       else if (waitqueue_active(&n->wq))
-               wake_up(&n->wq);
+       else if (swait_active(&n->wq))
+               swake_up(&n->wq);
 }
 
 static void apf_task_wake_all(void)
@@ -189,14 +190,14 @@ static void apf_task_wake_all(void)
        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++) {
                struct hlist_node *p, *next;
                struct kvm_task_sleep_head *b = &async_pf_sleepers[i];
-               spin_lock(&b->lock);
+               raw_spin_lock(&b->lock);
                hlist_for_each_safe(p, next, &b->list) {
                        struct kvm_task_sleep_node *n =
                                hlist_entry(p, typeof(*n), link);
                        if (n->cpu == smp_processor_id())
                                apf_task_wake_one(n);
                }
-               spin_unlock(&b->lock);
+               raw_spin_unlock(&b->lock);
        }
 }
 
@@ -212,7 +213,7 @@ void kvm_async_pf_task_wake(u32 token)
        }
 
 again:
-       spin_lock(&b->lock);
+       raw_spin_lock(&b->lock);
        n = _find_apf_task(b, token);
        if (!n) {
                /*
@@ -225,17 +226,17 @@ again:
                         * Allocation failed! Busy wait while other cpu
                         * handles async PF.
                         */
-                       spin_unlock(&b->lock);
+                       raw_spin_unlock(&b->lock);
                        cpu_relax();
                        goto again;
                }
                n->token = token;
                n->cpu = smp_processor_id();
-               init_waitqueue_head(&n->wq);
+               init_swait_queue_head(&n->wq);
                hlist_add_head(&n->link, &b->list);
        } else
                apf_task_wake_one(n);
-       spin_unlock(&b->lock);
+       raw_spin_unlock(&b->lock);
        return;
 }
 EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake);
@@ -331,7 +332,7 @@ static void kvm_guest_apic_eoi_write(u32 reg, u32 val)
        apic_write(APIC_EOI, APIC_EOI_ACK);
 }
 
-void kvm_guest_cpu_init(void)
+static void kvm_guest_cpu_init(void)
 {
        if (!kvm_para_available())
                return;
@@ -486,7 +487,7 @@ void __init kvm_guest_init(void)
        paravirt_ops_setup();
        register_reboot_notifier(&kvm_pv_reboot_nb);
        for (i = 0; i < KVM_TASK_SLEEP_HASHSIZE; i++)
-               spin_lock_init(&async_pf_sleepers[i].lock);
+               raw_spin_lock_init(&async_pf_sleepers[i].lock);
        if (kvm_para_has_feature(KVM_FEATURE_ASYNC_PF))
                x86_init.irqs.trap_init = kvm_apf_trap_init;
 
@@ -584,6 +585,39 @@ static void kvm_kick_cpu(int cpu)
        kvm_hypercall2(KVM_HC_KICK_CPU, flags, apicid);
 }
 
+
+#ifdef CONFIG_QUEUED_SPINLOCKS
+
+#include <asm/qspinlock.h>
+
+static void kvm_wait(u8 *ptr, u8 val)
+{
+       unsigned long flags;
+
+       if (in_nmi())
+               return;
+
+       local_irq_save(flags);
+
+       if (READ_ONCE(*ptr) != val)
+               goto out;
+
+       /*
+        * halt until it's our turn and kicked. Note that we do safe halt
+        * for irq enabled case to avoid hang when lock info is overwritten
+        * in irq spinlock slowpath and no spurious interrupt occur to save us.
+        */
+       if (arch_irqs_disabled_flags(flags))
+               halt();
+       else
+               safe_halt();
+
+out:
+       local_irq_restore(flags);
+}
+
+#else /* !CONFIG_QUEUED_SPINLOCKS */
+
 enum kvm_contention_stat {
        TAKEN_SLOW,
        TAKEN_SLOW_PICKUP,
@@ -655,7 +689,7 @@ static inline void spin_time_accum_blocked(u64 start)
 static struct dentry *d_spin_debug;
 static struct dentry *d_kvm_debug;
 
-struct dentry *kvm_init_debugfs(void)
+static struct dentry *kvm_init_debugfs(void)
 {
        d_kvm_debug = debugfs_create_dir("kvm-guest", NULL);
        if (!d_kvm_debug)
@@ -817,6 +851,8 @@ static void kvm_unlock_kick(struct arch_spinlock *lock, __ticket_t ticket)
        }
 }
 
+#endif /* !CONFIG_QUEUED_SPINLOCKS */
+
 /*
  * Setup pv_lock_ops to exploit KVM_FEATURE_PV_UNHALT if present.
  */
@@ -828,8 +864,16 @@ void __init kvm_spinlock_init(void)
        if (!kvm_para_has_feature(KVM_FEATURE_PV_UNHALT))
                return;
 
+#ifdef CONFIG_QUEUED_SPINLOCKS
+       __pv_init_lock_hash();
+       pv_lock_ops.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
+       pv_lock_ops.queued_spin_unlock = PV_CALLEE_SAVE(__pv_queued_spin_unlock);
+       pv_lock_ops.wait = kvm_wait;
+       pv_lock_ops.kick = kvm_kick_cpu;
+#else /* !CONFIG_QUEUED_SPINLOCKS */
        pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(kvm_lock_spinning);
        pv_lock_ops.unlock_kick = kvm_unlock_kick;
+#endif
 }
 
 static __init int kvm_spinlock_init_jump(void)