These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / arch / x86 / mm / tlb.c

diff --git a/kernel/arch/x86/mm/tlb.c b/kernel/arch/x86/mm/tlb.c
index 90b924a..8f4cc3d 100644 (file)
--- a/kernel/arch/x86/mm/tlb.c
+++ b/kernel/arch/x86/mm/tlb.c
@@ -140,6 +140,7 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
        info.flush_end = end;
 
        count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
+       trace_tlb_flush(TLB_REMOTE_SEND_IPI, end - start);
        if (is_uv_system()) {
                unsigned int cpu;
 
@@ -160,7 +161,10 @@ void flush_tlb_current_task(void)
        preempt_disable();
 
        count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
+
+       /* This is an implicit full barrier that synchronizes with switch_mm. */
        local_flush_tlb();
+
        trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
                flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
@@ -187,17 +191,29 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
        unsigned long base_pages_to_flush = TLB_FLUSH_ALL;
 
        preempt_disable();
-       if (current->active_mm != mm)
+       if (current->active_mm != mm) {
+               /* Synchronize with switch_mm. */
+               smp_mb();
+
                goto out;
+       }
 
        if (!current->mm) {
                leave_mm(smp_processor_id());
+
+               /* Synchronize with switch_mm. */
+               smp_mb();
+
                goto out;
        }
 
        if ((end != TLB_FLUSH_ALL) && !(vmflag & VM_HUGETLB))
                base_pages_to_flush = (end - start) >> PAGE_SHIFT;
 
+       /*
+        * Both branches below are implicit full barriers (MOV to CR or
+        * INVLPG) that synchronize with switch_mm.
+        */
        if (base_pages_to_flush > tlb_single_page_flush_ceiling) {
                base_pages_to_flush = TLB_FLUSH_ALL;
                count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
@@ -227,10 +243,18 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
        preempt_disable();
 
        if (current->active_mm == mm) {
-               if (current->mm)
+               if (current->mm) {
+                       /*
+                        * Implicit full barrier (INVLPG) that synchronizes
+                        * with switch_mm.
+                        */
                        __flush_tlb_one(start);
-               else
+               } else {
                        leave_mm(smp_processor_id());
+
+                       /* Synchronize with switch_mm. */
+                       smp_mb();
+               }
        }
 
        if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)