Add the rt linux 4.1.3-rt3 as base
[kvmfornfv.git] / kernel / arch / x86 / include / asm / barrier.h
1 #ifndef _ASM_X86_BARRIER_H
2 #define _ASM_X86_BARRIER_H
3
4 #include <asm/alternative.h>
5 #include <asm/nops.h>
6
7 /*
8  * Force strict CPU ordering.
9  * And yes, this is required on UP too when we're talking
10  * to devices.
11  */
12
13 #ifdef CONFIG_X86_32
14 /*
15  * Some non-Intel clones support out of order store. wmb() ceases to be a
16  * nop for these.
17  */
18 #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
19 #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
20 #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)
21 #else
22 #define mb()    asm volatile("mfence":::"memory")
23 #define rmb()   asm volatile("lfence":::"memory")
24 #define wmb()   asm volatile("sfence" ::: "memory")
25 #endif
26
27 #ifdef CONFIG_X86_PPRO_FENCE
28 #define dma_rmb()       rmb()
29 #else
30 #define dma_rmb()       barrier()
31 #endif
32 #define dma_wmb()       barrier()
33
34 #ifdef CONFIG_SMP
35 #define smp_mb()        mb()
36 #define smp_rmb()       dma_rmb()
37 #define smp_wmb()       barrier()
38 #define set_mb(var, value) do { (void)xchg(&var, value); } while (0)
39 #else /* !SMP */
40 #define smp_mb()        barrier()
41 #define smp_rmb()       barrier()
42 #define smp_wmb()       barrier()
43 #define set_mb(var, value) do { var = value; barrier(); } while (0)
44 #endif /* SMP */
45
46 #define read_barrier_depends()          do { } while (0)
47 #define smp_read_barrier_depends()      do { } while (0)
48
49 #if defined(CONFIG_X86_PPRO_FENCE)
50
51 /*
52  * For this option x86 doesn't have a strong TSO memory
53  * model and we should fall back to full barriers.
54  */
55
56 #define smp_store_release(p, v)                                         \
57 do {                                                                    \
58         compiletime_assert_atomic_type(*p);                             \
59         smp_mb();                                                       \
60         ACCESS_ONCE(*p) = (v);                                          \
61 } while (0)
62
63 #define smp_load_acquire(p)                                             \
64 ({                                                                      \
65         typeof(*p) ___p1 = ACCESS_ONCE(*p);                             \
66         compiletime_assert_atomic_type(*p);                             \
67         smp_mb();                                                       \
68         ___p1;                                                          \
69 })
70
71 #else /* regular x86 TSO memory ordering */
72
73 #define smp_store_release(p, v)                                         \
74 do {                                                                    \
75         compiletime_assert_atomic_type(*p);                             \
76         barrier();                                                      \
77         ACCESS_ONCE(*p) = (v);                                          \
78 } while (0)
79
80 #define smp_load_acquire(p)                                             \
81 ({                                                                      \
82         typeof(*p) ___p1 = ACCESS_ONCE(*p);                             \
83         compiletime_assert_atomic_type(*p);                             \
84         barrier();                                                      \
85         ___p1;                                                          \
86 })
87
88 #endif
89
90 /* Atomic operations are already serializing on x86 */
91 #define smp_mb__before_atomic() barrier()
92 #define smp_mb__after_atomic()  barrier()
93
94 /*
95  * Stop RDTSC speculation. This is needed when you need to use RDTSC
96  * (or get_cycles or vread that possibly accesses the TSC) in a defined
97  * code region.
98  */
99 static __always_inline void rdtsc_barrier(void)
100 {
101         alternative_2("", "mfence", X86_FEATURE_MFENCE_RDTSC,
102                           "lfence", X86_FEATURE_LFENCE_RDTSC);
103 }
104
105 #endif /* _ASM_X86_BARRIER_H */