kernel/arch/powerpc/include/asm/cmpxchg.h

   1 #ifndef _ASM_POWERPC_CMPXCHG_H_
   2 #define _ASM_POWERPC_CMPXCHG_H_
   3
   4 #ifdef __KERNEL__
   5 #include <linux/compiler.h>
   6 #include <asm/synch.h>
   7 #include <asm/asm-compat.h>
   8
   9 /*
  10  * Atomic exchange
  11  *
  12  * Changes the memory location '*ptr' to be val and returns
  13  * the previous value stored there.
  14  */
  15 static __always_inline unsigned long
  16 __xchg_u32(volatile void *p, unsigned long val)
  17 {
  18         unsigned long prev;
  19
  20         __asm__ __volatile__(
  21         PPC_ATOMIC_ENTRY_BARRIER
  22 "1:     lwarx   %0,0,%2 \n"
  23         PPC405_ERR77(0,%2)
  24 "       stwcx.  %3,0,%2 \n\
  25         bne-    1b"
  26         PPC_ATOMIC_EXIT_BARRIER
  27         : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
  28         : "r" (p), "r" (val)
  29         : "cc", "memory");
  30
  31         return prev;
  32 }
  33
  34 /*
  35  * Atomic exchange
  36  *
  37  * Changes the memory location '*ptr' to be val and returns
  38  * the previous value stored there.
  39  */
  40 static __always_inline unsigned long
  41 __xchg_u32_local(volatile void *p, unsigned long val)
  42 {
  43         unsigned long prev;
  44
  45         __asm__ __volatile__(
  46 "1:     lwarx   %0,0,%2 \n"
  47         PPC405_ERR77(0,%2)
  48 "       stwcx.  %3,0,%2 \n\
  49         bne-    1b"
  50         : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
  51         : "r" (p), "r" (val)
  52         : "cc", "memory");
  53
  54         return prev;
  55 }
  56
  57 #ifdef CONFIG_PPC64
  58 static __always_inline unsigned long
  59 __xchg_u64(volatile void *p, unsigned long val)
  60 {
  61         unsigned long prev;
  62
  63         __asm__ __volatile__(
  64         PPC_ATOMIC_ENTRY_BARRIER
  65 "1:     ldarx   %0,0,%2 \n"
  66         PPC405_ERR77(0,%2)
  67 "       stdcx.  %3,0,%2 \n\
  68         bne-    1b"
  69         PPC_ATOMIC_EXIT_BARRIER
  70         : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
  71         : "r" (p), "r" (val)
  72         : "cc", "memory");
  73
  74         return prev;
  75 }
  76
  77 static __always_inline unsigned long
  78 __xchg_u64_local(volatile void *p, unsigned long val)
  79 {
  80         unsigned long prev;
  81
  82         __asm__ __volatile__(
  83 "1:     ldarx   %0,0,%2 \n"
  84         PPC405_ERR77(0,%2)
  85 "       stdcx.  %3,0,%2 \n\
  86         bne-    1b"
  87         : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
  88         : "r" (p), "r" (val)
  89         : "cc", "memory");
  90
  91         return prev;
  92 }
  93 #endif
  94
  95 /*
  96  * This function doesn't exist, so you'll get a linker error
  97  * if something tries to do an invalid xchg().
  98  */
  99 extern void __xchg_called_with_bad_pointer(void);
 100
 101 static __always_inline unsigned long
 102 __xchg(volatile void *ptr, unsigned long x, unsigned int size)
 103 {
 104         switch (size) {
 105         case 4:
 106                 return __xchg_u32(ptr, x);
 107 #ifdef CONFIG_PPC64
 108         case 8:
 109                 return __xchg_u64(ptr, x);
 110 #endif
 111         }
 112         __xchg_called_with_bad_pointer();
 113         return x;
 114 }
 115
 116 static __always_inline unsigned long
 117 __xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
 118 {
 119         switch (size) {
 120         case 4:
 121                 return __xchg_u32_local(ptr, x);
 122 #ifdef CONFIG_PPC64
 123         case 8:
 124                 return __xchg_u64_local(ptr, x);
 125 #endif
 126         }
 127         __xchg_called_with_bad_pointer();
 128         return x;
 129 }
 130 #define xchg(ptr,x)                                                          \
 131   ({                                                                         \
 132      __typeof__(*(ptr)) _x_ = (x);                                           \
 133      (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
 134   })
 135
 136 #define xchg_local(ptr,x)                                                    \
 137   ({                                                                         \
 138      __typeof__(*(ptr)) _x_ = (x);                                           \
 139      (__typeof__(*(ptr))) __xchg_local((ptr),                                \
 140                 (unsigned long)_x_, sizeof(*(ptr)));                         \
 141   })
 142
 143 /*
 144  * Compare and exchange - if *p == old, set it to new,
 145  * and return the old value of *p.
 146  */
 147
 148 static __always_inline unsigned long
 149 __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
 150 {
 151         unsigned int prev;
 152
 153         __asm__ __volatile__ (
 154         PPC_ATOMIC_ENTRY_BARRIER
 155 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 156         cmpw    0,%0,%3\n\
 157         bne-    2f\n"
 158         PPC405_ERR77(0,%2)
 159 "       stwcx.  %4,0,%2\n\
 160         bne-    1b"
 161         PPC_ATOMIC_EXIT_BARRIER
 162         "\n\
 163 2:"
 164         : "=&r" (prev), "+m" (*p)
 165         : "r" (p), "r" (old), "r" (new)
 166         : "cc", "memory");
 167
 168         return prev;
 169 }
 170
 171 static __always_inline unsigned long
 172 __cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
 173                         unsigned long new)
 174 {
 175         unsigned int prev;
 176
 177         __asm__ __volatile__ (
 178 "1:     lwarx   %0,0,%2         # __cmpxchg_u32\n\
 179         cmpw    0,%0,%3\n\
 180         bne-    2f\n"
 181         PPC405_ERR77(0,%2)
 182 "       stwcx.  %4,0,%2\n\
 183         bne-    1b"
 184         "\n\
 185 2:"
 186         : "=&r" (prev), "+m" (*p)
 187         : "r" (p), "r" (old), "r" (new)
 188         : "cc", "memory");
 189
 190         return prev;
 191 }
 192
 193 #ifdef CONFIG_PPC64
 194 static __always_inline unsigned long
 195 __cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
 196 {
 197         unsigned long prev;
 198
 199         __asm__ __volatile__ (
 200         PPC_ATOMIC_ENTRY_BARRIER
 201 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 202         cmpd    0,%0,%3\n\
 203         bne-    2f\n\
 204         stdcx.  %4,0,%2\n\
 205         bne-    1b"
 206         PPC_ATOMIC_EXIT_BARRIER
 207         "\n\
 208 2:"
 209         : "=&r" (prev), "+m" (*p)
 210         : "r" (p), "r" (old), "r" (new)
 211         : "cc", "memory");
 212
 213         return prev;
 214 }
 215
 216 static __always_inline unsigned long
 217 __cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
 218                         unsigned long new)
 219 {
 220         unsigned long prev;
 221
 222         __asm__ __volatile__ (
 223 "1:     ldarx   %0,0,%2         # __cmpxchg_u64\n\
 224         cmpd    0,%0,%3\n\
 225         bne-    2f\n\
 226         stdcx.  %4,0,%2\n\
 227         bne-    1b"
 228         "\n\
 229 2:"
 230         : "=&r" (prev), "+m" (*p)
 231         : "r" (p), "r" (old), "r" (new)
 232         : "cc", "memory");
 233
 234         return prev;
 235 }
 236 #endif
 237
 238 /* This function doesn't exist, so you'll get a linker error
 239    if something tries to do an invalid cmpxchg().  */
 240 extern void __cmpxchg_called_with_bad_pointer(void);
 241
 242 static __always_inline unsigned long
 243 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
 244           unsigned int size)
 245 {
 246         switch (size) {
 247         case 4:
 248                 return __cmpxchg_u32(ptr, old, new);
 249 #ifdef CONFIG_PPC64
 250         case 8:
 251                 return __cmpxchg_u64(ptr, old, new);
 252 #endif
 253         }
 254         __cmpxchg_called_with_bad_pointer();
 255         return old;
 256 }
 257
 258 static __always_inline unsigned long
 259 __cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
 260           unsigned int size)
 261 {
 262         switch (size) {
 263         case 4:
 264                 return __cmpxchg_u32_local(ptr, old, new);
 265 #ifdef CONFIG_PPC64
 266         case 8:
 267                 return __cmpxchg_u64_local(ptr, old, new);
 268 #endif
 269         }
 270         __cmpxchg_called_with_bad_pointer();
 271         return old;
 272 }
 273
 274 #define cmpxchg(ptr, o, n)                                               \
 275   ({                                                                     \
 276      __typeof__(*(ptr)) _o_ = (o);                                       \
 277      __typeof__(*(ptr)) _n_ = (n);                                       \
 278      (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_,           \
 279                                     (unsigned long)_n_, sizeof(*(ptr))); \
 280   })
 281
 282
 283 #define cmpxchg_local(ptr, o, n)                                         \
 284   ({                                                                     \
 285      __typeof__(*(ptr)) _o_ = (o);                                       \
 286      __typeof__(*(ptr)) _n_ = (n);                                       \
 287      (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_,     \
 288                                     (unsigned long)_n_, sizeof(*(ptr))); \
 289   })
 290
 291 #ifdef CONFIG_PPC64
 292 #define cmpxchg64(ptr, o, n)                                            \
 293   ({                                                                    \
 294         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 295         cmpxchg((ptr), (o), (n));                                       \
 296   })
 297 #define cmpxchg64_local(ptr, o, n)                                      \
 298   ({                                                                    \
 299         BUILD_BUG_ON(sizeof(*(ptr)) != 8);                              \
 300         cmpxchg_local((ptr), (o), (n));                                 \
 301   })
 302 #define cmpxchg64_relaxed       cmpxchg64_local
 303 #else
 304 #include <asm-generic/cmpxchg-local.h>
 305 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
 306 #endif
 307
 308 #endif /* __KERNEL__ */
 309 #endif /* _ASM_POWERPC_CMPXCHG_H_ */