These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / arch / arm64 / lib / bitops.S

diff --git a/kernel/arch/arm64/lib/bitops.S b/kernel/arch/arm64/lib/bitops.S
index 7dac371..43ac736 100644 (file)
--- a/kernel/arch/arm64/lib/bitops.S
+++ b/kernel/arch/arm64/lib/bitops.S
@@ -18,52 +18,59 @@
 
 #include <linux/linkage.h>
 #include <asm/assembler.h>
+#include <asm/lse.h>
 
 /*
  * x0: bits 5:0  bit offset
  *     bits 31:6 word offset
  * x1: address
  */
-       .macro  bitop, name, instr
+       .macro  bitop, name, llsc, lse
 ENTRY( \name   )
        and     w3, w0, #63             // Get bit offset
        eor     w0, w0, w3              // Clear low bits
        mov     x2, #1
        add     x1, x1, x0, lsr #3      // Get word offset
+alt_lse "      prfm    pstl1strm, [x1]",       "nop"
        lsl     x3, x2, x3              // Create mask
-1:     ldxr    x2, [x1]
-       \instr  x2, x2, x3
-       stxr    w0, x2, [x1]
-       cbnz    w0, 1b
+
+alt_lse        "1:     ldxr    x2, [x1]",              "\lse   x3, [x1]"
+alt_lse        "       \llsc   x2, x2, x3",            "nop"
+alt_lse        "       stxr    w0, x2, [x1]",          "nop"
+alt_lse        "       cbnz    w0, 1b",                "nop"
+
        ret
 ENDPROC(\name  )
        .endm
 
-       .macro  testop, name, instr
+       .macro  testop, name, llsc, lse
 ENTRY( \name   )
        and     w3, w0, #63             // Get bit offset
        eor     w0, w0, w3              // Clear low bits
        mov     x2, #1
        add     x1, x1, x0, lsr #3      // Get word offset
+alt_lse "      prfm    pstl1strm, [x1]",       "nop"
        lsl     x4, x2, x3              // Create mask
-1:     ldxr    x2, [x1]
-       lsr     x0, x2, x3              // Save old value of bit
-       \instr  x2, x2, x4              // toggle bit
-       stlxr   w5, x2, [x1]
-       cbnz    w5, 1b
-       dmb     ish
+
+alt_lse        "1:     ldxr    x2, [x1]",              "\lse   x4, x2, [x1]"
+       lsr     x0, x2, x3
+alt_lse        "       \llsc   x2, x2, x4",            "nop"
+alt_lse        "       stlxr   w5, x2, [x1]",          "nop"
+alt_lse        "       cbnz    w5, 1b",                "nop"
+alt_lse        "       dmb     ish",                   "nop"
+
        and     x0, x0, #1
-3:     ret
+       ret
 ENDPROC(\name  )
        .endm
 
 /*
  * Atomic bit operations.
  */
-       bitop   change_bit, eor
-       bitop   clear_bit, bic
-       bitop   set_bit, orr
+       bitop   change_bit, eor, steor
+       bitop   clear_bit, bic, stclr
+       bitop   set_bit, orr, stset
 
-       testop  test_and_change_bit, eor
-       testop  test_and_clear_bit, bic
-       testop  test_and_set_bit, orr
+       testop  test_and_change_bit, eor, ldeoral
+       testop  test_and_clear_bit, bic, ldclral
+       testop  test_and_set_bit, orr, ldsetal