Add the rt linux 4.1.3-rt3 as base

[kvmfornfv.git] / kernel / arch / x86 / lib / memcpy_64.S

diff --git a/kernel/arch/x86/lib/memcpy_64.S b/kernel/arch/x86/lib/memcpy_64.S
new file mode 100644 (file)
index 0000000..b046664
--- /dev/null
+++ b/kernel/arch/x86/lib/memcpy_64.S
@@ -0,0 +1,182 @@
+/* Copyright 2002 Andi Kleen */
+
+#include <linux/linkage.h>
+#include <asm/cpufeature.h>
+#include <asm/dwarf2.h>
+#include <asm/alternative-asm.h>
+
+/*
+ * We build a jump to memcpy_orig by default which gets NOPped out on
+ * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
+ * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
+ * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
+ */
+
+.weak memcpy
+
+/*
+ * memcpy - Copy a memory block.
+ *
+ * Input:
+ *  rdi destination
+ *  rsi source
+ *  rdx count
+ *
+ * Output:
+ * rax original destination
+ */
+ENTRY(__memcpy)
+ENTRY(memcpy)
+       ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
+                     "jmp memcpy_erms", X86_FEATURE_ERMS
+
+       movq %rdi, %rax
+       movq %rdx, %rcx
+       shrq $3, %rcx
+       andl $7, %edx
+       rep movsq
+       movl %edx, %ecx
+       rep movsb
+       ret
+ENDPROC(memcpy)
+ENDPROC(__memcpy)
+
+/*
+ * memcpy_erms() - enhanced fast string memcpy. This is faster and
+ * simpler than memcpy. Use memcpy_erms when possible.
+ */
+ENTRY(memcpy_erms)
+       movq %rdi, %rax
+       movq %rdx, %rcx
+       rep movsb
+       ret
+ENDPROC(memcpy_erms)
+
+ENTRY(memcpy_orig)
+       CFI_STARTPROC
+       movq %rdi, %rax
+
+       cmpq $0x20, %rdx
+       jb .Lhandle_tail
+
+       /*
+        * We check whether memory false dependence could occur,
+        * then jump to corresponding copy mode.
+        */
+       cmp  %dil, %sil
+       jl .Lcopy_backward
+       subq $0x20, %rdx
+.Lcopy_forward_loop:
+       subq $0x20,     %rdx
+
+       /*
+        * Move in blocks of 4x8 bytes:
+        */
+       movq 0*8(%rsi), %r8
+       movq 1*8(%rsi), %r9
+       movq 2*8(%rsi), %r10
+       movq 3*8(%rsi), %r11
+       leaq 4*8(%rsi), %rsi
+
+       movq %r8,       0*8(%rdi)
+       movq %r9,       1*8(%rdi)
+       movq %r10,      2*8(%rdi)
+       movq %r11,      3*8(%rdi)
+       leaq 4*8(%rdi), %rdi
+       jae  .Lcopy_forward_loop
+       addl $0x20,     %edx
+       jmp  .Lhandle_tail
+
+.Lcopy_backward:
+       /*
+        * Calculate copy position to tail.
+        */
+       addq %rdx,      %rsi
+       addq %rdx,      %rdi
+       subq $0x20,     %rdx
+       /*
+        * At most 3 ALU operations in one cycle,
+        * so append NOPS in the same 16 bytes trunk.
+        */
+       .p2align 4
+.Lcopy_backward_loop:
+       subq $0x20,     %rdx
+       movq -1*8(%rsi),        %r8
+       movq -2*8(%rsi),        %r9
+       movq -3*8(%rsi),        %r10
+       movq -4*8(%rsi),        %r11
+       leaq -4*8(%rsi),        %rsi
+       movq %r8,               -1*8(%rdi)
+       movq %r9,               -2*8(%rdi)
+       movq %r10,              -3*8(%rdi)
+       movq %r11,              -4*8(%rdi)
+       leaq -4*8(%rdi),        %rdi
+       jae  .Lcopy_backward_loop
+
+       /*
+        * Calculate copy position to head.
+        */
+       addl $0x20,     %edx
+       subq %rdx,      %rsi
+       subq %rdx,      %rdi
+.Lhandle_tail:
+       cmpl $16,       %edx
+       jb   .Lless_16bytes
+
+       /*
+        * Move data from 16 bytes to 31 bytes.
+        */
+       movq 0*8(%rsi), %r8
+       movq 1*8(%rsi), %r9
+       movq -2*8(%rsi, %rdx),  %r10
+       movq -1*8(%rsi, %rdx),  %r11
+       movq %r8,       0*8(%rdi)
+       movq %r9,       1*8(%rdi)
+       movq %r10,      -2*8(%rdi, %rdx)
+       movq %r11,      -1*8(%rdi, %rdx)
+       retq
+       .p2align 4
+.Lless_16bytes:
+       cmpl $8,        %edx
+       jb   .Lless_8bytes
+       /*
+        * Move data from 8 bytes to 15 bytes.
+        */
+       movq 0*8(%rsi), %r8
+       movq -1*8(%rsi, %rdx),  %r9
+       movq %r8,       0*8(%rdi)
+       movq %r9,       -1*8(%rdi, %rdx)
+       retq
+       .p2align 4
+.Lless_8bytes:
+       cmpl $4,        %edx
+       jb   .Lless_3bytes
+
+       /*
+        * Move data from 4 bytes to 7 bytes.
+        */
+       movl (%rsi), %ecx
+       movl -4(%rsi, %rdx), %r8d
+       movl %ecx, (%rdi)
+       movl %r8d, -4(%rdi, %rdx)
+       retq
+       .p2align 4
+.Lless_3bytes:
+       subl $1, %edx
+       jb .Lend
+       /*
+        * Move data from 1 bytes to 3 bytes.
+        */
+       movzbl (%rsi), %ecx
+       jz .Lstore_1byte
+       movzbq 1(%rsi), %r8
+       movzbq (%rsi, %rdx), %r9
+       movb %r8b, 1(%rdi)
+       movb %r9b, (%rdi, %rdx)
+.Lstore_1byte:
+       movb %cl, (%rdi)
+
+.Lend:
+       retq
+       CFI_ENDPROC
+ENDPROC(memcpy_orig)