Add the rt linux 4.1.3-rt3 as base

[kvmfornfv.git] / kernel / arch / x86 / lib / csum-copy_64.S

diff --git a/kernel/arch/x86/lib/csum-copy_64.S b/kernel/arch/x86/lib/csum-copy_64.S
new file mode 100644 (file)
index 0000000..9734182
--- /dev/null
+++ b/kernel/arch/x86/lib/csum-copy_64.S
@@ -0,0 +1,241 @@
+/*
+ * Copyright 2002, 2003 Andi Kleen, SuSE Labs.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file COPYING in the main directory of this archive
+ * for more details. No warranty for anything given at all.
+ */
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/errno.h>
+#include <asm/asm.h>
+
+/*
+ * Checksum copy with exception handling.
+ * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
+ * destination is zeroed.
+ *
+ * Input
+ * rdi  source
+ * rsi  destination
+ * edx  len (32bit)
+ * ecx  sum (32bit)
+ * r8   src_err_ptr (int)
+ * r9   dst_err_ptr (int)
+ *
+ * Output
+ * eax  64bit sum. undefined in case of exception.
+ *
+ * Wrappers need to take care of valid exception sum and zeroing.
+ * They also should align source or destination to 8 bytes.
+ */
+
+       .macro source
+10:
+       _ASM_EXTABLE(10b, .Lbad_source)
+       .endm
+
+       .macro dest
+20:
+       _ASM_EXTABLE(20b, .Lbad_dest)
+       .endm
+
+       .macro ignore L=.Lignore
+30:
+       _ASM_EXTABLE(30b, \L)
+       .endm
+
+
+ENTRY(csum_partial_copy_generic)
+       CFI_STARTPROC
+       cmpl    $3*64, %edx
+       jle     .Lignore
+
+.Lignore:
+       subq  $7*8, %rsp
+       CFI_ADJUST_CFA_OFFSET 7*8
+       movq  %rbx, 2*8(%rsp)
+       CFI_REL_OFFSET rbx, 2*8
+       movq  %r12, 3*8(%rsp)
+       CFI_REL_OFFSET r12, 3*8
+       movq  %r14, 4*8(%rsp)
+       CFI_REL_OFFSET r14, 4*8
+       movq  %r13, 5*8(%rsp)
+       CFI_REL_OFFSET r13, 5*8
+       movq  %rbp, 6*8(%rsp)
+       CFI_REL_OFFSET rbp, 6*8
+
+       movq  %r8, (%rsp)
+       movq  %r9, 1*8(%rsp)
+
+       movl  %ecx, %eax
+       movl  %edx, %ecx
+
+       xorl  %r9d, %r9d
+       movq  %rcx, %r12
+
+       shrq  $6, %r12
+       jz      .Lhandle_tail       /* < 64 */
+
+       clc
+
+       /* main loop. clear in 64 byte blocks */
+       /* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
+       /* r11: temp3, rdx: temp4, r12 loopcnt */
+       /* r10: temp5, rbp: temp6, r14 temp7, r13 temp8 */
+       .p2align 4
+.Lloop:
+       source
+       movq  (%rdi), %rbx
+       source
+       movq  8(%rdi), %r8
+       source
+       movq  16(%rdi), %r11
+       source
+       movq  24(%rdi), %rdx
+
+       source
+       movq  32(%rdi), %r10
+       source
+       movq  40(%rdi), %rbp
+       source
+       movq  48(%rdi), %r14
+       source
+       movq  56(%rdi), %r13
+
+       ignore 2f
+       prefetcht0 5*64(%rdi)
+2:
+       adcq  %rbx, %rax
+       adcq  %r8, %rax
+       adcq  %r11, %rax
+       adcq  %rdx, %rax
+       adcq  %r10, %rax
+       adcq  %rbp, %rax
+       adcq  %r14, %rax
+       adcq  %r13, %rax
+
+       decl %r12d
+
+       dest
+       movq %rbx, (%rsi)
+       dest
+       movq %r8, 8(%rsi)
+       dest
+       movq %r11, 16(%rsi)
+       dest
+       movq %rdx, 24(%rsi)
+
+       dest
+       movq %r10, 32(%rsi)
+       dest
+       movq %rbp, 40(%rsi)
+       dest
+       movq %r14, 48(%rsi)
+       dest
+       movq %r13, 56(%rsi)
+
+3:
+
+       leaq 64(%rdi), %rdi
+       leaq 64(%rsi), %rsi
+
+       jnz     .Lloop
+
+       adcq  %r9, %rax
+
+       /* do last up to 56 bytes */
+.Lhandle_tail:
+       /* ecx: count */
+       movl %ecx, %r10d
+       andl $63, %ecx
+       shrl $3, %ecx
+       jz      .Lfold
+       clc
+       .p2align 4
+.Lloop_8:
+       source
+       movq (%rdi), %rbx
+       adcq %rbx, %rax
+       decl %ecx
+       dest
+       movq %rbx, (%rsi)
+       leaq 8(%rsi), %rsi /* preserve carry */
+       leaq 8(%rdi), %rdi
+       jnz     .Lloop_8
+       adcq %r9, %rax  /* add in carry */
+
+.Lfold:
+       /* reduce checksum to 32bits */
+       movl %eax, %ebx
+       shrq $32, %rax
+       addl %ebx, %eax
+       adcl %r9d, %eax
+
+       /* do last up to 6 bytes */
+.Lhandle_7:
+       movl %r10d, %ecx
+       andl $7, %ecx
+       shrl $1, %ecx
+       jz   .Lhandle_1
+       movl $2, %edx
+       xorl %ebx, %ebx
+       clc
+       .p2align 4
+.Lloop_1:
+       source
+       movw (%rdi), %bx
+       adcl %ebx, %eax
+       decl %ecx
+       dest
+       movw %bx, (%rsi)
+       leaq 2(%rdi), %rdi
+       leaq 2(%rsi), %rsi
+       jnz .Lloop_1
+       adcl %r9d, %eax /* add in carry */
+
+       /* handle last odd byte */
+.Lhandle_1:
+       testb $1, %r10b
+       jz    .Lende
+       xorl  %ebx, %ebx
+       source
+       movb (%rdi), %bl
+       dest
+       movb %bl, (%rsi)
+       addl %ebx, %eax
+       adcl %r9d, %eax         /* carry */
+
+       CFI_REMEMBER_STATE
+.Lende:
+       movq 2*8(%rsp), %rbx
+       CFI_RESTORE rbx
+       movq 3*8(%rsp), %r12
+       CFI_RESTORE r12
+       movq 4*8(%rsp), %r14
+       CFI_RESTORE r14
+       movq 5*8(%rsp), %r13
+       CFI_RESTORE r13
+       movq 6*8(%rsp), %rbp
+       CFI_RESTORE rbp
+       addq $7*8, %rsp
+       CFI_ADJUST_CFA_OFFSET -7*8
+       ret
+       CFI_RESTORE_STATE
+
+       /* Exception handlers. Very simple, zeroing is done in the wrappers */
+.Lbad_source:
+       movq (%rsp), %rax
+       testq %rax, %rax
+       jz   .Lende
+       movl $-EFAULT, (%rax)
+       jmp  .Lende
+
+.Lbad_dest:
+       movq 8(%rsp), %rax
+       testq %rax, %rax
+       jz   .Lende
+       movl $-EFAULT, (%rax)
+       jmp .Lende
+       CFI_ENDPROC
+ENDPROC(csum_partial_copy_generic)