kernel/arch/x86/net/bpf_jit.S

   1 /* bpf_jit.S : BPF JIT helper functions
   2  *
   3  * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; version 2
   8  * of the License.
   9  */
  10 #include <linux/linkage.h>
  11 #include <asm/dwarf2.h>
  12
  13 /*
  14  * Calling convention :
  15  * rbx : skb pointer (callee saved)
  16  * esi : offset of byte(s) to fetch in skb (can be scratched)
  17  * r10 : copy of skb->data
  18  * r9d : hlen = skb->len - skb->data_len
  19  */
  20 #define SKBDATA %r10
  21 #define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
  22 #define MAX_BPF_STACK (512 /* from filter.h */ + \
  23         32 /* space for rbx,r13,r14,r15 */ + \
  24         8 /* space for skb_copy_bits */)
  25
  26 sk_load_word:
  27         .globl  sk_load_word
  28
  29         test    %esi,%esi
  30         js      bpf_slow_path_word_neg
  31
  32 sk_load_word_positive_offset:
  33         .globl  sk_load_word_positive_offset
  34
  35         mov     %r9d,%eax               # hlen
  36         sub     %esi,%eax               # hlen - offset
  37         cmp     $3,%eax
  38         jle     bpf_slow_path_word
  39         mov     (SKBDATA,%rsi),%eax
  40         bswap   %eax                    /* ntohl() */
  41         ret
  42
  43 sk_load_half:
  44         .globl  sk_load_half
  45
  46         test    %esi,%esi
  47         js      bpf_slow_path_half_neg
  48
  49 sk_load_half_positive_offset:
  50         .globl  sk_load_half_positive_offset
  51
  52         mov     %r9d,%eax
  53         sub     %esi,%eax               #       hlen - offset
  54         cmp     $1,%eax
  55         jle     bpf_slow_path_half
  56         movzwl  (SKBDATA,%rsi),%eax
  57         rol     $8,%ax                  # ntohs()
  58         ret
  59
  60 sk_load_byte:
  61         .globl  sk_load_byte
  62
  63         test    %esi,%esi
  64         js      bpf_slow_path_byte_neg
  65
  66 sk_load_byte_positive_offset:
  67         .globl  sk_load_byte_positive_offset
  68
  69         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
  70         jle     bpf_slow_path_byte
  71         movzbl  (SKBDATA,%rsi),%eax
  72         ret
  73
  74 /* rsi contains offset and can be scratched */
  75 #define bpf_slow_path_common(LEN)               \
  76         mov     %rbx, %rdi; /* arg1 == skb */   \
  77         push    %r9;                            \
  78         push    SKBDATA;                        \
  79 /* rsi already has offset */                    \
  80         mov     $LEN,%ecx;      /* len */       \
  81         lea     - MAX_BPF_STACK + 32(%rbp),%rdx;                        \
  82         call    skb_copy_bits;                  \
  83         test    %eax,%eax;                      \
  84         pop     SKBDATA;                        \
  85         pop     %r9;
  86
  87
  88 bpf_slow_path_word:
  89         bpf_slow_path_common(4)
  90         js      bpf_error
  91         mov     - MAX_BPF_STACK + 32(%rbp),%eax
  92         bswap   %eax
  93         ret
  94
  95 bpf_slow_path_half:
  96         bpf_slow_path_common(2)
  97         js      bpf_error
  98         mov     - MAX_BPF_STACK + 32(%rbp),%ax
  99         rol     $8,%ax
 100         movzwl  %ax,%eax
 101         ret
 102
 103 bpf_slow_path_byte:
 104         bpf_slow_path_common(1)
 105         js      bpf_error
 106         movzbl  - MAX_BPF_STACK + 32(%rbp),%eax
 107         ret
 108
 109 #define sk_negative_common(SIZE)                                \
 110         mov     %rbx, %rdi; /* arg1 == skb */                   \
 111         push    %r9;                                            \
 112         push    SKBDATA;                                        \
 113 /* rsi already has offset */                                    \
 114         mov     $SIZE,%edx;     /* size */                      \
 115         call    bpf_internal_load_pointer_neg_helper;           \
 116         test    %rax,%rax;                                      \
 117         pop     SKBDATA;                                        \
 118         pop     %r9;                                            \
 119         jz      bpf_error
 120
 121 bpf_slow_path_word_neg:
 122         cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
 123         jl      bpf_error       /* offset lower -> error  */
 124 sk_load_word_negative_offset:
 125         .globl  sk_load_word_negative_offset
 126         sk_negative_common(4)
 127         mov     (%rax), %eax
 128         bswap   %eax
 129         ret
 130
 131 bpf_slow_path_half_neg:
 132         cmp     SKF_MAX_NEG_OFF, %esi
 133         jl      bpf_error
 134 sk_load_half_negative_offset:
 135         .globl  sk_load_half_negative_offset
 136         sk_negative_common(2)
 137         mov     (%rax),%ax
 138         rol     $8,%ax
 139         movzwl  %ax,%eax
 140         ret
 141
 142 bpf_slow_path_byte_neg:
 143         cmp     SKF_MAX_NEG_OFF, %esi
 144         jl      bpf_error
 145 sk_load_byte_negative_offset:
 146         .globl  sk_load_byte_negative_offset
 147         sk_negative_common(1)
 148         movzbl  (%rax), %eax
 149         ret
 150
 151 bpf_error:
 152 # force a return 0 from jit handler
 153         xor     %eax,%eax
 154         mov     - MAX_BPF_STACK(%rbp),%rbx
 155         mov     - MAX_BPF_STACK + 8(%rbp),%r13
 156         mov     - MAX_BPF_STACK + 16(%rbp),%r14
 157         mov     - MAX_BPF_STACK + 24(%rbp),%r15
 158         leaveq
 159         ret