kernel/arch/x86/net/bpf_jit.S

   1 /* bpf_jit.S : BPF JIT helper functions
   2  *
   3  * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; version 2
   8  * of the License.
   9  */
  10 #include <linux/linkage.h>
  11
  12 /*
  13  * Calling convention :
  14  * rbx : skb pointer (callee saved)
  15  * esi : offset of byte(s) to fetch in skb (can be scratched)
  16  * r10 : copy of skb->data
  17  * r9d : hlen = skb->len - skb->data_len
  18  */
  19 #define SKBDATA %r10
  20 #define SKF_MAX_NEG_OFF    $(-0x200000) /* SKF_LL_OFF from filter.h */
  21 #define MAX_BPF_STACK (512 /* from filter.h */ + \
  22         32 /* space for rbx,r13,r14,r15 */ + \
  23         8 /* space for skb_copy_bits */)
  24
  25 sk_load_word:
  26         .globl  sk_load_word
  27
  28         test    %esi,%esi
  29         js      bpf_slow_path_word_neg
  30
  31 sk_load_word_positive_offset:
  32         .globl  sk_load_word_positive_offset
  33
  34         mov     %r9d,%eax               # hlen
  35         sub     %esi,%eax               # hlen - offset
  36         cmp     $3,%eax
  37         jle     bpf_slow_path_word
  38         mov     (SKBDATA,%rsi),%eax
  39         bswap   %eax                    /* ntohl() */
  40         ret
  41
  42 sk_load_half:
  43         .globl  sk_load_half
  44
  45         test    %esi,%esi
  46         js      bpf_slow_path_half_neg
  47
  48 sk_load_half_positive_offset:
  49         .globl  sk_load_half_positive_offset
  50
  51         mov     %r9d,%eax
  52         sub     %esi,%eax               #       hlen - offset
  53         cmp     $1,%eax
  54         jle     bpf_slow_path_half
  55         movzwl  (SKBDATA,%rsi),%eax
  56         rol     $8,%ax                  # ntohs()
  57         ret
  58
  59 sk_load_byte:
  60         .globl  sk_load_byte
  61
  62         test    %esi,%esi
  63         js      bpf_slow_path_byte_neg
  64
  65 sk_load_byte_positive_offset:
  66         .globl  sk_load_byte_positive_offset
  67
  68         cmp     %esi,%r9d   /* if (offset >= hlen) goto bpf_slow_path_byte */
  69         jle     bpf_slow_path_byte
  70         movzbl  (SKBDATA,%rsi),%eax
  71         ret
  72
  73 /* rsi contains offset and can be scratched */
  74 #define bpf_slow_path_common(LEN)               \
  75         mov     %rbx, %rdi; /* arg1 == skb */   \
  76         push    %r9;                            \
  77         push    SKBDATA;                        \
  78 /* rsi already has offset */                    \
  79         mov     $LEN,%ecx;      /* len */       \
  80         lea     - MAX_BPF_STACK + 32(%rbp),%rdx;                        \
  81         call    skb_copy_bits;                  \
  82         test    %eax,%eax;                      \
  83         pop     SKBDATA;                        \
  84         pop     %r9;
  85
  86
  87 bpf_slow_path_word:
  88         bpf_slow_path_common(4)
  89         js      bpf_error
  90         mov     - MAX_BPF_STACK + 32(%rbp),%eax
  91         bswap   %eax
  92         ret
  93
  94 bpf_slow_path_half:
  95         bpf_slow_path_common(2)
  96         js      bpf_error
  97         mov     - MAX_BPF_STACK + 32(%rbp),%ax
  98         rol     $8,%ax
  99         movzwl  %ax,%eax
 100         ret
 101
 102 bpf_slow_path_byte:
 103         bpf_slow_path_common(1)
 104         js      bpf_error
 105         movzbl  - MAX_BPF_STACK + 32(%rbp),%eax
 106         ret
 107
 108 #define sk_negative_common(SIZE)                                \
 109         mov     %rbx, %rdi; /* arg1 == skb */                   \
 110         push    %r9;                                            \
 111         push    SKBDATA;                                        \
 112 /* rsi already has offset */                                    \
 113         mov     $SIZE,%edx;     /* size */                      \
 114         call    bpf_internal_load_pointer_neg_helper;           \
 115         test    %rax,%rax;                                      \
 116         pop     SKBDATA;                                        \
 117         pop     %r9;                                            \
 118         jz      bpf_error
 119
 120 bpf_slow_path_word_neg:
 121         cmp     SKF_MAX_NEG_OFF, %esi   /* test range */
 122         jl      bpf_error       /* offset lower -> error  */
 123 sk_load_word_negative_offset:
 124         .globl  sk_load_word_negative_offset
 125         sk_negative_common(4)
 126         mov     (%rax), %eax
 127         bswap   %eax
 128         ret
 129
 130 bpf_slow_path_half_neg:
 131         cmp     SKF_MAX_NEG_OFF, %esi
 132         jl      bpf_error
 133 sk_load_half_negative_offset:
 134         .globl  sk_load_half_negative_offset
 135         sk_negative_common(2)
 136         mov     (%rax),%ax
 137         rol     $8,%ax
 138         movzwl  %ax,%eax
 139         ret
 140
 141 bpf_slow_path_byte_neg:
 142         cmp     SKF_MAX_NEG_OFF, %esi
 143         jl      bpf_error
 144 sk_load_byte_negative_offset:
 145         .globl  sk_load_byte_negative_offset
 146         sk_negative_common(1)
 147         movzbl  (%rax), %eax
 148         ret
 149
 150 bpf_error:
 151 # force a return 0 from jit handler
 152         xor     %eax,%eax
 153         mov     - MAX_BPF_STACK(%rbp),%rbx
 154         mov     - MAX_BPF_STACK + 8(%rbp),%r13
 155         mov     - MAX_BPF_STACK + 16(%rbp),%r14
 156         mov     - MAX_BPF_STACK + 24(%rbp),%r15
 157         leaveq
 158         ret