These changes are the raw update to linux-4.4.6-rt14. Kernel sources

[kvmfornfv.git] / kernel / net / sched / act_bpf.c

diff --git a/kernel/net/sched/act_bpf.c b/kernel/net/sched/act_bpf.c
index 521ffca..0bc6f91 100644 (file)
--- a/kernel/net/sched/act_bpf.c
+++ b/kernel/net/sched/act_bpf.c
@@ -37,19 +37,25 @@ static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act,
                   struct tcf_result *res)
 {
        struct tcf_bpf *prog = act->priv;
+       struct bpf_prog *filter;
        int action, filter_res;
+       bool at_ingress = G_TC_AT(skb->tc_verd) & AT_INGRESS;
 
        if (unlikely(!skb_mac_header_was_set(skb)))
                return TC_ACT_UNSPEC;
 
-       spin_lock(&prog->tcf_lock);
-
-       prog->tcf_tm.lastuse = jiffies;
-       bstats_update(&prog->tcf_bstats, skb);
+       tcf_lastuse_update(&prog->tcf_tm);
+       bstats_cpu_update(this_cpu_ptr(prog->common.cpu_bstats), skb);
 
-       /* Needed here for accessing maps. */
        rcu_read_lock();
-       filter_res = BPF_PROG_RUN(prog->filter, skb);
+       filter = rcu_dereference(prog->filter);
+       if (at_ingress) {
+               __skb_push(skb, skb->mac_len);
+               filter_res = BPF_PROG_RUN(filter, skb);
+               __skb_pull(skb, skb->mac_len);
+       } else {
+               filter_res = BPF_PROG_RUN(filter, skb);
+       }
        rcu_read_unlock();
 
        /* A BPF program may overwrite the default action opcode.
@@ -66,11 +72,12 @@ static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act,
        case TC_ACT_PIPE:
        case TC_ACT_RECLASSIFY:
        case TC_ACT_OK:
+       case TC_ACT_REDIRECT:
                action = filter_res;
                break;
        case TC_ACT_SHOT:
                action = filter_res;
-               prog->tcf_qstats.drops++;
+               qstats_drop_inc(this_cpu_ptr(prog->common.cpu_qstats));
                break;
        case TC_ACT_UNSPEC:
                action = prog->tcf_action;
@@ -80,7 +87,6 @@ static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act,
                break;
        }
 
-       spin_unlock(&prog->tcf_lock);
        return action;
 }
 
@@ -256,7 +262,10 @@ static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf *prog,
                                  struct tcf_bpf_cfg *cfg)
 {
        cfg->is_ebpf = tcf_bpf_is_ebpf(prog);
-       cfg->filter = prog->filter;
+       /* updates to prog->filter are prevented, since it's called either
+        * with rtnl lock or during final cleanup in rcu callback
+        */
+       cfg->filter = rcu_dereference_protected(prog->filter, 1);
 
        cfg->bpf_ops = prog->bpf_ops;
        cfg->bpf_name = prog->bpf_name;
@@ -271,7 +280,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla,
        struct tc_act_bpf *parm;
        struct tcf_bpf *prog;
        bool is_bpf, is_ebpf;
-       int ret;
+       int ret, res = 0;
 
        if (!nla)
                return -EINVAL;
@@ -280,45 +289,47 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla,
        if (ret < 0)
                return ret;
 
-       is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS];
-       is_ebpf = tb[TCA_ACT_BPF_FD];
-
-       if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf) ||
-           !tb[TCA_ACT_BPF_PARMS])
+       if (!tb[TCA_ACT_BPF_PARMS])
                return -EINVAL;
 
        parm = nla_data(tb[TCA_ACT_BPF_PARMS]);
 
-       memset(&cfg, 0, sizeof(cfg));
-
-       ret = is_bpf ? tcf_bpf_init_from_ops(tb, &cfg) :
-                      tcf_bpf_init_from_efd(tb, &cfg);
-       if (ret < 0)
-               return ret;
-
        if (!tcf_hash_check(parm->index, act, bind)) {
                ret = tcf_hash_create(parm->index, est, act,
-                                     sizeof(*prog), bind);
+                                     sizeof(*prog), bind, true);
                if (ret < 0)
-                       goto destroy_fp;
+                       return ret;
 
-               ret = ACT_P_CREATED;
+               res = ACT_P_CREATED;
        } else {
                /* Don't override defaults. */
                if (bind)
-                       goto destroy_fp;
+                       return 0;
 
                tcf_hash_release(act, bind);
-               if (!replace) {
-                       ret = -EEXIST;
-                       goto destroy_fp;
-               }
+               if (!replace)
+                       return -EEXIST;
        }
 
+       is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS];
+       is_ebpf = tb[TCA_ACT_BPF_FD];
+
+       if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) {
+               ret = -EINVAL;
+               goto out;
+       }
+
+       memset(&cfg, 0, sizeof(cfg));
+
+       ret = is_bpf ? tcf_bpf_init_from_ops(tb, &cfg) :
+                      tcf_bpf_init_from_efd(tb, &cfg);
+       if (ret < 0)
+               goto out;
+
        prog = to_bpf(act);
-       spin_lock_bh(&prog->tcf_lock);
+       ASSERT_RTNL();
 
-       if (ret != ACT_P_CREATED)
+       if (res != ACT_P_CREATED)
                tcf_bpf_prog_fill_cfg(prog, &old);
 
        prog->bpf_ops = cfg.bpf_ops;
@@ -330,19 +341,21 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla,
                prog->bpf_fd = cfg.bpf_fd;
 
        prog->tcf_action = parm->action;
-       prog->filter = cfg.filter;
-
-       spin_unlock_bh(&prog->tcf_lock);
+       rcu_assign_pointer(prog->filter, cfg.filter);
 
-       if (ret == ACT_P_CREATED)
+       if (res == ACT_P_CREATED) {
                tcf_hash_insert(act);
-       else
+       } else {
+               /* make sure the program being replaced is no longer executing */
+               synchronize_rcu();
                tcf_bpf_cfg_cleanup(&old);
+       }
 
-       return ret;
+       return res;
+out:
+       if (res == ACT_P_CREATED)
+               tcf_hash_cleanup(act, est);
 
-destroy_fp:
-       tcf_bpf_cfg_cleanup(&cfg);
        return ret;
 }