kernel/net/ipv4/route.c

   1 /*
   2  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   3  *              operating system.  INET is implemented using the  BSD Socket
   4  *              interface as the means of communication with the user level.
   5  *
   6  *              ROUTE - implementation of the IP router.
   7  *
   8  * Authors:     Ross Biro
   9  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  10  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  11  *              Linus Torvalds, <Linus.Torvalds@helsinki.fi>
  12  *              Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  13  *
  14  * Fixes:
  15  *              Alan Cox        :       Verify area fixes.
  16  *              Alan Cox        :       cli() protects routing changes
  17  *              Rui Oliveira    :       ICMP routing table updates
  18  *              (rco@di.uminho.pt)      Routing table insertion and update
  19  *              Linus Torvalds  :       Rewrote bits to be sensible
  20  *              Alan Cox        :       Added BSD route gw semantics
  21  *              Alan Cox        :       Super /proc >4K
  22  *              Alan Cox        :       MTU in route table
  23  *              Alan Cox        :       MSS actually. Also added the window
  24  *                                      clamper.
  25  *              Sam Lantinga    :       Fixed route matching in rt_del()
  26  *              Alan Cox        :       Routing cache support.
  27  *              Alan Cox        :       Removed compatibility cruft.
  28  *              Alan Cox        :       RTF_REJECT support.
  29  *              Alan Cox        :       TCP irtt support.
  30  *              Jonathan Naylor :       Added Metric support.
  31  *      Miquel van Smoorenburg  :       BSD API fixes.
  32  *      Miquel van Smoorenburg  :       Metrics.
  33  *              Alan Cox        :       Use __u32 properly
  34  *              Alan Cox        :       Aligned routing errors more closely with BSD
  35  *                                      our system is still very different.
  36  *              Alan Cox        :       Faster /proc handling
  37  *      Alexey Kuznetsov        :       Massive rework to support tree based routing,
  38  *                                      routing caches and better behaviour.
  39  *
  40  *              Olaf Erb        :       irtt wasn't being copied right.
  41  *              Bjorn Ekwall    :       Kerneld route support.
  42  *              Alan Cox        :       Multicast fixed (I hope)
  43  *              Pavel Krauz     :       Limited broadcast fixed
  44  *              Mike McLagan    :       Routing by source
  45  *      Alexey Kuznetsov        :       End of old history. Split to fib.c and
  46  *                                      route.c and rewritten from scratch.
  47  *              Andi Kleen      :       Load-limit warning messages.
  48  *      Vitaly E. Lavrov        :       Transparent proxy revived after year coma.
  49  *      Vitaly E. Lavrov        :       Race condition in ip_route_input_slow.
  50  *      Tobias Ringstrom        :       Uninitialized res.type in ip_route_output_slow.
  51  *      Vladimir V. Ivanov      :       IP rule info (flowid) is really useful.
  52  *              Marc Boucher    :       routing by fwmark
  53  *      Robert Olsson           :       Added rt_cache statistics
  54  *      Arnaldo C. Melo         :       Convert proc stuff to seq_file
  55  *      Eric Dumazet            :       hashed spinlocks and rt_check_expire() fixes.
  56  *      Ilia Sotnikov           :       Ignore TOS on PMTUD and Redirect
  57  *      Ilia Sotnikov           :       Removed TOS from hash calculations
  58  *
  59  *              This program is free software; you can redistribute it and/or
  60  *              modify it under the terms of the GNU General Public License
  61  *              as published by the Free Software Foundation; either version
  62  *              2 of the License, or (at your option) any later version.
  63  */
  64
  65 #define pr_fmt(fmt) "IPv4: " fmt
  66
  67 #include <linux/module.h>
  68 #include <asm/uaccess.h>
  69 #include <linux/bitops.h>
  70 #include <linux/types.h>
  71 #include <linux/kernel.h>
  72 #include <linux/mm.h>
  73 #include <linux/string.h>
  74 #include <linux/socket.h>
  75 #include <linux/sockios.h>
  76 #include <linux/errno.h>
  77 #include <linux/in.h>
  78 #include <linux/inet.h>
  79 #include <linux/netdevice.h>
  80 #include <linux/proc_fs.h>
  81 #include <linux/init.h>
  82 #include <linux/skbuff.h>
  83 #include <linux/inetdevice.h>
  84 #include <linux/igmp.h>
  85 #include <linux/pkt_sched.h>
  86 #include <linux/mroute.h>
  87 #include <linux/netfilter_ipv4.h>
  88 #include <linux/random.h>
  89 #include <linux/rcupdate.h>
  90 #include <linux/times.h>
  91 #include <linux/slab.h>
  92 #include <linux/jhash.h>
  93 #include <net/dst.h>
  94 #include <net/dst_metadata.h>
  95 #include <net/net_namespace.h>
  96 #include <net/protocol.h>
  97 #include <net/ip.h>
  98 #include <net/route.h>
  99 #include <net/inetpeer.h>
 100 #include <net/sock.h>
 101 #include <net/ip_fib.h>
 102 #include <net/arp.h>
 103 #include <net/tcp.h>
 104 #include <net/icmp.h>
 105 #include <net/xfrm.h>
 106 #include <net/lwtunnel.h>
 107 #include <net/netevent.h>
 108 #include <net/rtnetlink.h>
 109 #ifdef CONFIG_SYSCTL
 110 #include <linux/sysctl.h>
 111 #include <linux/kmemleak.h>
 112 #endif
 113 #include <net/secure_seq.h>
 114 #include <net/ip_tunnels.h>
 115 #include <net/l3mdev.h>
 116
 117 #define RT_FL_TOS(oldflp4) \
 118         ((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
 119
 120 #define RT_GC_TIMEOUT (300*HZ)
 121
 122 static int ip_rt_max_size;
 123 static int ip_rt_redirect_number __read_mostly  = 9;
 124 static int ip_rt_redirect_load __read_mostly    = HZ / 50;
 125 static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
 126 static int ip_rt_error_cost __read_mostly       = HZ;
 127 static int ip_rt_error_burst __read_mostly      = 5 * HZ;
 128 static int ip_rt_mtu_expires __read_mostly      = 10 * 60 * HZ;
 129 static int ip_rt_min_pmtu __read_mostly         = 512 + 20 + 20;
 130 static int ip_rt_min_advmss __read_mostly       = 256;
 131
 132 static int ip_rt_gc_timeout __read_mostly       = RT_GC_TIMEOUT;
 133 /*
 134  *      Interface to generic destination cache.
 135  */
 136
 137 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
 138 static unsigned int      ipv4_default_advmss(const struct dst_entry *dst);
 139 static unsigned int      ipv4_mtu(const struct dst_entry *dst);
 140 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
 141 static void              ipv4_link_failure(struct sk_buff *skb);
 142 static void              ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 143                                            struct sk_buff *skb, u32 mtu);
 144 static void              ip_do_redirect(struct dst_entry *dst, struct sock *sk,
 145                                         struct sk_buff *skb);
 146 static void             ipv4_dst_destroy(struct dst_entry *dst);
 147
 148 static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
 149 {
 150         WARN_ON(1);
 151         return NULL;
 152 }
 153
 154 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 155                                            struct sk_buff *skb,
 156                                            const void *daddr);
 157
 158 static struct dst_ops ipv4_dst_ops = {
 159         .family =               AF_INET,
 160         .check =                ipv4_dst_check,
 161         .default_advmss =       ipv4_default_advmss,
 162         .mtu =                  ipv4_mtu,
 163         .cow_metrics =          ipv4_cow_metrics,
 164         .destroy =              ipv4_dst_destroy,
 165         .negative_advice =      ipv4_negative_advice,
 166         .link_failure =         ipv4_link_failure,
 167         .update_pmtu =          ip_rt_update_pmtu,
 168         .redirect =             ip_do_redirect,
 169         .local_out =            __ip_local_out,
 170         .neigh_lookup =         ipv4_neigh_lookup,
 171 };
 172
 173 #define ECN_OR_COST(class)      TC_PRIO_##class
 174
 175 const __u8 ip_tos2prio[16] = {
 176         TC_PRIO_BESTEFFORT,
 177         ECN_OR_COST(BESTEFFORT),
 178         TC_PRIO_BESTEFFORT,
 179         ECN_OR_COST(BESTEFFORT),
 180         TC_PRIO_BULK,
 181         ECN_OR_COST(BULK),
 182         TC_PRIO_BULK,
 183         ECN_OR_COST(BULK),
 184         TC_PRIO_INTERACTIVE,
 185         ECN_OR_COST(INTERACTIVE),
 186         TC_PRIO_INTERACTIVE,
 187         ECN_OR_COST(INTERACTIVE),
 188         TC_PRIO_INTERACTIVE_BULK,
 189         ECN_OR_COST(INTERACTIVE_BULK),
 190         TC_PRIO_INTERACTIVE_BULK,
 191         ECN_OR_COST(INTERACTIVE_BULK)
 192 };
 193 EXPORT_SYMBOL(ip_tos2prio);
 194
 195 static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
 196 #define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
 197
 198 #ifdef CONFIG_PROC_FS
 199 static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
 200 {
 201         if (*pos)
 202                 return NULL;
 203         return SEQ_START_TOKEN;
 204 }
 205
 206 static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 207 {
 208         ++*pos;
 209         return NULL;
 210 }
 211
 212 static void rt_cache_seq_stop(struct seq_file *seq, void *v)
 213 {
 214 }
 215
 216 static int rt_cache_seq_show(struct seq_file *seq, void *v)
 217 {
 218         if (v == SEQ_START_TOKEN)
 219                 seq_printf(seq, "%-127s\n",
 220                            "Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
 221                            "Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
 222                            "HHUptod\tSpecDst");
 223         return 0;
 224 }
 225
 226 static const struct seq_operations rt_cache_seq_ops = {
 227         .start  = rt_cache_seq_start,
 228         .next   = rt_cache_seq_next,
 229         .stop   = rt_cache_seq_stop,
 230         .show   = rt_cache_seq_show,
 231 };
 232
 233 static int rt_cache_seq_open(struct inode *inode, struct file *file)
 234 {
 235         return seq_open(file, &rt_cache_seq_ops);
 236 }
 237
 238 static const struct file_operations rt_cache_seq_fops = {
 239         .owner   = THIS_MODULE,
 240         .open    = rt_cache_seq_open,
 241         .read    = seq_read,
 242         .llseek  = seq_lseek,
 243         .release = seq_release,
 244 };
 245
 246
 247 static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
 248 {
 249         int cpu;
 250
 251         if (*pos == 0)
 252                 return SEQ_START_TOKEN;
 253
 254         for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
 255                 if (!cpu_possible(cpu))
 256                         continue;
 257                 *pos = cpu+1;
 258                 return &per_cpu(rt_cache_stat, cpu);
 259         }
 260         return NULL;
 261 }
 262
 263 static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 264 {
 265         int cpu;
 266
 267         for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
 268                 if (!cpu_possible(cpu))
 269                         continue;
 270                 *pos = cpu+1;
 271                 return &per_cpu(rt_cache_stat, cpu);
 272         }
 273         return NULL;
 274
 275 }
 276
 277 static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
 278 {
 279
 280 }
 281
 282 static int rt_cpu_seq_show(struct seq_file *seq, void *v)
 283 {
 284         struct rt_cache_stat *st = v;
 285
 286         if (v == SEQ_START_TOKEN) {
 287                 seq_printf(seq, "entries  in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src  out_hit out_slow_tot out_slow_mc  gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
 288                 return 0;
 289         }
 290
 291         seq_printf(seq,"%08x  %08x %08x %08x %08x %08x %08x %08x "
 292                    " %08x %08x %08x %08x %08x %08x %08x %08x %08x \n",
 293                    dst_entries_get_slow(&ipv4_dst_ops),
 294                    0, /* st->in_hit */
 295                    st->in_slow_tot,
 296                    st->in_slow_mc,
 297                    st->in_no_route,
 298                    st->in_brd,
 299                    st->in_martian_dst,
 300                    st->in_martian_src,
 301
 302                    0, /* st->out_hit */
 303                    st->out_slow_tot,
 304                    st->out_slow_mc,
 305
 306                    0, /* st->gc_total */
 307                    0, /* st->gc_ignored */
 308                    0, /* st->gc_goal_miss */
 309                    0, /* st->gc_dst_overflow */
 310                    0, /* st->in_hlist_search */
 311                    0  /* st->out_hlist_search */
 312                 );
 313         return 0;
 314 }
 315
 316 static const struct seq_operations rt_cpu_seq_ops = {
 317         .start  = rt_cpu_seq_start,
 318         .next   = rt_cpu_seq_next,
 319         .stop   = rt_cpu_seq_stop,
 320         .show   = rt_cpu_seq_show,
 321 };
 322
 323
 324 static int rt_cpu_seq_open(struct inode *inode, struct file *file)
 325 {
 326         return seq_open(file, &rt_cpu_seq_ops);
 327 }
 328
 329 static const struct file_operations rt_cpu_seq_fops = {
 330         .owner   = THIS_MODULE,
 331         .open    = rt_cpu_seq_open,
 332         .read    = seq_read,
 333         .llseek  = seq_lseek,
 334         .release = seq_release,
 335 };
 336
 337 #ifdef CONFIG_IP_ROUTE_CLASSID
 338 static int rt_acct_proc_show(struct seq_file *m, void *v)
 339 {
 340         struct ip_rt_acct *dst, *src;
 341         unsigned int i, j;
 342
 343         dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
 344         if (!dst)
 345                 return -ENOMEM;
 346
 347         for_each_possible_cpu(i) {
 348                 src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
 349                 for (j = 0; j < 256; j++) {
 350                         dst[j].o_bytes   += src[j].o_bytes;
 351                         dst[j].o_packets += src[j].o_packets;
 352                         dst[j].i_bytes   += src[j].i_bytes;
 353                         dst[j].i_packets += src[j].i_packets;
 354                 }
 355         }
 356
 357         seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
 358         kfree(dst);
 359         return 0;
 360 }
 361
 362 static int rt_acct_proc_open(struct inode *inode, struct file *file)
 363 {
 364         return single_open(file, rt_acct_proc_show, NULL);
 365 }
 366
 367 static const struct file_operations rt_acct_proc_fops = {
 368         .owner          = THIS_MODULE,
 369         .open           = rt_acct_proc_open,
 370         .read           = seq_read,
 371         .llseek         = seq_lseek,
 372         .release        = single_release,
 373 };
 374 #endif
 375
 376 static int __net_init ip_rt_do_proc_init(struct net *net)
 377 {
 378         struct proc_dir_entry *pde;
 379
 380         pde = proc_create("rt_cache", S_IRUGO, net->proc_net,
 381                           &rt_cache_seq_fops);
 382         if (!pde)
 383                 goto err1;
 384
 385         pde = proc_create("rt_cache", S_IRUGO,
 386                           net->proc_net_stat, &rt_cpu_seq_fops);
 387         if (!pde)
 388                 goto err2;
 389
 390 #ifdef CONFIG_IP_ROUTE_CLASSID
 391         pde = proc_create("rt_acct", 0, net->proc_net, &rt_acct_proc_fops);
 392         if (!pde)
 393                 goto err3;
 394 #endif
 395         return 0;
 396
 397 #ifdef CONFIG_IP_ROUTE_CLASSID
 398 err3:
 399         remove_proc_entry("rt_cache", net->proc_net_stat);
 400 #endif
 401 err2:
 402         remove_proc_entry("rt_cache", net->proc_net);
 403 err1:
 404         return -ENOMEM;
 405 }
 406
 407 static void __net_exit ip_rt_do_proc_exit(struct net *net)
 408 {
 409         remove_proc_entry("rt_cache", net->proc_net_stat);
 410         remove_proc_entry("rt_cache", net->proc_net);
 411 #ifdef CONFIG_IP_ROUTE_CLASSID
 412         remove_proc_entry("rt_acct", net->proc_net);
 413 #endif
 414 }
 415
 416 static struct pernet_operations ip_rt_proc_ops __net_initdata =  {
 417         .init = ip_rt_do_proc_init,
 418         .exit = ip_rt_do_proc_exit,
 419 };
 420
 421 static int __init ip_rt_proc_init(void)
 422 {
 423         return register_pernet_subsys(&ip_rt_proc_ops);
 424 }
 425
 426 #else
 427 static inline int ip_rt_proc_init(void)
 428 {
 429         return 0;
 430 }
 431 #endif /* CONFIG_PROC_FS */
 432
 433 static inline bool rt_is_expired(const struct rtable *rth)
 434 {
 435         return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
 436 }
 437
 438 void rt_cache_flush(struct net *net)
 439 {
 440         rt_genid_bump_ipv4(net);
 441 }
 442
 443 static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
 444                                            struct sk_buff *skb,
 445                                            const void *daddr)
 446 {
 447         struct net_device *dev = dst->dev;
 448         const __be32 *pkey = daddr;
 449         const struct rtable *rt;
 450         struct neighbour *n;
 451
 452         rt = (const struct rtable *) dst;
 453         if (rt->rt_gateway)
 454                 pkey = (const __be32 *) &rt->rt_gateway;
 455         else if (skb)
 456                 pkey = &ip_hdr(skb)->daddr;
 457
 458         n = __ipv4_neigh_lookup(dev, *(__force u32 *)pkey);
 459         if (n)
 460                 return n;
 461         return neigh_create(&arp_tbl, pkey, dev);
 462 }
 463
 464 #define IP_IDENTS_SZ 2048u
 465
 466 static atomic_t *ip_idents __read_mostly;
 467 static u32 *ip_tstamps __read_mostly;
 468
 469 /* In order to protect privacy, we add a perturbation to identifiers
 470  * if one generator is seldom used. This makes hard for an attacker
 471  * to infer how many packets were sent between two points in time.
 472  */
 473 u32 ip_idents_reserve(u32 hash, int segs)
 474 {
 475         u32 *p_tstamp = ip_tstamps + hash % IP_IDENTS_SZ;
 476         atomic_t *p_id = ip_idents + hash % IP_IDENTS_SZ;
 477         u32 old = ACCESS_ONCE(*p_tstamp);
 478         u32 now = (u32)jiffies;
 479         u32 delta = 0;
 480
 481         if (old != now && cmpxchg(p_tstamp, old, now) == old)
 482                 delta = prandom_u32_max(now - old);
 483
 484         return atomic_add_return(segs + delta, p_id) - segs;
 485 }
 486 EXPORT_SYMBOL(ip_idents_reserve);
 487
 488 void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
 489 {
 490         static u32 ip_idents_hashrnd __read_mostly;
 491         u32 hash, id;
 492
 493         net_get_random_once(&ip_idents_hashrnd, sizeof(ip_idents_hashrnd));
 494
 495         hash = jhash_3words((__force u32)iph->daddr,
 496                             (__force u32)iph->saddr,
 497                             iph->protocol ^ net_hash_mix(net),
 498                             ip_idents_hashrnd);
 499         id = ip_idents_reserve(hash, segs);
 500         iph->id = htons(id);
 501 }
 502 EXPORT_SYMBOL(__ip_select_ident);
 503
 504 static void __build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 505                              const struct iphdr *iph,
 506                              int oif, u8 tos,
 507                              u8 prot, u32 mark, int flow_flags)
 508 {
 509         if (sk) {
 510                 const struct inet_sock *inet = inet_sk(sk);
 511
 512                 oif = sk->sk_bound_dev_if;
 513                 mark = sk->sk_mark;
 514                 tos = RT_CONN_FLAGS(sk);
 515                 prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
 516         }
 517         flowi4_init_output(fl4, oif, mark, tos,
 518                            RT_SCOPE_UNIVERSE, prot,
 519                            flow_flags,
 520                            iph->daddr, iph->saddr, 0, 0);
 521 }
 522
 523 static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
 524                                const struct sock *sk)
 525 {
 526         const struct iphdr *iph = ip_hdr(skb);
 527         int oif = skb->dev->ifindex;
 528         u8 tos = RT_TOS(iph->tos);
 529         u8 prot = iph->protocol;
 530         u32 mark = skb->mark;
 531
 532         __build_flow_key(fl4, sk, iph, oif, tos, prot, mark, 0);
 533 }
 534
 535 static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
 536 {
 537         const struct inet_sock *inet = inet_sk(sk);
 538         const struct ip_options_rcu *inet_opt;
 539         __be32 daddr = inet->inet_daddr;
 540
 541         rcu_read_lock();
 542         inet_opt = rcu_dereference(inet->inet_opt);
 543         if (inet_opt && inet_opt->opt.srr)
 544                 daddr = inet_opt->opt.faddr;
 545         flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
 546                            RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
 547                            inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
 548                            inet_sk_flowi_flags(sk),
 549                            daddr, inet->inet_saddr, 0, 0);
 550         rcu_read_unlock();
 551 }
 552
 553 static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
 554                                  const struct sk_buff *skb)
 555 {
 556         if (skb)
 557                 build_skb_flow_key(fl4, skb, sk);
 558         else
 559                 build_sk_flow_key(fl4, sk);
 560 }
 561
 562 static inline void rt_free(struct rtable *rt)
 563 {
 564         call_rcu(&rt->dst.rcu_head, dst_rcu_free);
 565 }
 566
 567 static DEFINE_SPINLOCK(fnhe_lock);
 568
 569 static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
 570 {
 571         struct rtable *rt;
 572
 573         rt = rcu_dereference(fnhe->fnhe_rth_input);
 574         if (rt) {
 575                 RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
 576                 rt_free(rt);
 577         }
 578         rt = rcu_dereference(fnhe->fnhe_rth_output);
 579         if (rt) {
 580                 RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
 581                 rt_free(rt);
 582         }
 583 }
 584
 585 static struct fib_nh_exception *fnhe_oldest(struct fnhe_hash_bucket *hash)
 586 {
 587         struct fib_nh_exception *fnhe, *oldest;
 588
 589         oldest = rcu_dereference(hash->chain);
 590         for (fnhe = rcu_dereference(oldest->fnhe_next); fnhe;
 591              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 592                 if (time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp))
 593                         oldest = fnhe;
 594         }
 595         fnhe_flush_routes(oldest);
 596         return oldest;
 597 }
 598
 599 static inline u32 fnhe_hashfun(__be32 daddr)
 600 {
 601         static u32 fnhe_hashrnd __read_mostly;
 602         u32 hval;
 603
 604         net_get_random_once(&fnhe_hashrnd, sizeof(fnhe_hashrnd));
 605         hval = jhash_1word((__force u32) daddr, fnhe_hashrnd);
 606         return hash_32(hval, FNHE_HASH_SHIFT);
 607 }
 608
 609 static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
 610 {
 611         rt->rt_pmtu = fnhe->fnhe_pmtu;
 612         rt->dst.expires = fnhe->fnhe_expires;
 613
 614         if (fnhe->fnhe_gw) {
 615                 rt->rt_flags |= RTCF_REDIRECTED;
 616                 rt->rt_gateway = fnhe->fnhe_gw;
 617                 rt->rt_uses_gateway = 1;
 618         }
 619 }
 620
 621 static void update_or_create_fnhe(struct fib_nh *nh, __be32 daddr, __be32 gw,
 622                                   u32 pmtu, unsigned long expires)
 623 {
 624         struct fnhe_hash_bucket *hash;
 625         struct fib_nh_exception *fnhe;
 626         struct rtable *rt;
 627         unsigned int i;
 628         int depth;
 629         u32 hval = fnhe_hashfun(daddr);
 630
 631         spin_lock_bh(&fnhe_lock);
 632
 633         hash = rcu_dereference(nh->nh_exceptions);
 634         if (!hash) {
 635                 hash = kzalloc(FNHE_HASH_SIZE * sizeof(*hash), GFP_ATOMIC);
 636                 if (!hash)
 637                         goto out_unlock;
 638                 rcu_assign_pointer(nh->nh_exceptions, hash);
 639         }
 640
 641         hash += hval;
 642
 643         depth = 0;
 644         for (fnhe = rcu_dereference(hash->chain); fnhe;
 645              fnhe = rcu_dereference(fnhe->fnhe_next)) {
 646                 if (fnhe->fnhe_daddr == daddr)
 647                         break;
 648                 depth++;
 649         }
 650
 651         if (fnhe) {
 652                 if (gw)
 653                         fnhe->fnhe_gw = gw;
 654                 if (pmtu) {
 655                         fnhe->fnhe_pmtu = pmtu;
 656                         fnhe->fnhe_expires = max(1UL, expires);
 657                 }
 658                 /* Update all cached dsts too */
 659                 rt = rcu_dereference(fnhe->fnhe_rth_input);
 660                 if (rt)
 661                         fill_route_from_fnhe(rt, fnhe);
 662                 rt = rcu_dereference(fnhe->fnhe_rth_output);
 663                 if (rt)
 664                         fill_route_from_fnhe(rt, fnhe);
 665         } else {
 666                 if (depth > FNHE_RECLAIM_DEPTH)
 667                         fnhe = fnhe_oldest(hash);
 668                 else {
 669                         fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
 670                         if (!fnhe)
 671                                 goto out_unlock;
 672
 673                         fnhe->fnhe_next = hash->chain;
 674                         rcu_assign_pointer(hash->chain, fnhe);
 675                 }
 676                 fnhe->fnhe_genid = fnhe_genid(dev_net(nh->nh_dev));
 677                 fnhe->fnhe_daddr = daddr;
 678                 fnhe->fnhe_gw = gw;
 679                 fnhe->fnhe_pmtu = pmtu;
 680                 fnhe->fnhe_expires = expires;
 681
 682                 /* Exception created; mark the cached routes for the nexthop
 683                  * stale, so anyone caching it rechecks if this exception
 684                  * applies to them.
 685                  */
 686                 rt = rcu_dereference(nh->nh_rth_input);
 687                 if (rt)
 688                         rt->dst.obsolete = DST_OBSOLETE_KILL;
 689
 690                 for_each_possible_cpu(i) {
 691                         struct rtable __rcu **prt;
 692                         prt = per_cpu_ptr(nh->nh_pcpu_rth_output, i);
 693                         rt = rcu_dereference(*prt);
 694                         if (rt)
 695                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 696                 }
 697         }
 698
 699         fnhe->fnhe_stamp = jiffies;
 700
 701 out_unlock:
 702         spin_unlock_bh(&fnhe_lock);
 703 }
 704
 705 static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
 706                              bool kill_route)
 707 {
 708         __be32 new_gw = icmp_hdr(skb)->un.gateway;
 709         __be32 old_gw = ip_hdr(skb)->saddr;
 710         struct net_device *dev = skb->dev;
 711         struct in_device *in_dev;
 712         struct fib_result res;
 713         struct neighbour *n;
 714         struct net *net;
 715
 716         switch (icmp_hdr(skb)->code & 7) {
 717         case ICMP_REDIR_NET:
 718         case ICMP_REDIR_NETTOS:
 719         case ICMP_REDIR_HOST:
 720         case ICMP_REDIR_HOSTTOS:
 721                 break;
 722
 723         default:
 724                 return;
 725         }
 726
 727         if (rt->rt_gateway != old_gw)
 728                 return;
 729
 730         in_dev = __in_dev_get_rcu(dev);
 731         if (!in_dev)
 732                 return;
 733
 734         net = dev_net(dev);
 735         if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
 736             ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
 737             ipv4_is_zeronet(new_gw))
 738                 goto reject_redirect;
 739
 740         if (!IN_DEV_SHARED_MEDIA(in_dev)) {
 741                 if (!inet_addr_onlink(in_dev, new_gw, old_gw))
 742                         goto reject_redirect;
 743                 if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
 744                         goto reject_redirect;
 745         } else {
 746                 if (inet_addr_type(net, new_gw) != RTN_UNICAST)
 747                         goto reject_redirect;
 748         }
 749
 750         n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
 751         if (!n)
 752                 n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
 753         if (!IS_ERR(n)) {
 754                 if (!(n->nud_state & NUD_VALID)) {
 755                         neigh_event_send(n, NULL);
 756                 } else {
 757                         if (fib_lookup(net, fl4, &res, 0) == 0) {
 758                                 struct fib_nh *nh = &FIB_RES_NH(res);
 759
 760                                 update_or_create_fnhe(nh, fl4->daddr, new_gw,
 761                                                 0, jiffies + ip_rt_gc_timeout);
 762                         }
 763                         if (kill_route)
 764                                 rt->dst.obsolete = DST_OBSOLETE_KILL;
 765                         call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
 766                 }
 767                 neigh_release(n);
 768         }
 769         return;
 770
 771 reject_redirect:
 772 #ifdef CONFIG_IP_ROUTE_VERBOSE
 773         if (IN_DEV_LOG_MARTIANS(in_dev)) {
 774                 const struct iphdr *iph = (const struct iphdr *) skb->data;
 775                 __be32 daddr = iph->daddr;
 776                 __be32 saddr = iph->saddr;
 777
 778                 net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
 779                                      "  Advised path = %pI4 -> %pI4\n",
 780                                      &old_gw, dev->name, &new_gw,
 781                                      &saddr, &daddr);
 782         }
 783 #endif
 784         ;
 785 }
 786
 787 static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
 788 {
 789         struct rtable *rt;
 790         struct flowi4 fl4;
 791         const struct iphdr *iph = (const struct iphdr *) skb->data;
 792         int oif = skb->dev->ifindex;
 793         u8 tos = RT_TOS(iph->tos);
 794         u8 prot = iph->protocol;
 795         u32 mark = skb->mark;
 796
 797         rt = (struct rtable *) dst;
 798
 799         __build_flow_key(&fl4, sk, iph, oif, tos, prot, mark, 0);
 800         __ip_do_redirect(rt, skb, &fl4, true);
 801 }
 802
 803 static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
 804 {
 805         struct rtable *rt = (struct rtable *)dst;
 806         struct dst_entry *ret = dst;
 807
 808         if (rt) {
 809                 if (dst->obsolete > 0) {
 810                         ip_rt_put(rt);
 811                         ret = NULL;
 812                 } else if ((rt->rt_flags & RTCF_REDIRECTED) ||
 813                            rt->dst.expires) {
 814                         ip_rt_put(rt);
 815                         ret = NULL;
 816                 }
 817         }
 818         return ret;
 819 }
 820
 821 /*
 822  * Algorithm:
 823  *      1. The first ip_rt_redirect_number redirects are sent
 824  *         with exponential backoff, then we stop sending them at all,
 825  *         assuming that the host ignores our redirects.
 826  *      2. If we did not see packets requiring redirects
 827  *         during ip_rt_redirect_silence, we assume that the host
 828  *         forgot redirected route and start to send redirects again.
 829  *
 830  * This algorithm is much cheaper and more intelligent than dumb load limiting
 831  * in icmp.c.
 832  *
 833  * NOTE. Do not forget to inhibit load limiting for redirects (redundant)
 834  * and "frag. need" (breaks PMTU discovery) in icmp.c.
 835  */
 836
 837 void ip_rt_send_redirect(struct sk_buff *skb)
 838 {
 839         struct rtable *rt = skb_rtable(skb);
 840         struct in_device *in_dev;
 841         struct inet_peer *peer;
 842         struct net *net;
 843         int log_martians;
 844         int vif;
 845
 846         rcu_read_lock();
 847         in_dev = __in_dev_get_rcu(rt->dst.dev);
 848         if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
 849                 rcu_read_unlock();
 850                 return;
 851         }
 852         log_martians = IN_DEV_LOG_MARTIANS(in_dev);
 853         vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
 854         rcu_read_unlock();
 855
 856         net = dev_net(rt->dst.dev);
 857         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
 858         if (!peer) {
 859                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
 860                           rt_nexthop(rt, ip_hdr(skb)->daddr));
 861                 return;
 862         }
 863
 864         /* No redirected packets during ip_rt_redirect_silence;
 865          * reset the algorithm.
 866          */
 867         if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence))
 868                 peer->rate_tokens = 0;
 869
 870         /* Too many ignored redirects; do not send anything
 871          * set dst.rate_last to the last seen redirected packet.
 872          */
 873         if (peer->rate_tokens >= ip_rt_redirect_number) {
 874                 peer->rate_last = jiffies;
 875                 goto out_put_peer;
 876         }
 877
 878         /* Check for load limit; set rate_last to the latest sent
 879          * redirect.
 880          */
 881         if (peer->rate_tokens == 0 ||
 882             time_after(jiffies,
 883                        (peer->rate_last +
 884                         (ip_rt_redirect_load << peer->rate_tokens)))) {
 885                 __be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
 886
 887                 icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
 888                 peer->rate_last = jiffies;
 889                 ++peer->rate_tokens;
 890 #ifdef CONFIG_IP_ROUTE_VERBOSE
 891                 if (log_martians &&
 892                     peer->rate_tokens == ip_rt_redirect_number)
 893                         net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
 894                                              &ip_hdr(skb)->saddr, inet_iif(skb),
 895                                              &ip_hdr(skb)->daddr, &gw);
 896 #endif
 897         }
 898 out_put_peer:
 899         inet_putpeer(peer);
 900 }
 901
 902 static int ip_error(struct sk_buff *skb)
 903 {
 904         struct in_device *in_dev = __in_dev_get_rcu(skb->dev);
 905         struct rtable *rt = skb_rtable(skb);
 906         struct inet_peer *peer;
 907         unsigned long now;
 908         struct net *net;
 909         bool send;
 910         int code;
 911
 912         /* IP on this device is disabled. */
 913         if (!in_dev)
 914                 goto out;
 915
 916         net = dev_net(rt->dst.dev);
 917         if (!IN_DEV_FORWARD(in_dev)) {
 918                 switch (rt->dst.error) {
 919                 case EHOSTUNREACH:
 920                         IP_INC_STATS_BH(net, IPSTATS_MIB_INADDRERRORS);
 921                         break;
 922
 923                 case ENETUNREACH:
 924                         IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 925                         break;
 926                 }
 927                 goto out;
 928         }
 929
 930         switch (rt->dst.error) {
 931         case EINVAL:
 932         default:
 933                 goto out;
 934         case EHOSTUNREACH:
 935                 code = ICMP_HOST_UNREACH;
 936                 break;
 937         case ENETUNREACH:
 938                 code = ICMP_NET_UNREACH;
 939                 IP_INC_STATS_BH(net, IPSTATS_MIB_INNOROUTES);
 940                 break;
 941         case EACCES:
 942                 code = ICMP_PKT_FILTERED;
 943                 break;
 944         }
 945
 946         peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
 947                                l3mdev_master_ifindex(skb->dev), 1);
 948
 949         send = true;
 950         if (peer) {
 951                 now = jiffies;
 952                 peer->rate_tokens += now - peer->rate_last;
 953                 if (peer->rate_tokens > ip_rt_error_burst)
 954                         peer->rate_tokens = ip_rt_error_burst;
 955                 peer->rate_last = now;
 956                 if (peer->rate_tokens >= ip_rt_error_cost)
 957                         peer->rate_tokens -= ip_rt_error_cost;
 958                 else
 959                         send = false;
 960                 inet_putpeer(peer);
 961         }
 962         if (send)
 963                 icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
 964
 965 out:    kfree_skb(skb);
 966         return 0;
 967 }
 968
 969 static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
 970 {
 971         struct dst_entry *dst = &rt->dst;
 972         struct fib_result res;
 973
 974         if (dst_metric_locked(dst, RTAX_MTU))
 975                 return;
 976
 977         if (ipv4_mtu(dst) < mtu)
 978                 return;
 979
 980         if (mtu < ip_rt_min_pmtu)
 981                 mtu = ip_rt_min_pmtu;
 982
 983         if (rt->rt_pmtu == mtu &&
 984             time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
 985                 return;
 986
 987         rcu_read_lock();
 988         if (fib_lookup(dev_net(dst->dev), fl4, &res, 0) == 0) {
 989                 struct fib_nh *nh = &FIB_RES_NH(res);
 990
 991                 update_or_create_fnhe(nh, fl4->daddr, 0, mtu,
 992                                       jiffies + ip_rt_mtu_expires);
 993         }
 994         rcu_read_unlock();
 995 }
 996
 997 static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
 998                               struct sk_buff *skb, u32 mtu)
 999 {
1000         struct rtable *rt = (struct rtable *) dst;
1001         struct flowi4 fl4;
1002
1003         ip_rt_build_flow_key(&fl4, sk, skb);
1004         __ip_rt_update_pmtu(rt, &fl4, mtu);
1005 }
1006
1007 void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
1008                       int oif, u32 mark, u8 protocol, int flow_flags)
1009 {
1010         const struct iphdr *iph = (const struct iphdr *) skb->data;
1011         struct flowi4 fl4;
1012         struct rtable *rt;
1013
1014         if (!mark)
1015                 mark = IP4_REPLY_MARK(net, skb->mark);
1016
1017         __build_flow_key(&fl4, NULL, iph, oif,
1018                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1019         rt = __ip_route_output_key(net, &fl4);
1020         if (!IS_ERR(rt)) {
1021                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1022                 ip_rt_put(rt);
1023         }
1024 }
1025 EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
1026
1027 static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1028 {
1029         const struct iphdr *iph = (const struct iphdr *) skb->data;
1030         struct flowi4 fl4;
1031         struct rtable *rt;
1032
1033         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1034
1035         if (!fl4.flowi4_mark)
1036                 fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
1037
1038         rt = __ip_route_output_key(sock_net(sk), &fl4);
1039         if (!IS_ERR(rt)) {
1040                 __ip_rt_update_pmtu(rt, &fl4, mtu);
1041                 ip_rt_put(rt);
1042         }
1043 }
1044
1045 void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
1046 {
1047         const struct iphdr *iph = (const struct iphdr *) skb->data;
1048         struct flowi4 fl4;
1049         struct rtable *rt;
1050         struct dst_entry *odst = NULL;
1051         bool new = false;
1052
1053         bh_lock_sock(sk);
1054
1055         if (!ip_sk_accept_pmtu(sk))
1056                 goto out;
1057
1058         odst = sk_dst_get(sk);
1059
1060         if (sock_owned_by_user(sk) || !odst) {
1061                 __ipv4_sk_update_pmtu(skb, sk, mtu);
1062                 goto out;
1063         }
1064
1065         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1066
1067         rt = (struct rtable *)odst;
1068         if (odst->obsolete && !odst->ops->check(odst, 0)) {
1069                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1070                 if (IS_ERR(rt))
1071                         goto out;
1072
1073                 new = true;
1074         }
1075
1076         __ip_rt_update_pmtu((struct rtable *) rt->dst.path, &fl4, mtu);
1077
1078         if (!dst_check(&rt->dst, 0)) {
1079                 if (new)
1080                         dst_release(&rt->dst);
1081
1082                 rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
1083                 if (IS_ERR(rt))
1084                         goto out;
1085
1086                 new = true;
1087         }
1088
1089         if (new)
1090                 sk_dst_set(sk, &rt->dst);
1091
1092 out:
1093         bh_unlock_sock(sk);
1094         dst_release(odst);
1095 }
1096 EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
1097
1098 void ipv4_redirect(struct sk_buff *skb, struct net *net,
1099                    int oif, u32 mark, u8 protocol, int flow_flags)
1100 {
1101         const struct iphdr *iph = (const struct iphdr *) skb->data;
1102         struct flowi4 fl4;
1103         struct rtable *rt;
1104
1105         __build_flow_key(&fl4, NULL, iph, oif,
1106                          RT_TOS(iph->tos), protocol, mark, flow_flags);
1107         rt = __ip_route_output_key(net, &fl4);
1108         if (!IS_ERR(rt)) {
1109                 __ip_do_redirect(rt, skb, &fl4, false);
1110                 ip_rt_put(rt);
1111         }
1112 }
1113 EXPORT_SYMBOL_GPL(ipv4_redirect);
1114
1115 void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
1116 {
1117         const struct iphdr *iph = (const struct iphdr *) skb->data;
1118         struct flowi4 fl4;
1119         struct rtable *rt;
1120
1121         __build_flow_key(&fl4, sk, iph, 0, 0, 0, 0, 0);
1122         rt = __ip_route_output_key(sock_net(sk), &fl4);
1123         if (!IS_ERR(rt)) {
1124                 __ip_do_redirect(rt, skb, &fl4, false);
1125                 ip_rt_put(rt);
1126         }
1127 }
1128 EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
1129
1130 static struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie)
1131 {
1132         struct rtable *rt = (struct rtable *) dst;
1133
1134         /* All IPV4 dsts are created with ->obsolete set to the value
1135          * DST_OBSOLETE_FORCE_CHK which forces validation calls down
1136          * into this function always.
1137          *
1138          * When a PMTU/redirect information update invalidates a route,
1139          * this is indicated by setting obsolete to DST_OBSOLETE_KILL or
1140          * DST_OBSOLETE_DEAD by dst_free().
1141          */
1142         if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
1143                 return NULL;
1144         return dst;
1145 }
1146
1147 static void ipv4_link_failure(struct sk_buff *skb)
1148 {
1149         struct rtable *rt;
1150
1151         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
1152
1153         rt = skb_rtable(skb);
1154         if (rt)
1155                 dst_set_expires(&rt->dst, 0);
1156 }
1157
1158 static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
1159 {
1160         pr_debug("%s: %pI4 -> %pI4, %s\n",
1161                  __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
1162                  skb->dev ? skb->dev->name : "?");
1163         kfree_skb(skb);
1164         WARN_ON(1);
1165         return 0;
1166 }
1167
1168 /*
1169    We do not cache source address of outgoing interface,
1170    because it is used only by IP RR, TS and SRR options,
1171    so that it out of fast path.
1172
1173    BTW remember: "addr" is allowed to be not aligned
1174    in IP options!
1175  */
1176
1177 void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
1178 {
1179         __be32 src;
1180
1181         if (rt_is_output_route(rt))
1182                 src = ip_hdr(skb)->saddr;
1183         else {
1184                 struct fib_result res;
1185                 struct flowi4 fl4;
1186                 struct iphdr *iph;
1187
1188                 iph = ip_hdr(skb);
1189
1190                 memset(&fl4, 0, sizeof(fl4));
1191                 fl4.daddr = iph->daddr;
1192                 fl4.saddr = iph->saddr;
1193                 fl4.flowi4_tos = RT_TOS(iph->tos);
1194                 fl4.flowi4_oif = rt->dst.dev->ifindex;
1195                 fl4.flowi4_iif = skb->dev->ifindex;
1196                 fl4.flowi4_mark = skb->mark;
1197
1198                 rcu_read_lock();
1199                 if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
1200                         src = FIB_RES_PREFSRC(dev_net(rt->dst.dev), res);
1201                 else
1202                         src = inet_select_addr(rt->dst.dev,
1203                                                rt_nexthop(rt, iph->daddr),
1204                                                RT_SCOPE_UNIVERSE);
1205                 rcu_read_unlock();
1206         }
1207         memcpy(addr, &src, 4);
1208 }
1209
1210 #ifdef CONFIG_IP_ROUTE_CLASSID
1211 static void set_class_tag(struct rtable *rt, u32 tag)
1212 {
1213         if (!(rt->dst.tclassid & 0xFFFF))
1214                 rt->dst.tclassid |= tag & 0xFFFF;
1215         if (!(rt->dst.tclassid & 0xFFFF0000))
1216                 rt->dst.tclassid |= tag & 0xFFFF0000;
1217 }
1218 #endif
1219
1220 static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
1221 {
1222         unsigned int advmss = dst_metric_raw(dst, RTAX_ADVMSS);
1223
1224         if (advmss == 0) {
1225                 advmss = max_t(unsigned int, dst->dev->mtu - 40,
1226                                ip_rt_min_advmss);
1227                 if (advmss > 65535 - 40)
1228                         advmss = 65535 - 40;
1229         }
1230         return advmss;
1231 }
1232
1233 static unsigned int ipv4_mtu(const struct dst_entry *dst)
1234 {
1235         const struct rtable *rt = (const struct rtable *) dst;
1236         unsigned int mtu = rt->rt_pmtu;
1237
1238         if (!mtu || time_after_eq(jiffies, rt->dst.expires))
1239                 mtu = dst_metric_raw(dst, RTAX_MTU);
1240
1241         if (mtu)
1242                 return mtu;
1243
1244         mtu = dst->dev->mtu;
1245
1246         if (unlikely(dst_metric_locked(dst, RTAX_MTU))) {
1247                 if (rt->rt_uses_gateway && mtu > 576)
1248                         mtu = 576;
1249         }
1250
1251         return min_t(unsigned int, mtu, IP_MAX_MTU);
1252 }
1253
1254 static struct fib_nh_exception *find_exception(struct fib_nh *nh, __be32 daddr)
1255 {
1256         struct fnhe_hash_bucket *hash = rcu_dereference(nh->nh_exceptions);
1257         struct fib_nh_exception *fnhe;
1258         u32 hval;
1259
1260         if (!hash)
1261                 return NULL;
1262
1263         hval = fnhe_hashfun(daddr);
1264
1265         for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
1266              fnhe = rcu_dereference(fnhe->fnhe_next)) {
1267                 if (fnhe->fnhe_daddr == daddr)
1268                         return fnhe;
1269         }
1270         return NULL;
1271 }
1272
1273 static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
1274                               __be32 daddr)
1275 {
1276         bool ret = false;
1277
1278         spin_lock_bh(&fnhe_lock);
1279
1280         if (daddr == fnhe->fnhe_daddr) {
1281                 struct rtable __rcu **porig;
1282                 struct rtable *orig;
1283                 int genid = fnhe_genid(dev_net(rt->dst.dev));
1284
1285                 if (rt_is_input_route(rt))
1286                         porig = &fnhe->fnhe_rth_input;
1287                 else
1288                         porig = &fnhe->fnhe_rth_output;
1289                 orig = rcu_dereference(*porig);
1290
1291                 if (fnhe->fnhe_genid != genid) {
1292                         fnhe->fnhe_genid = genid;
1293                         fnhe->fnhe_gw = 0;
1294                         fnhe->fnhe_pmtu = 0;
1295                         fnhe->fnhe_expires = 0;
1296                         fnhe_flush_routes(fnhe);
1297                         orig = NULL;
1298                 }
1299                 fill_route_from_fnhe(rt, fnhe);
1300                 if (!rt->rt_gateway)
1301                         rt->rt_gateway = daddr;
1302
1303                 if (!(rt->dst.flags & DST_NOCACHE)) {
1304                         rcu_assign_pointer(*porig, rt);
1305                         if (orig)
1306                                 rt_free(orig);
1307                         ret = true;
1308                 }
1309
1310                 fnhe->fnhe_stamp = jiffies;
1311         }
1312         spin_unlock_bh(&fnhe_lock);
1313
1314         return ret;
1315 }
1316
1317 static bool rt_cache_route(struct fib_nh *nh, struct rtable *rt)
1318 {
1319         struct rtable *orig, *prev, **p;
1320         bool ret = true;
1321
1322         if (rt_is_input_route(rt)) {
1323                 p = (struct rtable **)&nh->nh_rth_input;
1324         } else {
1325                 p = (struct rtable **)raw_cpu_ptr(nh->nh_pcpu_rth_output);
1326         }
1327         orig = *p;
1328
1329         prev = cmpxchg(p, orig, rt);
1330         if (prev == orig) {
1331                 if (orig)
1332                         rt_free(orig);
1333         } else
1334                 ret = false;
1335
1336         return ret;
1337 }
1338
1339 struct uncached_list {
1340         spinlock_t              lock;
1341         struct list_head        head;
1342 };
1343
1344 static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
1345
1346 static void rt_add_uncached_list(struct rtable *rt)
1347 {
1348         struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
1349
1350         rt->rt_uncached_list = ul;
1351
1352         spin_lock_bh(&ul->lock);
1353         list_add_tail(&rt->rt_uncached, &ul->head);
1354         spin_unlock_bh(&ul->lock);
1355 }
1356
1357 static void ipv4_dst_destroy(struct dst_entry *dst)
1358 {
1359         struct rtable *rt = (struct rtable *) dst;
1360
1361         if (!list_empty(&rt->rt_uncached)) {
1362                 struct uncached_list *ul = rt->rt_uncached_list;
1363
1364                 spin_lock_bh(&ul->lock);
1365                 list_del(&rt->rt_uncached);
1366                 spin_unlock_bh(&ul->lock);
1367         }
1368 }
1369
1370 void rt_flush_dev(struct net_device *dev)
1371 {
1372         struct net *net = dev_net(dev);
1373         struct rtable *rt;
1374         int cpu;
1375
1376         for_each_possible_cpu(cpu) {
1377                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
1378
1379                 spin_lock_bh(&ul->lock);
1380                 list_for_each_entry(rt, &ul->head, rt_uncached) {
1381                         if (rt->dst.dev != dev)
1382                                 continue;
1383                         rt->dst.dev = net->loopback_dev;
1384                         dev_hold(rt->dst.dev);
1385                         dev_put(dev);
1386                 }
1387                 spin_unlock_bh(&ul->lock);
1388         }
1389 }
1390
1391 static bool rt_cache_valid(const struct rtable *rt)
1392 {
1393         return  rt &&
1394                 rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
1395                 !rt_is_expired(rt);
1396 }
1397
1398 static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
1399                            const struct fib_result *res,
1400                            struct fib_nh_exception *fnhe,
1401                            struct fib_info *fi, u16 type, u32 itag)
1402 {
1403         bool cached = false;
1404
1405         if (fi) {
1406                 struct fib_nh *nh = &FIB_RES_NH(*res);
1407
1408                 if (nh->nh_gw && nh->nh_scope == RT_SCOPE_LINK) {
1409                         rt->rt_gateway = nh->nh_gw;
1410                         rt->rt_uses_gateway = 1;
1411                 }
1412                 dst_init_metrics(&rt->dst, fi->fib_metrics, true);
1413 #ifdef CONFIG_IP_ROUTE_CLASSID
1414                 rt->dst.tclassid = nh->nh_tclassid;
1415 #endif
1416                 rt->dst.lwtstate = lwtstate_get(nh->nh_lwtstate);
1417                 if (unlikely(fnhe))
1418                         cached = rt_bind_exception(rt, fnhe, daddr);
1419                 else if (!(rt->dst.flags & DST_NOCACHE))
1420                         cached = rt_cache_route(nh, rt);
1421                 if (unlikely(!cached)) {
1422                         /* Routes we intend to cache in nexthop exception or
1423                          * FIB nexthop have the DST_NOCACHE bit clear.
1424                          * However, if we are unsuccessful at storing this
1425                          * route into the cache we really need to set it.
1426                          */
1427                         rt->dst.flags |= DST_NOCACHE;
1428                         if (!rt->rt_gateway)
1429                                 rt->rt_gateway = daddr;
1430                         rt_add_uncached_list(rt);
1431                 }
1432         } else
1433                 rt_add_uncached_list(rt);
1434
1435 #ifdef CONFIG_IP_ROUTE_CLASSID
1436 #ifdef CONFIG_IP_MULTIPLE_TABLES
1437         set_class_tag(rt, res->tclassid);
1438 #endif
1439         set_class_tag(rt, itag);
1440 #endif
1441 }
1442
1443 static struct rtable *rt_dst_alloc(struct net_device *dev,
1444                                    unsigned int flags, u16 type,
1445                                    bool nopolicy, bool noxfrm, bool will_cache)
1446 {
1447         struct rtable *rt;
1448
1449         rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
1450                        (will_cache ? 0 : (DST_HOST | DST_NOCACHE)) |
1451                        (nopolicy ? DST_NOPOLICY : 0) |
1452                        (noxfrm ? DST_NOXFRM : 0));
1453
1454         if (rt) {
1455                 rt->rt_genid = rt_genid_ipv4(dev_net(dev));
1456                 rt->rt_flags = flags;
1457                 rt->rt_type = type;
1458                 rt->rt_is_input = 0;
1459                 rt->rt_iif = 0;
1460                 rt->rt_pmtu = 0;
1461                 rt->rt_gateway = 0;
1462                 rt->rt_uses_gateway = 0;
1463                 rt->rt_table_id = 0;
1464                 INIT_LIST_HEAD(&rt->rt_uncached);
1465
1466                 rt->dst.output = ip_output;
1467                 if (flags & RTCF_LOCAL)
1468                         rt->dst.input = ip_local_deliver;
1469         }
1470
1471         return rt;
1472 }
1473
1474 /* called in rcu_read_lock() section */
1475 static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1476                                 u8 tos, struct net_device *dev, int our)
1477 {
1478         struct rtable *rth;
1479         struct in_device *in_dev = __in_dev_get_rcu(dev);
1480         unsigned int flags = RTCF_MULTICAST;
1481         u32 itag = 0;
1482         int err;
1483
1484         /* Primary sanity checks. */
1485
1486         if (!in_dev)
1487                 return -EINVAL;
1488
1489         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
1490             skb->protocol != htons(ETH_P_IP))
1491                 goto e_inval;
1492
1493         if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
1494                 goto e_inval;
1495
1496         if (ipv4_is_zeronet(saddr)) {
1497                 if (!ipv4_is_local_multicast(daddr))
1498                         goto e_inval;
1499         } else {
1500                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1501                                           in_dev, &itag);
1502                 if (err < 0)
1503                         goto e_err;
1504         }
1505         if (our)
1506                 flags |= RTCF_LOCAL;
1507
1508         rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
1509                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, false);
1510         if (!rth)
1511                 goto e_nobufs;
1512
1513 #ifdef CONFIG_IP_ROUTE_CLASSID
1514         rth->dst.tclassid = itag;
1515 #endif
1516         rth->dst.output = ip_rt_bug;
1517         rth->rt_is_input= 1;
1518
1519 #ifdef CONFIG_IP_MROUTE
1520         if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
1521                 rth->dst.input = ip_mr_input;
1522 #endif
1523         RT_CACHE_STAT_INC(in_slow_mc);
1524
1525         skb_dst_set(skb, &rth->dst);
1526         return 0;
1527
1528 e_nobufs:
1529         return -ENOBUFS;
1530 e_inval:
1531         return -EINVAL;
1532 e_err:
1533         return err;
1534 }
1535
1536
1537 static void ip_handle_martian_source(struct net_device *dev,
1538                                      struct in_device *in_dev,
1539                                      struct sk_buff *skb,
1540                                      __be32 daddr,
1541                                      __be32 saddr)
1542 {
1543         RT_CACHE_STAT_INC(in_martian_src);
1544 #ifdef CONFIG_IP_ROUTE_VERBOSE
1545         if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
1546                 /*
1547                  *      RFC1812 recommendation, if source is martian,
1548                  *      the only hint is MAC header.
1549                  */
1550                 pr_warn("martian source %pI4 from %pI4, on dev %s\n",
1551                         &daddr, &saddr, dev->name);
1552                 if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
1553                         print_hex_dump(KERN_WARNING, "ll header: ",
1554                                        DUMP_PREFIX_OFFSET, 16, 1,
1555                                        skb_mac_header(skb),
1556                                        dev->hard_header_len, true);
1557                 }
1558         }
1559 #endif
1560 }
1561
1562 static void ip_del_fnhe(struct fib_nh *nh, __be32 daddr)
1563 {
1564         struct fnhe_hash_bucket *hash;
1565         struct fib_nh_exception *fnhe, __rcu **fnhe_p;
1566         u32 hval = fnhe_hashfun(daddr);
1567
1568         spin_lock_bh(&fnhe_lock);
1569
1570         hash = rcu_dereference_protected(nh->nh_exceptions,
1571                                          lockdep_is_held(&fnhe_lock));
1572         hash += hval;
1573
1574         fnhe_p = &hash->chain;
1575         fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
1576         while (fnhe) {
1577                 if (fnhe->fnhe_daddr == daddr) {
1578                         rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
1579                                 fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
1580                         fnhe_flush_routes(fnhe);
1581                         kfree_rcu(fnhe, rcu);
1582                         break;
1583                 }
1584                 fnhe_p = &fnhe->fnhe_next;
1585                 fnhe = rcu_dereference_protected(fnhe->fnhe_next,
1586                                                  lockdep_is_held(&fnhe_lock));
1587         }
1588
1589         spin_unlock_bh(&fnhe_lock);
1590 }
1591
1592 /* called in rcu_read_lock() section */
1593 static int __mkroute_input(struct sk_buff *skb,
1594                            const struct fib_result *res,
1595                            struct in_device *in_dev,
1596                            __be32 daddr, __be32 saddr, u32 tos)
1597 {
1598         struct fib_nh_exception *fnhe;
1599         struct rtable *rth;
1600         int err;
1601         struct in_device *out_dev;
1602         bool do_cache;
1603         u32 itag = 0;
1604
1605         /* get a working reference to the output device */
1606         out_dev = __in_dev_get_rcu(FIB_RES_DEV(*res));
1607         if (!out_dev) {
1608                 net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
1609                 return -EINVAL;
1610         }
1611
1612         err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
1613                                   in_dev->dev, in_dev, &itag);
1614         if (err < 0) {
1615                 ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
1616                                          saddr);
1617
1618                 goto cleanup;
1619         }
1620
1621         do_cache = res->fi && !itag;
1622         if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
1623             skb->protocol == htons(ETH_P_IP) &&
1624             (IN_DEV_SHARED_MEDIA(out_dev) ||
1625              inet_addr_onlink(out_dev, saddr, FIB_RES_GW(*res))))
1626                 IPCB(skb)->flags |= IPSKB_DOREDIRECT;
1627
1628         if (skb->protocol != htons(ETH_P_IP)) {
1629                 /* Not IP (i.e. ARP). Do not create route, if it is
1630                  * invalid for proxy arp. DNAT routes are always valid.
1631                  *
1632                  * Proxy arp feature have been extended to allow, ARP
1633                  * replies back to the same interface, to support
1634                  * Private VLAN switch technologies. See arp.c.
1635                  */
1636                 if (out_dev == in_dev &&
1637                     IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
1638                         err = -EINVAL;
1639                         goto cleanup;
1640                 }
1641         }
1642
1643         fnhe = find_exception(&FIB_RES_NH(*res), daddr);
1644         if (do_cache) {
1645                 if (fnhe) {
1646                         rth = rcu_dereference(fnhe->fnhe_rth_input);
1647                         if (rth && rth->dst.expires &&
1648                             time_after(jiffies, rth->dst.expires)) {
1649                                 ip_del_fnhe(&FIB_RES_NH(*res), daddr);
1650                                 fnhe = NULL;
1651                         } else {
1652                                 goto rt_cache;
1653                         }
1654                 }
1655
1656                 rth = rcu_dereference(FIB_RES_NH(*res).nh_rth_input);
1657
1658 rt_cache:
1659                 if (rt_cache_valid(rth)) {
1660                         skb_dst_set_noref(skb, &rth->dst);
1661                         goto out;
1662                 }
1663         }
1664
1665         rth = rt_dst_alloc(out_dev->dev, 0, res->type,
1666                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
1667                            IN_DEV_CONF_GET(out_dev, NOXFRM), do_cache);
1668         if (!rth) {
1669                 err = -ENOBUFS;
1670                 goto cleanup;
1671         }
1672
1673         rth->rt_is_input = 1;
1674         if (res->table)
1675                 rth->rt_table_id = res->table->tb_id;
1676         RT_CACHE_STAT_INC(in_slow_tot);
1677
1678         rth->dst.input = ip_forward;
1679
1680         rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag);
1681         if (lwtunnel_output_redirect(rth->dst.lwtstate)) {
1682                 rth->dst.lwtstate->orig_output = rth->dst.output;
1683                 rth->dst.output = lwtunnel_output;
1684         }
1685         if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
1686                 rth->dst.lwtstate->orig_input = rth->dst.input;
1687                 rth->dst.input = lwtunnel_input;
1688         }
1689         skb_dst_set(skb, &rth->dst);
1690 out:
1691         err = 0;
1692  cleanup:
1693         return err;
1694 }
1695
1696 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1697
1698 /* To make ICMP packets follow the right flow, the multipath hash is
1699  * calculated from the inner IP addresses in reverse order.
1700  */
1701 static int ip_multipath_icmp_hash(struct sk_buff *skb)
1702 {
1703         const struct iphdr *outer_iph = ip_hdr(skb);
1704         struct icmphdr _icmph;
1705         const struct icmphdr *icmph;
1706         struct iphdr _inner_iph;
1707         const struct iphdr *inner_iph;
1708
1709         if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
1710                 goto standard_hash;
1711
1712         icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
1713                                    &_icmph);
1714         if (!icmph)
1715                 goto standard_hash;
1716
1717         if (icmph->type != ICMP_DEST_UNREACH &&
1718             icmph->type != ICMP_REDIRECT &&
1719             icmph->type != ICMP_TIME_EXCEEDED &&
1720             icmph->type != ICMP_PARAMETERPROB) {
1721                 goto standard_hash;
1722         }
1723
1724         inner_iph = skb_header_pointer(skb,
1725                                        outer_iph->ihl * 4 + sizeof(_icmph),
1726                                        sizeof(_inner_iph), &_inner_iph);
1727         if (!inner_iph)
1728                 goto standard_hash;
1729
1730         return fib_multipath_hash(inner_iph->daddr, inner_iph->saddr);
1731
1732 standard_hash:
1733         return fib_multipath_hash(outer_iph->saddr, outer_iph->daddr);
1734 }
1735
1736 #endif /* CONFIG_IP_ROUTE_MULTIPATH */
1737
1738 static int ip_mkroute_input(struct sk_buff *skb,
1739                             struct fib_result *res,
1740                             const struct flowi4 *fl4,
1741                             struct in_device *in_dev,
1742                             __be32 daddr, __be32 saddr, u32 tos)
1743 {
1744 #ifdef CONFIG_IP_ROUTE_MULTIPATH
1745         if (res->fi && res->fi->fib_nhs > 1) {
1746                 int h;
1747
1748                 if (unlikely(ip_hdr(skb)->protocol == IPPROTO_ICMP))
1749                         h = ip_multipath_icmp_hash(skb);
1750                 else
1751                         h = fib_multipath_hash(saddr, daddr);
1752                 fib_select_multipath(res, h);
1753         }
1754 #endif
1755
1756         /* create a routing cache entry */
1757         return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
1758 }
1759
1760 /*
1761  *      NOTE. We drop all the packets that has local source
1762  *      addresses, because every properly looped back packet
1763  *      must have correct destination already attached by output routine.
1764  *
1765  *      Such approach solves two big problems:
1766  *      1. Not simplex devices are handled properly.
1767  *      2. IP spoofing attempts are filtered with 100% of guarantee.
1768  *      called with rcu_read_lock()
1769  */
1770
1771 static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1772                                u8 tos, struct net_device *dev)
1773 {
1774         struct fib_result res;
1775         struct in_device *in_dev = __in_dev_get_rcu(dev);
1776         struct ip_tunnel_info *tun_info;
1777         struct flowi4   fl4;
1778         unsigned int    flags = 0;
1779         u32             itag = 0;
1780         struct rtable   *rth;
1781         int             err = -EINVAL;
1782         struct net    *net = dev_net(dev);
1783         bool do_cache;
1784
1785         /* IP on this device is disabled. */
1786
1787         if (!in_dev)
1788                 goto out;
1789
1790         /* Check for the most weird martians, which can be not detected
1791            by fib_lookup.
1792          */
1793
1794         tun_info = skb_tunnel_info(skb);
1795         if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
1796                 fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
1797         else
1798                 fl4.flowi4_tun_key.tun_id = 0;
1799         skb_dst_drop(skb);
1800
1801         if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
1802                 goto martian_source;
1803
1804         res.fi = NULL;
1805         res.table = NULL;
1806         if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
1807                 goto brd_input;
1808
1809         /* Accept zero addresses only to limited broadcast;
1810          * I even do not know to fix it or not. Waiting for complains :-)
1811          */
1812         if (ipv4_is_zeronet(saddr))
1813                 goto martian_source;
1814
1815         if (ipv4_is_zeronet(daddr))
1816                 goto martian_destination;
1817
1818         /* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
1819          * and call it once if daddr or/and saddr are loopback addresses
1820          */
1821         if (ipv4_is_loopback(daddr)) {
1822                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1823                         goto martian_destination;
1824         } else if (ipv4_is_loopback(saddr)) {
1825                 if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
1826                         goto martian_source;
1827         }
1828
1829         /*
1830          *      Now we are ready to route packet.
1831          */
1832         fl4.flowi4_oif = 0;
1833         fl4.flowi4_iif = l3mdev_fib_oif_rcu(dev);
1834         fl4.flowi4_mark = skb->mark;
1835         fl4.flowi4_tos = tos;
1836         fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
1837         fl4.flowi4_flags = 0;
1838         fl4.daddr = daddr;
1839         fl4.saddr = saddr;
1840         err = fib_lookup(net, &fl4, &res, 0);
1841         if (err != 0) {
1842                 if (!IN_DEV_FORWARD(in_dev))
1843                         err = -EHOSTUNREACH;
1844                 goto no_route;
1845         }
1846
1847         if (res.type == RTN_BROADCAST)
1848                 goto brd_input;
1849
1850         if (res.type == RTN_LOCAL) {
1851                 err = fib_validate_source(skb, saddr, daddr, tos,
1852                                           0, dev, in_dev, &itag);
1853                 if (err < 0)
1854                         goto martian_source;
1855                 goto local_input;
1856         }
1857
1858         if (!IN_DEV_FORWARD(in_dev)) {
1859                 err = -EHOSTUNREACH;
1860                 goto no_route;
1861         }
1862         if (res.type != RTN_UNICAST)
1863                 goto martian_destination;
1864
1865         err = ip_mkroute_input(skb, &res, &fl4, in_dev, daddr, saddr, tos);
1866 out:    return err;
1867
1868 brd_input:
1869         if (skb->protocol != htons(ETH_P_IP))
1870                 goto e_inval;
1871
1872         if (!ipv4_is_zeronet(saddr)) {
1873                 err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
1874                                           in_dev, &itag);
1875                 if (err < 0)
1876                         goto martian_source;
1877         }
1878         flags |= RTCF_BROADCAST;
1879         res.type = RTN_BROADCAST;
1880         RT_CACHE_STAT_INC(in_brd);
1881
1882 local_input:
1883         do_cache = false;
1884         if (res.fi) {
1885                 if (!itag) {
1886                         rth = rcu_dereference(FIB_RES_NH(res).nh_rth_input);
1887                         if (rt_cache_valid(rth)) {
1888                                 skb_dst_set_noref(skb, &rth->dst);
1889                                 err = 0;
1890                                 goto out;
1891                         }
1892                         do_cache = true;
1893                 }
1894         }
1895
1896         rth = rt_dst_alloc(net->loopback_dev, flags | RTCF_LOCAL, res.type,
1897                            IN_DEV_CONF_GET(in_dev, NOPOLICY), false, do_cache);
1898         if (!rth)
1899                 goto e_nobufs;
1900
1901         rth->dst.output= ip_rt_bug;
1902 #ifdef CONFIG_IP_ROUTE_CLASSID
1903         rth->dst.tclassid = itag;
1904 #endif
1905         rth->rt_is_input = 1;
1906         if (res.table)
1907                 rth->rt_table_id = res.table->tb_id;
1908
1909         RT_CACHE_STAT_INC(in_slow_tot);
1910         if (res.type == RTN_UNREACHABLE) {
1911                 rth->dst.input= ip_error;
1912                 rth->dst.error= -err;
1913                 rth->rt_flags   &= ~RTCF_LOCAL;
1914         }
1915         if (do_cache) {
1916                 if (unlikely(!rt_cache_route(&FIB_RES_NH(res), rth))) {
1917                         rth->dst.flags |= DST_NOCACHE;
1918                         rt_add_uncached_list(rth);
1919                 }
1920         }
1921         skb_dst_set(skb, &rth->dst);
1922         err = 0;
1923         goto out;
1924
1925 no_route:
1926         RT_CACHE_STAT_INC(in_no_route);
1927         res.type = RTN_UNREACHABLE;
1928         res.fi = NULL;
1929         res.table = NULL;
1930         goto local_input;
1931
1932         /*
1933          *      Do not cache martian addresses: they should be logged (RFC1812)
1934          */
1935 martian_destination:
1936         RT_CACHE_STAT_INC(in_martian_dst);
1937 #ifdef CONFIG_IP_ROUTE_VERBOSE
1938         if (IN_DEV_LOG_MARTIANS(in_dev))
1939                 net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
1940                                      &daddr, &saddr, dev->name);
1941 #endif
1942
1943 e_inval:
1944         err = -EINVAL;
1945         goto out;
1946
1947 e_nobufs:
1948         err = -ENOBUFS;
1949         goto out;
1950
1951 martian_source:
1952         ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
1953         goto out;
1954 }
1955
1956 int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
1957                          u8 tos, struct net_device *dev)
1958 {
1959         int res;
1960
1961         rcu_read_lock();
1962
1963         /* Multicast recognition logic is moved from route cache to here.
1964            The problem was that too many Ethernet cards have broken/missing
1965            hardware multicast filters :-( As result the host on multicasting
1966            network acquires a lot of useless route cache entries, sort of
1967            SDR messages from all the world. Now we try to get rid of them.
1968            Really, provided software IP multicast filter is organized
1969            reasonably (at least, hashed), it does not result in a slowdown
1970            comparing with route cache reject entries.
1971            Note, that multicast routers are not affected, because
1972            route cache entry is created eventually.
1973          */
1974         if (ipv4_is_multicast(daddr)) {
1975                 struct in_device *in_dev = __in_dev_get_rcu(dev);
1976
1977                 if (in_dev) {
1978                         int our = ip_check_mc_rcu(in_dev, daddr, saddr,
1979                                                   ip_hdr(skb)->protocol);
1980                         if (our
1981 #ifdef CONFIG_IP_MROUTE
1982                                 ||
1983                             (!ipv4_is_local_multicast(daddr) &&
1984                              IN_DEV_MFORWARD(in_dev))
1985 #endif
1986                            ) {
1987                                 int res = ip_route_input_mc(skb, daddr, saddr,
1988                                                             tos, dev, our);
1989                                 rcu_read_unlock();
1990                                 return res;
1991                         }
1992                 }
1993                 rcu_read_unlock();
1994                 return -EINVAL;
1995         }
1996         res = ip_route_input_slow(skb, daddr, saddr, tos, dev);
1997         rcu_read_unlock();
1998         return res;
1999 }
2000 EXPORT_SYMBOL(ip_route_input_noref);
2001
2002 /* called with rcu_read_lock() */
2003 static struct rtable *__mkroute_output(const struct fib_result *res,
2004                                        const struct flowi4 *fl4, int orig_oif,
2005                                        struct net_device *dev_out,
2006                                        unsigned int flags)
2007 {
2008         struct fib_info *fi = res->fi;
2009         struct fib_nh_exception *fnhe;
2010         struct in_device *in_dev;
2011         u16 type = res->type;
2012         struct rtable *rth;
2013         bool do_cache;
2014
2015         in_dev = __in_dev_get_rcu(dev_out);
2016         if (!in_dev)
2017                 return ERR_PTR(-EINVAL);
2018
2019         if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
2020                 if (ipv4_is_loopback(fl4->saddr) && !(dev_out->flags & IFF_LOOPBACK))
2021                         return ERR_PTR(-EINVAL);
2022
2023         if (ipv4_is_lbcast(fl4->daddr))
2024                 type = RTN_BROADCAST;
2025         else if (ipv4_is_multicast(fl4->daddr))
2026                 type = RTN_MULTICAST;
2027         else if (ipv4_is_zeronet(fl4->daddr))
2028                 return ERR_PTR(-EINVAL);
2029
2030         if (dev_out->flags & IFF_LOOPBACK)
2031                 flags |= RTCF_LOCAL;
2032
2033         do_cache = true;
2034         if (type == RTN_BROADCAST) {
2035                 flags |= RTCF_BROADCAST | RTCF_LOCAL;
2036                 fi = NULL;
2037         } else if (type == RTN_MULTICAST) {
2038                 flags |= RTCF_MULTICAST | RTCF_LOCAL;
2039                 if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
2040                                      fl4->flowi4_proto))
2041                         flags &= ~RTCF_LOCAL;
2042                 else
2043                         do_cache = false;
2044                 /* If multicast route do not exist use
2045                  * default one, but do not gateway in this case.
2046                  * Yes, it is hack.
2047                  */
2048                 if (fi && res->prefixlen < 4)
2049                         fi = NULL;
2050         } else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
2051                    (orig_oif != dev_out->ifindex)) {
2052                 /* For local routes that require a particular output interface
2053                  * we do not want to cache the result.  Caching the result
2054                  * causes incorrect behaviour when there are multiple source
2055                  * addresses on the interface, the end result being that if the
2056                  * intended recipient is waiting on that interface for the
2057                  * packet he won't receive it because it will be delivered on
2058                  * the loopback interface and the IP_PKTINFO ipi_ifindex will
2059                  * be set to the loopback interface as well.
2060                  */
2061                 fi = NULL;
2062         }
2063
2064         fnhe = NULL;
2065         do_cache &= fi != NULL;
2066         if (do_cache) {
2067                 struct rtable __rcu **prth;
2068                 struct fib_nh *nh = &FIB_RES_NH(*res);
2069
2070                 fnhe = find_exception(nh, fl4->daddr);
2071                 if (fnhe) {
2072                         prth = &fnhe->fnhe_rth_output;
2073                         rth = rcu_dereference(*prth);
2074                         if (rth && rth->dst.expires &&
2075                             time_after(jiffies, rth->dst.expires)) {
2076                                 ip_del_fnhe(nh, fl4->daddr);
2077                                 fnhe = NULL;
2078                         } else {
2079                                 goto rt_cache;
2080                         }
2081                 }
2082
2083                 if (unlikely(fl4->flowi4_flags &
2084                              FLOWI_FLAG_KNOWN_NH &&
2085                              !(nh->nh_gw &&
2086                                nh->nh_scope == RT_SCOPE_LINK))) {
2087                         do_cache = false;
2088                         goto add;
2089                 }
2090                 prth = raw_cpu_ptr(nh->nh_pcpu_rth_output);
2091                 rth = rcu_dereference(*prth);
2092
2093 rt_cache:
2094                 if (rt_cache_valid(rth)) {
2095                         dst_hold(&rth->dst);
2096                         return rth;
2097                 }
2098         }
2099
2100 add:
2101         rth = rt_dst_alloc(dev_out, flags, type,
2102                            IN_DEV_CONF_GET(in_dev, NOPOLICY),
2103                            IN_DEV_CONF_GET(in_dev, NOXFRM),
2104                            do_cache);
2105         if (!rth)
2106                 return ERR_PTR(-ENOBUFS);
2107
2108         rth->rt_iif     = orig_oif ? : 0;
2109         if (res->table)
2110                 rth->rt_table_id = res->table->tb_id;
2111
2112         RT_CACHE_STAT_INC(out_slow_tot);
2113
2114         if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
2115                 if (flags & RTCF_LOCAL &&
2116                     !(dev_out->flags & IFF_LOOPBACK)) {
2117                         rth->dst.output = ip_mc_output;
2118                         RT_CACHE_STAT_INC(out_slow_mc);
2119                 }
2120 #ifdef CONFIG_IP_MROUTE
2121                 if (type == RTN_MULTICAST) {
2122                         if (IN_DEV_MFORWARD(in_dev) &&
2123                             !ipv4_is_local_multicast(fl4->daddr)) {
2124                                 rth->dst.input = ip_mr_input;
2125                                 rth->dst.output = ip_mc_output;
2126                         }
2127                 }
2128 #endif
2129         }
2130
2131         rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0);
2132         if (lwtunnel_output_redirect(rth->dst.lwtstate))
2133                 rth->dst.output = lwtunnel_output;
2134
2135         return rth;
2136 }
2137
2138 /*
2139  * Major route resolver routine.
2140  */
2141
2142 struct rtable *__ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
2143                                           int mp_hash)
2144 {
2145         struct net_device *dev_out = NULL;
2146         __u8 tos = RT_FL_TOS(fl4);
2147         unsigned int flags = 0;
2148         struct fib_result res;
2149         struct rtable *rth;
2150         int orig_oif;
2151         int err = -ENETUNREACH;
2152
2153         res.tclassid    = 0;
2154         res.fi          = NULL;
2155         res.table       = NULL;
2156
2157         orig_oif = fl4->flowi4_oif;
2158
2159         fl4->flowi4_iif = LOOPBACK_IFINDEX;
2160         fl4->flowi4_tos = tos & IPTOS_RT_MASK;
2161         fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
2162                          RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
2163
2164         rcu_read_lock();
2165         if (fl4->saddr) {
2166                 rth = ERR_PTR(-EINVAL);
2167                 if (ipv4_is_multicast(fl4->saddr) ||
2168                     ipv4_is_lbcast(fl4->saddr) ||
2169                     ipv4_is_zeronet(fl4->saddr))
2170                         goto out;
2171
2172                 /* I removed check for oif == dev_out->oif here.
2173                    It was wrong for two reasons:
2174                    1. ip_dev_find(net, saddr) can return wrong iface, if saddr
2175                       is assigned to multiple interfaces.
2176                    2. Moreover, we are allowed to send packets with saddr
2177                       of another iface. --ANK
2178                  */
2179
2180                 if (fl4->flowi4_oif == 0 &&
2181                     (ipv4_is_multicast(fl4->daddr) ||
2182                      ipv4_is_lbcast(fl4->daddr))) {
2183                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2184                         dev_out = __ip_dev_find(net, fl4->saddr, false);
2185                         if (!dev_out)
2186                                 goto out;
2187
2188                         /* Special hack: user can direct multicasts
2189                            and limited broadcast via necessary interface
2190                            without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
2191                            This hack is not just for fun, it allows
2192                            vic,vat and friends to work.
2193                            They bind socket to loopback, set ttl to zero
2194                            and expect that it will work.
2195                            From the viewpoint of routing cache they are broken,
2196                            because we are not allowed to build multicast path
2197                            with loopback source addr (look, routing cache
2198                            cannot know, that ttl is zero, so that packet
2199                            will not leave this host and route is valid).
2200                            Luckily, this hack is good workaround.
2201                          */
2202
2203                         fl4->flowi4_oif = dev_out->ifindex;
2204                         goto make_route;
2205                 }
2206
2207                 if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
2208                         /* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
2209                         if (!__ip_dev_find(net, fl4->saddr, false))
2210                                 goto out;
2211                 }
2212         }
2213
2214
2215         if (fl4->flowi4_oif) {
2216                 dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
2217                 rth = ERR_PTR(-ENODEV);
2218                 if (!dev_out)
2219                         goto out;
2220
2221                 /* RACE: Check return value of inet_select_addr instead. */
2222                 if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
2223                         rth = ERR_PTR(-ENETUNREACH);
2224                         goto out;
2225                 }
2226                 if (ipv4_is_local_multicast(fl4->daddr) ||
2227                     ipv4_is_lbcast(fl4->daddr) ||
2228                     fl4->flowi4_proto == IPPROTO_IGMP) {
2229                         if (!fl4->saddr)
2230                                 fl4->saddr = inet_select_addr(dev_out, 0,
2231                                                               RT_SCOPE_LINK);
2232                         goto make_route;
2233                 }
2234                 if (!fl4->saddr) {
2235                         if (ipv4_is_multicast(fl4->daddr))
2236                                 fl4->saddr = inet_select_addr(dev_out, 0,
2237                                                               fl4->flowi4_scope);
2238                         else if (!fl4->daddr)
2239                                 fl4->saddr = inet_select_addr(dev_out, 0,
2240                                                               RT_SCOPE_HOST);
2241                 }
2242
2243                 rth = l3mdev_get_rtable(dev_out, fl4);
2244                 if (rth)
2245                         goto out;
2246         }
2247
2248         if (!fl4->daddr) {
2249                 fl4->daddr = fl4->saddr;
2250                 if (!fl4->daddr)
2251                         fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
2252                 dev_out = net->loopback_dev;
2253                 fl4->flowi4_oif = LOOPBACK_IFINDEX;
2254                 res.type = RTN_LOCAL;
2255                 flags |= RTCF_LOCAL;
2256                 goto make_route;
2257         }
2258
2259         err = fib_lookup(net, fl4, &res, 0);
2260         if (err) {
2261                 res.fi = NULL;
2262                 res.table = NULL;
2263                 if (fl4->flowi4_oif &&
2264                     !netif_index_is_l3_master(net, fl4->flowi4_oif)) {
2265                         /* Apparently, routing tables are wrong. Assume,
2266                            that the destination is on link.
2267
2268                            WHY? DW.
2269                            Because we are allowed to send to iface
2270                            even if it has NO routes and NO assigned
2271                            addresses. When oif is specified, routing
2272                            tables are looked up with only one purpose:
2273                            to catch if destination is gatewayed, rather than
2274                            direct. Moreover, if MSG_DONTROUTE is set,
2275                            we send packet, ignoring both routing tables
2276                            and ifaddr state. --ANK
2277
2278
2279                            We could make it even if oif is unknown,
2280                            likely IPv6, but we do not.
2281                          */
2282
2283                         if (fl4->saddr == 0)
2284                                 fl4->saddr = inet_select_addr(dev_out, 0,
2285                                                               RT_SCOPE_LINK);
2286                         res.type = RTN_UNICAST;
2287                         goto make_route;
2288                 }
2289                 rth = ERR_PTR(err);
2290                 goto out;
2291         }
2292
2293         if (res.type == RTN_LOCAL) {
2294                 if (!fl4->saddr) {
2295                         if (res.fi->fib_prefsrc)
2296                                 fl4->saddr = res.fi->fib_prefsrc;
2297                         else
2298                                 fl4->saddr = fl4->daddr;
2299                 }
2300                 dev_out = net->loopback_dev;
2301                 fl4->flowi4_oif = dev_out->ifindex;
2302                 flags |= RTCF_LOCAL;
2303                 goto make_route;
2304         }
2305
2306         fib_select_path(net, &res, fl4, mp_hash);
2307
2308         dev_out = FIB_RES_DEV(res);
2309         fl4->flowi4_oif = dev_out->ifindex;
2310
2311
2312 make_route:
2313         rth = __mkroute_output(&res, fl4, orig_oif, dev_out, flags);
2314
2315 out:
2316         rcu_read_unlock();
2317         return rth;
2318 }
2319 EXPORT_SYMBOL_GPL(__ip_route_output_key_hash);
2320
2321 static struct dst_entry *ipv4_blackhole_dst_check(struct dst_entry *dst, u32 cookie)
2322 {
2323         return NULL;
2324 }
2325
2326 static unsigned int ipv4_blackhole_mtu(const struct dst_entry *dst)
2327 {
2328         unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2329
2330         return mtu ? : dst->dev->mtu;
2331 }
2332
2333 static void ipv4_rt_blackhole_update_pmtu(struct dst_entry *dst, struct sock *sk,
2334                                           struct sk_buff *skb, u32 mtu)
2335 {
2336 }
2337
2338 static void ipv4_rt_blackhole_redirect(struct dst_entry *dst, struct sock *sk,
2339                                        struct sk_buff *skb)
2340 {
2341 }
2342
2343 static u32 *ipv4_rt_blackhole_cow_metrics(struct dst_entry *dst,
2344                                           unsigned long old)
2345 {
2346         return NULL;
2347 }
2348
2349 static struct dst_ops ipv4_dst_blackhole_ops = {
2350         .family                 =       AF_INET,
2351         .check                  =       ipv4_blackhole_dst_check,
2352         .mtu                    =       ipv4_blackhole_mtu,
2353         .default_advmss         =       ipv4_default_advmss,
2354         .update_pmtu            =       ipv4_rt_blackhole_update_pmtu,
2355         .redirect               =       ipv4_rt_blackhole_redirect,
2356         .cow_metrics            =       ipv4_rt_blackhole_cow_metrics,
2357         .neigh_lookup           =       ipv4_neigh_lookup,
2358 };
2359
2360 struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
2361 {
2362         struct rtable *ort = (struct rtable *) dst_orig;
2363         struct rtable *rt;
2364
2365         rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_NONE, 0);
2366         if (rt) {
2367                 struct dst_entry *new = &rt->dst;
2368
2369                 new->__use = 1;
2370                 new->input = dst_discard;
2371                 new->output = dst_discard_out;
2372
2373                 new->dev = ort->dst.dev;
2374                 if (new->dev)
2375                         dev_hold(new->dev);
2376
2377                 rt->rt_is_input = ort->rt_is_input;
2378                 rt->rt_iif = ort->rt_iif;
2379                 rt->rt_pmtu = ort->rt_pmtu;
2380
2381                 rt->rt_genid = rt_genid_ipv4(net);
2382                 rt->rt_flags = ort->rt_flags;
2383                 rt->rt_type = ort->rt_type;
2384                 rt->rt_gateway = ort->rt_gateway;
2385                 rt->rt_uses_gateway = ort->rt_uses_gateway;
2386
2387                 INIT_LIST_HEAD(&rt->rt_uncached);
2388                 dst_free(new);
2389         }
2390
2391         dst_release(dst_orig);
2392
2393         return rt ? &rt->dst : ERR_PTR(-ENOMEM);
2394 }
2395
2396 struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
2397                                     const struct sock *sk)
2398 {
2399         struct rtable *rt = __ip_route_output_key(net, flp4);
2400
2401         if (IS_ERR(rt))
2402                 return rt;
2403
2404         if (flp4->flowi4_proto)
2405                 rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
2406                                                         flowi4_to_flowi(flp4),
2407                                                         sk, 0);
2408
2409         return rt;
2410 }
2411 EXPORT_SYMBOL_GPL(ip_route_output_flow);
2412
2413 static int rt_fill_info(struct net *net,  __be32 dst, __be32 src, u32 table_id,
2414                         struct flowi4 *fl4, struct sk_buff *skb, u32 portid,
2415                         u32 seq, int event, int nowait, unsigned int flags)
2416 {
2417         struct rtable *rt = skb_rtable(skb);
2418         struct rtmsg *r;
2419         struct nlmsghdr *nlh;
2420         unsigned long expires = 0;
2421         u32 error;
2422         u32 metrics[RTAX_MAX];
2423
2424         nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
2425         if (!nlh)
2426                 return -EMSGSIZE;
2427
2428         r = nlmsg_data(nlh);
2429         r->rtm_family    = AF_INET;
2430         r->rtm_dst_len  = 32;
2431         r->rtm_src_len  = 0;
2432         r->rtm_tos      = fl4->flowi4_tos;
2433         r->rtm_table    = table_id < 256 ? table_id : RT_TABLE_COMPAT;
2434         if (nla_put_u32(skb, RTA_TABLE, table_id))
2435                 goto nla_put_failure;
2436         r->rtm_type     = rt->rt_type;
2437         r->rtm_scope    = RT_SCOPE_UNIVERSE;
2438         r->rtm_protocol = RTPROT_UNSPEC;
2439         r->rtm_flags    = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
2440         if (rt->rt_flags & RTCF_NOTIFY)
2441                 r->rtm_flags |= RTM_F_NOTIFY;
2442         if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
2443                 r->rtm_flags |= RTCF_DOREDIRECT;
2444
2445         if (nla_put_in_addr(skb, RTA_DST, dst))
2446                 goto nla_put_failure;
2447         if (src) {
2448                 r->rtm_src_len = 32;
2449                 if (nla_put_in_addr(skb, RTA_SRC, src))
2450                         goto nla_put_failure;
2451         }
2452         if (rt->dst.dev &&
2453             nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
2454                 goto nla_put_failure;
2455 #ifdef CONFIG_IP_ROUTE_CLASSID
2456         if (rt->dst.tclassid &&
2457             nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
2458                 goto nla_put_failure;
2459 #endif
2460         if (!rt_is_input_route(rt) &&
2461             fl4->saddr != src) {
2462                 if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
2463                         goto nla_put_failure;
2464         }
2465         if (rt->rt_uses_gateway &&
2466             nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gateway))
2467                 goto nla_put_failure;
2468
2469         expires = rt->dst.expires;
2470         if (expires) {
2471                 unsigned long now = jiffies;
2472
2473                 if (time_before(now, expires))
2474                         expires -= now;
2475                 else
2476                         expires = 0;
2477         }
2478
2479         memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
2480         if (rt->rt_pmtu && expires)
2481                 metrics[RTAX_MTU - 1] = rt->rt_pmtu;
2482         if (rtnetlink_put_metrics(skb, metrics) < 0)
2483                 goto nla_put_failure;
2484
2485         if (fl4->flowi4_mark &&
2486             nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
2487                 goto nla_put_failure;
2488
2489         error = rt->dst.error;
2490
2491         if (rt_is_input_route(rt)) {
2492 #ifdef CONFIG_IP_MROUTE
2493                 if (ipv4_is_multicast(dst) && !ipv4_is_local_multicast(dst) &&
2494                     IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
2495                         int err = ipmr_get_route(net, skb,
2496                                                  fl4->saddr, fl4->daddr,
2497                                                  r, nowait, portid);
2498
2499                         if (err <= 0) {
2500                                 if (!nowait) {
2501                                         if (err == 0)
2502                                                 return 0;
2503                                         goto nla_put_failure;
2504                                 } else {
2505                                         if (err == -EMSGSIZE)
2506                                                 goto nla_put_failure;
2507                                         error = err;
2508                                 }
2509                         }
2510                 } else
2511 #endif
2512                         if (nla_put_u32(skb, RTA_IIF, skb->dev->ifindex))
2513                                 goto nla_put_failure;
2514         }
2515
2516         if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
2517                 goto nla_put_failure;
2518
2519         nlmsg_end(skb, nlh);
2520         return 0;
2521
2522 nla_put_failure:
2523         nlmsg_cancel(skb, nlh);
2524         return -EMSGSIZE;
2525 }
2526
2527 static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh)
2528 {
2529         struct net *net = sock_net(in_skb->sk);
2530         struct rtmsg *rtm;
2531         struct nlattr *tb[RTA_MAX+1];
2532         struct rtable *rt = NULL;
2533         struct flowi4 fl4;
2534         __be32 dst = 0;
2535         __be32 src = 0;
2536         u32 iif;
2537         int err;
2538         int mark;
2539         struct sk_buff *skb;
2540         u32 table_id = RT_TABLE_MAIN;
2541
2542         err = nlmsg_parse(nlh, sizeof(*rtm), tb, RTA_MAX, rtm_ipv4_policy);
2543         if (err < 0)
2544                 goto errout;
2545
2546         rtm = nlmsg_data(nlh);
2547
2548         skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
2549         if (!skb) {
2550                 err = -ENOBUFS;
2551                 goto errout;
2552         }
2553
2554         /* Reserve room for dummy headers, this skb can pass
2555            through good chunk of routing engine.
2556          */
2557         skb_reset_mac_header(skb);
2558         skb_reset_network_header(skb);
2559
2560         /* Bugfix: need to give ip_route_input enough of an IP header to not gag. */
2561         ip_hdr(skb)->protocol = IPPROTO_ICMP;
2562         skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
2563
2564         src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
2565         dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
2566         iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
2567         mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
2568
2569         memset(&fl4, 0, sizeof(fl4));
2570         fl4.daddr = dst;
2571         fl4.saddr = src;
2572         fl4.flowi4_tos = rtm->rtm_tos;
2573         fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
2574         fl4.flowi4_mark = mark;
2575
2576         if (netif_index_is_l3_master(net, fl4.flowi4_oif))
2577                 fl4.flowi4_flags = FLOWI_FLAG_L3MDEV_SRC | FLOWI_FLAG_SKIP_NH_OIF;
2578
2579         if (iif) {
2580                 struct net_device *dev;
2581
2582                 dev = __dev_get_by_index(net, iif);
2583                 if (!dev) {
2584                         err = -ENODEV;
2585                         goto errout_free;
2586                 }
2587
2588                 skb->protocol   = htons(ETH_P_IP);
2589                 skb->dev        = dev;
2590                 skb->mark       = mark;
2591                 local_bh_disable();
2592                 err = ip_route_input(skb, dst, src, rtm->rtm_tos, dev);
2593                 local_bh_enable();
2594
2595                 rt = skb_rtable(skb);
2596                 if (err == 0 && rt->dst.error)
2597                         err = -rt->dst.error;
2598         } else {
2599                 rt = ip_route_output_key(net, &fl4);
2600
2601                 err = 0;
2602                 if (IS_ERR(rt))
2603                         err = PTR_ERR(rt);
2604         }
2605
2606         if (err)
2607                 goto errout_free;
2608
2609         skb_dst_set(skb, &rt->dst);
2610         if (rtm->rtm_flags & RTM_F_NOTIFY)
2611                 rt->rt_flags |= RTCF_NOTIFY;
2612
2613         if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
2614                 table_id = rt->rt_table_id;
2615
2616         err = rt_fill_info(net, dst, src, table_id, &fl4, skb,
2617                            NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
2618                            RTM_NEWROUTE, 0, 0);
2619         if (err < 0)
2620                 goto errout_free;
2621
2622         err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
2623 errout:
2624         return err;
2625
2626 errout_free:
2627         kfree_skb(skb);
2628         goto errout;
2629 }
2630
2631 void ip_rt_multicast_event(struct in_device *in_dev)
2632 {
2633         rt_cache_flush(dev_net(in_dev->dev));
2634 }
2635
2636 #ifdef CONFIG_SYSCTL
2637 static int ip_rt_gc_interval __read_mostly  = 60 * HZ;
2638 static int ip_rt_gc_min_interval __read_mostly  = HZ / 2;
2639 static int ip_rt_gc_elasticity __read_mostly    = 8;
2640
2641 static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
2642                                         void __user *buffer,
2643                                         size_t *lenp, loff_t *ppos)
2644 {
2645         struct net *net = (struct net *)__ctl->extra1;
2646
2647         if (write) {
2648                 rt_cache_flush(net);
2649                 fnhe_genid_bump(net);
2650                 return 0;
2651         }
2652
2653         return -EINVAL;
2654 }
2655
2656 static struct ctl_table ipv4_route_table[] = {
2657         {
2658                 .procname       = "gc_thresh",
2659                 .data           = &ipv4_dst_ops.gc_thresh,
2660                 .maxlen         = sizeof(int),
2661                 .mode           = 0644,
2662                 .proc_handler   = proc_dointvec,
2663         },
2664         {
2665                 .procname       = "max_size",
2666                 .data           = &ip_rt_max_size,
2667                 .maxlen         = sizeof(int),
2668                 .mode           = 0644,
2669                 .proc_handler   = proc_dointvec,
2670         },
2671         {
2672                 /*  Deprecated. Use gc_min_interval_ms */
2673
2674                 .procname       = "gc_min_interval",
2675                 .data           = &ip_rt_gc_min_interval,
2676                 .maxlen         = sizeof(int),
2677                 .mode           = 0644,
2678                 .proc_handler   = proc_dointvec_jiffies,
2679         },
2680         {
2681                 .procname       = "gc_min_interval_ms",
2682                 .data           = &ip_rt_gc_min_interval,
2683                 .maxlen         = sizeof(int),
2684                 .mode           = 0644,
2685                 .proc_handler   = proc_dointvec_ms_jiffies,
2686         },
2687         {
2688                 .procname       = "gc_timeout",
2689                 .data           = &ip_rt_gc_timeout,
2690                 .maxlen         = sizeof(int),
2691                 .mode           = 0644,
2692                 .proc_handler   = proc_dointvec_jiffies,
2693         },
2694         {
2695                 .procname       = "gc_interval",
2696                 .data           = &ip_rt_gc_interval,
2697                 .maxlen         = sizeof(int),
2698                 .mode           = 0644,
2699                 .proc_handler   = proc_dointvec_jiffies,
2700         },
2701         {
2702                 .procname       = "redirect_load",
2703                 .data           = &ip_rt_redirect_load,
2704                 .maxlen         = sizeof(int),
2705                 .mode           = 0644,
2706                 .proc_handler   = proc_dointvec,
2707         },
2708         {
2709                 .procname       = "redirect_number",
2710                 .data           = &ip_rt_redirect_number,
2711                 .maxlen         = sizeof(int),
2712                 .mode           = 0644,
2713                 .proc_handler   = proc_dointvec,
2714         },
2715         {
2716                 .procname       = "redirect_silence",
2717                 .data           = &ip_rt_redirect_silence,
2718                 .maxlen         = sizeof(int),
2719                 .mode           = 0644,
2720                 .proc_handler   = proc_dointvec,
2721         },
2722         {
2723                 .procname       = "error_cost",
2724                 .data           = &ip_rt_error_cost,
2725                 .maxlen         = sizeof(int),
2726                 .mode           = 0644,
2727                 .proc_handler   = proc_dointvec,
2728         },
2729         {
2730                 .procname       = "error_burst",
2731                 .data           = &ip_rt_error_burst,
2732                 .maxlen         = sizeof(int),
2733                 .mode           = 0644,
2734                 .proc_handler   = proc_dointvec,
2735         },
2736         {
2737                 .procname       = "gc_elasticity",
2738                 .data           = &ip_rt_gc_elasticity,
2739                 .maxlen         = sizeof(int),
2740                 .mode           = 0644,
2741                 .proc_handler   = proc_dointvec,
2742         },
2743         {
2744                 .procname       = "mtu_expires",
2745                 .data           = &ip_rt_mtu_expires,
2746                 .maxlen         = sizeof(int),
2747                 .mode           = 0644,
2748                 .proc_handler   = proc_dointvec_jiffies,
2749         },
2750         {
2751                 .procname       = "min_pmtu",
2752                 .data           = &ip_rt_min_pmtu,
2753                 .maxlen         = sizeof(int),
2754                 .mode           = 0644,
2755                 .proc_handler   = proc_dointvec,
2756         },
2757         {
2758                 .procname       = "min_adv_mss",
2759                 .data           = &ip_rt_min_advmss,
2760                 .maxlen         = sizeof(int),
2761                 .mode           = 0644,
2762                 .proc_handler   = proc_dointvec,
2763         },
2764         { }
2765 };
2766
2767 static struct ctl_table ipv4_route_flush_table[] = {
2768         {
2769                 .procname       = "flush",
2770                 .maxlen         = sizeof(int),
2771                 .mode           = 0200,
2772                 .proc_handler   = ipv4_sysctl_rtcache_flush,
2773         },
2774         { },
2775 };
2776
2777 static __net_init int sysctl_route_net_init(struct net *net)
2778 {
2779         struct ctl_table *tbl;
2780
2781         tbl = ipv4_route_flush_table;
2782         if (!net_eq(net, &init_net)) {
2783                 tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
2784                 if (!tbl)
2785                         goto err_dup;
2786
2787                 /* Don't export sysctls to unprivileged users */
2788                 if (net->user_ns != &init_user_ns)
2789                         tbl[0].procname = NULL;
2790         }
2791         tbl[0].extra1 = net;
2792
2793         net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
2794         if (!net->ipv4.route_hdr)
2795                 goto err_reg;
2796         return 0;
2797
2798 err_reg:
2799         if (tbl != ipv4_route_flush_table)
2800                 kfree(tbl);
2801 err_dup:
2802         return -ENOMEM;
2803 }
2804
2805 static __net_exit void sysctl_route_net_exit(struct net *net)
2806 {
2807         struct ctl_table *tbl;
2808
2809         tbl = net->ipv4.route_hdr->ctl_table_arg;
2810         unregister_net_sysctl_table(net->ipv4.route_hdr);
2811         BUG_ON(tbl == ipv4_route_flush_table);
2812         kfree(tbl);
2813 }
2814
2815 static __net_initdata struct pernet_operations sysctl_route_ops = {
2816         .init = sysctl_route_net_init,
2817         .exit = sysctl_route_net_exit,
2818 };
2819 #endif
2820
2821 static __net_init int rt_genid_init(struct net *net)
2822 {
2823         atomic_set(&net->ipv4.rt_genid, 0);
2824         atomic_set(&net->fnhe_genid, 0);
2825         get_random_bytes(&net->ipv4.dev_addr_genid,
2826                          sizeof(net->ipv4.dev_addr_genid));
2827         return 0;
2828 }
2829
2830 static __net_initdata struct pernet_operations rt_genid_ops = {
2831         .init = rt_genid_init,
2832 };
2833
2834 static int __net_init ipv4_inetpeer_init(struct net *net)
2835 {
2836         struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
2837
2838         if (!bp)
2839                 return -ENOMEM;
2840         inet_peer_base_init(bp);
2841         net->ipv4.peers = bp;
2842         return 0;
2843 }
2844
2845 static void __net_exit ipv4_inetpeer_exit(struct net *net)
2846 {
2847         struct inet_peer_base *bp = net->ipv4.peers;
2848
2849         net->ipv4.peers = NULL;
2850         inetpeer_invalidate_tree(bp);
2851         kfree(bp);
2852 }
2853
2854 static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
2855         .init   =       ipv4_inetpeer_init,
2856         .exit   =       ipv4_inetpeer_exit,
2857 };
2858
2859 #ifdef CONFIG_IP_ROUTE_CLASSID
2860 struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
2861 #endif /* CONFIG_IP_ROUTE_CLASSID */
2862
2863 int __init ip_rt_init(void)
2864 {
2865         int rc = 0;
2866         int cpu;
2867
2868         ip_idents = kmalloc(IP_IDENTS_SZ * sizeof(*ip_idents), GFP_KERNEL);
2869         if (!ip_idents)
2870                 panic("IP: failed to allocate ip_idents\n");
2871
2872         prandom_bytes(ip_idents, IP_IDENTS_SZ * sizeof(*ip_idents));
2873
2874         ip_tstamps = kcalloc(IP_IDENTS_SZ, sizeof(*ip_tstamps), GFP_KERNEL);
2875         if (!ip_tstamps)
2876                 panic("IP: failed to allocate ip_tstamps\n");
2877
2878         for_each_possible_cpu(cpu) {
2879                 struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
2880
2881                 INIT_LIST_HEAD(&ul->head);
2882                 spin_lock_init(&ul->lock);
2883         }
2884 #ifdef CONFIG_IP_ROUTE_CLASSID
2885         ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
2886         if (!ip_rt_acct)
2887                 panic("IP: failed to allocate ip_rt_acct\n");
2888 #endif
2889
2890         ipv4_dst_ops.kmem_cachep =
2891                 kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
2892                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
2893
2894         ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
2895
2896         if (dst_entries_init(&ipv4_dst_ops) < 0)
2897                 panic("IP: failed to allocate ipv4_dst_ops counter\n");
2898
2899         if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
2900                 panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
2901
2902         ipv4_dst_ops.gc_thresh = ~0;
2903         ip_rt_max_size = INT_MAX;
2904
2905         devinet_init();
2906         ip_fib_init();
2907
2908         if (ip_rt_proc_init())
2909                 pr_err("Unable to create route proc files\n");
2910 #ifdef CONFIG_XFRM
2911         xfrm_init();
2912         xfrm4_init();
2913 #endif
2914         rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL, NULL);
2915
2916 #ifdef CONFIG_SYSCTL
2917         register_pernet_subsys(&sysctl_route_ops);
2918 #endif
2919         register_pernet_subsys(&rt_genid_ops);
2920         register_pernet_subsys(&ipv4_inetpeer_ops);
2921         return rc;
2922 }
2923
2924 #ifdef CONFIG_SYSCTL
2925 /*
2926  * We really need to sanitize the damn ipv4 init order, then all
2927  * this nonsense will go away.
2928  */
2929 void __init ip_static_sysctl_init(void)
2930 {
2931         register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
2932 }
2933 #endif