VNFs/UDP_Replay/main.c

   1 /*
   2 // Copyright (c) 2017 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //      http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 /* Changes for Correlated traffic VNF
  18 1.      Receive UDP packet
  19 2.      Modify received packet
  20            a.exchange src mac and destination mac
  21            b.exchange src ip and destination IP for both IPv4 and IPv6 cases
  22            c.exchange UDP src port and UDP destination port
  23            d.change the len of the response according to the IMIX definition (
  24                option to make traffic more realistic to emulate some IoT payloads)
  25 3.      send modified packet to the port where it was received.
  26
  27 Such VNF does not need LPM and routing table implementations.
  28 As the packet modification is very minimal  and there is no memory access as the packet is stored in L3 cache the
  29 performance of the solution should be sufficient for testing the UDP NAT performance.
  30 */
  31 #include <stdlib.h>
  32 #include <stdint.h>
  33 #include <inttypes.h>
  34 #include <sys/types.h>
  35 #include <string.h>
  36 #include <sys/queue.h>
  37 #include <stdarg.h>
  38 #include <errno.h>
  39 #include <getopt.h>
  40
  41 #include <rte_common.h>
  42 #include <rte_vect.h>
  43 #include <rte_byteorder.h>
  44 #include <rte_log.h>
  45 #include <rte_memory.h>
  46 #include <rte_memcpy.h>
  47 #include <rte_memzone.h>
  48 #include <rte_eal.h>
  49 #include <rte_per_lcore.h>
  50 #include <rte_launch.h>
  51 #include <rte_atomic.h>
  52 #include <rte_cycles.h>
  53 #include <rte_prefetch.h>
  54 #include <rte_lcore.h>
  55 #include <rte_per_lcore.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_interrupts.h>
  58 #include <rte_pci.h>
  59 #include <rte_random.h>
  60 #include <rte_debug.h>
  61 #include <rte_ether.h>
  62 #include <rte_ethdev.h>
  63 #include <rte_ring.h>
  64 #include <rte_mempool.h>
  65 #include <rte_mbuf.h>
  66 #include <rte_ip.h>
  67 #include <rte_tcp.h>
  68 #include <rte_udp.h>
  69 #include <rte_string_fns.h>
  70
  71 #include <cmdline_parse.h>
  72 #include <cmdline_parse_etheraddr.h>
  73 #include <cmdline_rdline.h>
  74 #include <cmdline_socket.h>
  75 #include <cmdline.h>
  76 #include <cmdline_parse_num.h>
  77 #include <cmdline_parse_string.h>
  78 #include <cmdline_parse_ipaddr.h>
  79 #include <rte_errno.h>
  80 #include <rte_cfgfile.h>
  81
  82 #include "parse_obj_list.h"
  83
  84 #include <lib_arp.h>
  85 #include "l2_proto.h"
  86 #include "interface.h"
  87 #include "l3fwd_common.h"
  88 #include "l3fwd_lpm4.h"
  89 #include "l3fwd_lpm6.h"
  90 #include "lib_icmpv6.h"
  91 #include "app.h"
  92 #include "vnf_common.h"
  93 #define IN6ADDRSZ 16
  94 #define INADDRSZ 4
  95 #define APP_LOOKUP_EXACT_MATCH          0
  96 #define APP_LOOKUP_LPM                  1
  97 #define DO_RFC_1812_CHECKS
  98 #if 1
  99 #ifndef APP_LOOKUP_METHOD
 100 #define APP_LOOKUP_METHOD             APP_LOOKUP_EXACT_MATCH
 101 #endif
 102 #endif
 103 /*raji*/
 104
 105 #include <stdio.h>
 106 #include <netinet/in.h>
 107 #include <termios.h>
 108 /*
 109  *  When set to zero, simple forwaring path is eanbled.
 110  *  When set to one, optimized forwarding path is enabled.
 111  *  Note that LPM optimisation path uses SSE4.1 instructions.
 112  */
 113 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__))
 114 #define ENABLE_MULTI_BUFFER_OPTIMIZE    0
 115 #else
 116 #define ENABLE_MULTI_BUFFER_OPTIMIZE    1
 117 #endif
 118
 119 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 120 #include <rte_hash.h>
 121 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 122 #include <rte_lpm.h>
 123 #include <rte_lpm6.h>
 124 #else
 125 #error "APP_LOOKUP_METHOD set to incorrect value"
 126 #endif
 127
 128 #ifndef IPv6_BYTES
 129 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
 130                        "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
 131 #define IPv6_BYTES(addr) \
 132         addr[0],  addr[1], addr[2],  addr[3], \
 133         addr[4],  addr[5], addr[6],  addr[7], \
 134         addr[8],  addr[9], addr[10], addr[11],\
 135         addr[12], addr[13],addr[14], addr[15]
 136 #endif
 137
 138
 139 #define RTE_LOGTYPE_L3FWD RTE_LOGTYPE_USER1
 140
 141 #define MAX_JUMBO_PKT_LEN  9600
 142
 143 #define IPV6_ADDR_LEN 16
 144
 145 #define MEMPOOL_CACHE_SIZE 256
 146
 147 /*
 148  * This expression is used to calculate the number of mbufs needed depending on user input, taking
 149  *  into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore.
 150  *  RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192
 151  */
 152
 153 #define NB_MBUF RTE_MAX (                                                                                                                                       \
 154                                 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT +                                                        \
 155                                 nb_ports*nb_lcores*MAX_PKT_BURST +                                                                                      \
 156                                 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT +                                                          \
 157                                 nb_lcores*MEMPOOL_CACHE_SIZE),                                                                                          \
 158                                 (unsigned)8192)
 159
 160 #define MAX_PKT_BURST     32
 161 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
 162
 163 /*
 164  * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send.
 165  */
 166 #define MAX_TX_BURST    (MAX_PKT_BURST / 2)
 167
 168 #define NB_SOCKETS 8
 169
 170 /* Configure how many packets ahead to prefetch, when reading packets */
 171 #define PREFETCH_OFFSET 3
 172
 173 /* Used to mark destination port as 'invalid'. */
 174 #define BAD_PORT        ((uint16_t)-1)
 175
 176 #define FWDSTEP 4
 177
 178 /*
 179  * Configurable number of RX/TX ring descriptors
 180  */
 181 #define RTE_TEST_RX_DESC_DEFAULT 128
 182 #define RTE_TEST_TX_DESC_DEFAULT 512
 183 static uint64_t rcv_pkt_count[32] = {0};
 184 static uint64_t tx_pkt_count[32] = {0};
 185 unsigned num_ports;
 186 struct sockaddr_in ipaddr1, ipaddr2;
 187
 188 /* ethernet addresses of ports */
 189 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
 190 static __m128i val_eth[RTE_MAX_ETHPORTS];
 191
 192 cmdline_parse_ctx_t main_ctx[];
 193
 194 uint32_t timer_lcore;
 195 uint32_t exit_loop = 1;
 196
 197 port_config_t *port_config;
 198 #define MEMPOOL_SIZE    32 * 1024
 199 #define BUFFER_SIZE             2048
 200 #define CACHE_SIZE              256
 201
 202
 203 /* replace first 12B of the ethernet header. */
 204 #define MASK_ETH        0x3f
 205
 206 #define IP_TYPE_IPV4    0
 207 #define IP_TYPE_IPV6    1
 208 #define MAX_IP          32
 209
 210 const char* ipv4[MAX_IP];
 211 uint8_t link_ipv6[MAX_IP][16];
 212 uint32_t        type, numports;
 213
 214 /* mask of enabled ports */
 215 static uint32_t enabled_port_mask = 0;
 216 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */
 217 static int numa_on = 1; /**< NUMA is enabled by default. */
 218 static int csum_on = 1; /**< NUMA is enabled by default. */
 219 struct pipeline_params def_pipeline_params = {
 220         .n_ports_in = 0,
 221         .n_ports_out = 0,
 222         .n_msgq = 0,
 223         .socket_id = 0,
 224         .n_args = 0,
 225         .log_level = 0,
 226 };
 227
 228 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 229 static int ipv6 = 0; /**< ipv6 is false by default. */
 230 #endif
 231
 232 void convert_ipstr_to_numeric(void);
 233
 234 int print_l4stats(void);
 235 int clear_stats(void);
 236
 237 struct mbuf_table {
 238         uint16_t len;
 239         struct rte_mbuf *m_table[MAX_PKT_BURST];
 240 };
 241
 242 struct lcore_rx_queue {
 243         uint8_t port_id;
 244         uint8_t queue_id;
 245 } __rte_cache_aligned;
 246
 247 #define MAX_RX_QUEUE_PER_LCORE 16
 248 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
 249 #define MAX_RX_QUEUE_PER_PORT 128
 250
 251 #define MAX_LCORE_PARAMS 1024
 252 struct lcore_params {
 253         uint8_t port_id;
 254         uint8_t queue_id;
 255         uint8_t lcore_id;
 256 } __rte_cache_aligned;
 257
 258 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 259 static struct lcore_params lcore_params_array_default[] = {
 260         {0, 0, 2},
 261         {0, 1, 2},
 262         {0, 2, 2},
 263         {1, 0, 2},
 264         {1, 1, 2},
 265         {1, 2, 2},
 266         {2, 0, 2},
 267         {3, 0, 3},
 268         {3, 1, 3},
 269 };
 270
 271 static struct lcore_params * lcore_params = lcore_params_array_default;
 272 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 273                                 sizeof(lcore_params_array_default[0]);
 274
 275 static struct rte_eth_conf port_conf = {
 276         .rxmode = {
 277                 .mq_mode = ETH_MQ_RX_RSS,
 278                 .max_rx_pkt_len = ETHER_MAX_LEN,
 279                 .split_hdr_size = 0,
 280                 .header_split   = 0, /**< Header Split disabled */
 281                 .hw_ip_checksum = 1, /**< IP checksum offload enabled */
 282                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 283                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 284                 .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 285         },
 286         .rx_adv_conf = {
 287                 .rss_conf = {
 288                         .rss_key = NULL,
 289                         .rss_hf = ETH_RSS_IP,
 290                 },
 291         },
 292         .txmode = {
 293                 .mq_mode = ETH_MQ_TX_NONE,
 294         },
 295 };
 296
 297 /* empty vmdq configuration structure. Filled in programatically */
 298 static struct rte_eth_rxconf rx_conf = {
 299                 .rx_thresh = {
 300                         .pthresh = 8,
 301                         .hthresh = 8,
 302                         .wthresh = 4,
 303                 },
 304                 .rx_free_thresh = 64,
 305                 .rx_drop_en = 0,
 306                 .rx_deferred_start = 0,
 307 };
 308
 309 static struct rte_eth_txconf tx_conf = {
 310                 .tx_thresh = {
 311                         .pthresh = 36,
 312                         .hthresh = 0,
 313                         .wthresh = 0,
 314                 },
 315                 .tx_rs_thresh = 0,
 316                 .tx_free_thresh = 0,
 317                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 318                         ETH_TXQ_FLAGS_NOOFFLOADS,
 319                 .tx_deferred_start = 0,
 320 };
 321
 322 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 323
 324 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 325 #include <rte_hash_crc.h>
 326 #define DEFAULT_HASH_FUNC       rte_hash_crc
 327 #else
 328 #include <rte_jhash.h>
 329 #define DEFAULT_HASH_FUNC       rte_jhash
 330 #endif
 331
 332 struct ipv4_5tuple {
 333         uint32_t ip_dst;
 334         uint32_t ip_src;
 335         uint16_t port_dst;
 336         uint16_t port_src;
 337         uint8_t  proto;
 338 } __attribute__((__packed__));
 339
 340 union ipv4_5tuple_host {
 341         struct {
 342                 uint8_t  pad0;
 343                 uint8_t  proto;
 344                 uint16_t pad1;
 345                 uint32_t ip_src;
 346                 uint32_t ip_dst;
 347                 uint16_t port_src;
 348                 uint16_t port_dst;
 349         };
 350         __m128i xmm;
 351 };
 352
 353 #define XMM_NUM_IN_IPV6_5TUPLE 3
 354
 355 struct ipv6_5tuple {
 356         uint8_t  ip_dst[IPV6_ADDR_LEN];
 357         uint8_t  ip_src[IPV6_ADDR_LEN];
 358         uint16_t port_dst;
 359         uint16_t port_src;
 360         uint8_t  proto;
 361 } __attribute__((__packed__));
 362
 363 union ipv6_5tuple_host {
 364         struct {
 365                 uint16_t pad0;
 366                 uint8_t  proto;
 367                 uint8_t  pad1;
 368                 uint8_t  ip_src[IPV6_ADDR_LEN];
 369                 uint8_t  ip_dst[IPV6_ADDR_LEN];
 370                 uint16_t port_src;
 371                 uint16_t port_dst;
 372                 uint64_t reserve;
 373         };
 374         __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE];
 375 };
 376
 377 struct ipv4_l3fwd_route {
 378         struct ipv4_5tuple key;
 379         uint8_t if_out;
 380 };
 381
 382 struct ipv6_l3fwd_route {
 383         struct ipv6_5tuple key;
 384         uint8_t if_out;
 385 };
 386
 387 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
 388         {{IPv4(101,0,0,0), IPv4(100,10,0,1),  101, 11, IPPROTO_TCP}, 0},
 389         {{IPv4(201,0,0,0), IPv4(200,20,0,1),  102, 12, IPPROTO_TCP}, 1},
 390         {{IPv4(111,0,0,0), IPv4(100,30,0,1),  101, 11, IPPROTO_TCP}, 2},
 391         {{IPv4(211,0,0,0), IPv4(200,40,0,1),  102, 12, IPPROTO_TCP}, 3},
 392 };
 393
 394 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
 395         {{
 396         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 397         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 398         101, 11, IPPROTO_TCP}, 0},
 399
 400         {{
 401         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 402         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 403         102, 12, IPPROTO_TCP}, 1},
 404
 405         {{
 406         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 407         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 408         101, 11, IPPROTO_TCP}, 2},
 409
 410         {{
 411         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 412         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 413         102, 12, IPPROTO_TCP}, 3},
 414 };
 415
 416 typedef struct rte_hash lookup_struct_t;
 417
 418 #ifdef RTE_ARCH_X86_64
 419 /* default to 4 million hash entries (approx) */
 420 #define L3FWD_HASH_ENTRIES              1024*1024*4
 421 #else
 422 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */
 423 #define L3FWD_HASH_ENTRIES              1024*1024*1
 424 #endif
 425 #define HASH_ENTRY_NUMBER_DEFAULT       4
 426
 427 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
 428
 429 void
 430 app_link_up_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
 431 {
 432         cp->state = 1;
 433 }
 434
 435 void
 436 app_link_down_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
 437 {
 438         cp->state = 0;
 439 }
 440
 441 /* int
 442  * inet_pton_ipv4(src, dst)
 443  *      like inet_aton() but without all the hexadecimal and shorthand.
 444  * return:
 445  *      1 if `src' is a valid dotted quad, else 0.
 446  * notice:
 447  *      does not touch `dst' unless it's returning 1.
 448  * author:
 449  *      Paul Vixie, 1996.
 450  */
 451 static int inet_pton_ipv4(const char *src, unsigned char *dst)
 452 {
 453         static const char digits[] = "0123456789";
 454         int saw_digit, octets, ch;
 455         unsigned char tmp[INADDRSZ], *tp;
 456
 457         saw_digit = 0;
 458         octets = 0;
 459         *(tp = tmp) = 0;
 460         while ((ch = *src++) != '\0') {
 461                 const char *pch;
 462
 463                 if ((pch = strchr(digits, ch)) != NULL) {
 464                         unsigned int new = *tp * 10 + (pch - digits);
 465
 466                         if (new > 255)
 467                                 return 0;
 468                         if (!saw_digit) {
 469                                 if (++octets > 4)
 470                                         return 0;
 471                                 saw_digit = 1;
 472                         }
 473                         *tp = (unsigned char)new;
 474                 } else if (ch == '.' && saw_digit) {
 475                         if (octets == 4)
 476                                 return 0;
 477                         *++tp = 0;
 478                         saw_digit = 0;
 479                 } else
 480                         return 0;
 481         }
 482         if (octets < 4)
 483                 return 0;
 484
 485         memcpy(dst, tmp, INADDRSZ);
 486         return 1;
 487 }
 488
 489 /* int
 490  * inet_pton_ipv6(src, dst)
 491  *      convert presentation level address to network order binary form.
 492  * return:
 493  *      1 if `src' is a valid [RFC1884 2.2] address, else 0.
 494  * notice:
 495  *      (1) does not touch `dst' unless it's returning 1.
 496  *      (2) :: in a full address is silently ignored.
 497  * credit:
 498  *      inspired by Mark Andrews.
 499  * author:
 500  *      Paul Vixie, 1996.
 501  */
 502 static int inet_pton_ipv6(const char *src, unsigned char *dst)
 503 {
 504         static const char xdigits_l[] = "0123456789abcdef",
 505             xdigits_u[] = "0123456789ABCDEF";
 506         unsigned char tmp[IN6ADDRSZ], *tp = 0, *endp = 0, *colonp = 0;
 507         const char *xdigits = 0, *curtok = 0;
 508         int ch = 0, saw_xdigit = 0, count_xdigit = 0;
 509         unsigned int val = 0;
 510         unsigned dbloct_count = 0;
 511
 512         memset((tp = tmp), '\0', IN6ADDRSZ);
 513         endp = tp + IN6ADDRSZ;
 514         colonp = NULL;
 515         /* Leading :: requires some special handling. */
 516         if (*src == ':')
 517                 if (*++src != ':')
 518                         return 0;
 519         curtok = src;
 520         saw_xdigit = count_xdigit = 0;
 521         val = 0;
 522
 523         while ((ch = *src++) != '\0') {
 524                 const char *pch;
 525
 526                 if ((pch = strchr((xdigits = xdigits_l), ch)) == NULL)
 527                         pch = strchr((xdigits = xdigits_u), ch);
 528                 if (pch != NULL) {
 529                         if (count_xdigit >= 4)
 530                                 return 0;
 531                         val <<= 4;
 532                         val |= (pch - xdigits);
 533                         if (val > 0xffff)
 534                                 return 0;
 535                         saw_xdigit = 1;
 536                         count_xdigit++;
 537                         continue;
 538                 }
 539                 if (ch == ':') {
 540                         curtok = src;
 541                         if (!saw_xdigit) {
 542                                 if (colonp)
 543                                         return 0;
 544                                 colonp = tp;
 545                                 continue;
 546                         } else if (*src == '\0') {
 547                                 return 0;
 548                         }
 549                         if (tp + sizeof(int16_t) > endp)
 550                                 return 0;
 551                         *tp++ = (unsigned char)((val >> 8) & 0xff);
 552                         *tp++ = (unsigned char)(val & 0xff);
 553                         saw_xdigit = 0;
 554                         count_xdigit = 0;
 555                         val = 0;
 556                         dbloct_count++;
 557                         continue;
 558                 }
 559                 if (ch == '.' && ((tp + INADDRSZ) <= endp) &&
 560                     inet_pton_ipv4(curtok, tp) > 0) {
 561                         tp += INADDRSZ;
 562                         saw_xdigit = 0;
 563                         dbloct_count += 2;
 564                         break;  /* '\0' was seen by inet_pton4(). */
 565                 }
 566                 return 0;
 567         }
 568         if (saw_xdigit) {
 569                 if (tp + sizeof(int16_t) > endp)
 570                         return 0;
 571                 *tp++ = (unsigned char)((val >> 8) & 0xff);
 572                 *tp++ = (unsigned char)(val & 0xff);
 573                 dbloct_count++;
 574         }
 575         if (colonp != NULL) {
 576                 /* if we already have 8 double octets, having a colon means error */
 577                 if (dbloct_count == 8)
 578                         return 0;
 579
 580                 /*
 581                  * Since some memmove()'s erroneously fail to handle
 582                  * overlapping regions, we'll do the shift by hand.
 583                  */
 584                 const int n = tp - colonp;
 585                 int i;
 586
 587                 for (i = 1; i <= n; i++) {
 588                         endp[-i] = colonp[n - i];
 589                         colonp[n - i] = 0;
 590                 }
 591                 tp = endp;
 592         }
 593         if (tp != endp)
 594                 return 0;
 595         memcpy(dst, tmp, IN6ADDRSZ);
 596         return 1;
 597 }
 598
 599 static int my_inet_pton_ipv6(int af, const char *src, void *dst)
 600 {
 601         switch (af) {
 602         case AF_INET:
 603                 return inet_pton_ipv4(src, dst);
 604         case AF_INET6:
 605                 return inet_pton_ipv6(src, dst);
 606         default:
 607                 errno = EAFNOSUPPORT;
 608                 return -1;
 609         }
 610 }
 611
 612 void convert_ipstr_to_numeric(void)
 613 {
 614         uint32_t i;
 615
 616         for (i = 0; i < numports; i++)
 617         {
 618                 if (type == IP_TYPE_IPV4) {
 619                         memset(&ipaddr1, '\0', sizeof(struct sockaddr_in));
 620                         ipaddr1.sin_addr.s_addr = inet_addr(ipv4[i]);
 621                         ifm_add_ipv4_port(i, ipaddr1.sin_addr.s_addr, 24);
 622                 } else if (type == IP_TYPE_IPV6) {
 623                         ifm_add_ipv6_port(i, &link_ipv6[i][0], 128);
 624                 }
 625         }
 626 }
 627
 628 static inline uint32_t
 629 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
 630         uint32_t init_val)
 631 {
 632         const union ipv4_5tuple_host *k;
 633         uint32_t t;
 634         const uint32_t *p;
 635
 636         k = data;
 637         t = k->proto;
 638         p = (const uint32_t *)&k->port_src;
 639
 640 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 641         init_val = rte_hash_crc_4byte(t, init_val);
 642         init_val = rte_hash_crc_4byte(k->ip_src, init_val);
 643         init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
 644         init_val = rte_hash_crc_4byte(*p, init_val);
 645 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 646         init_val = rte_jhash_1word(t, init_val);
 647         init_val = rte_jhash_1word(k->ip_src, init_val);
 648         init_val = rte_jhash_1word(k->ip_dst, init_val);
 649         init_val = rte_jhash_1word(*p, init_val);
 650 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 651         return (init_val);
 652 }
 653 static uint64_t arp_pkts[32] = {0};
 654
 655 static inline int check_arpicmp(struct rte_mbuf *pkt)
 656 {
 657
 658         uint8_t in_port_id = pkt->port;
 659         uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
 660
 661         uint16_t *eth_proto =
 662                         RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
 663
 664         uint8_t *protocol;
 665         uint32_t prot_offset =
 666                         MBUF_HDR_ROOM + ETH_HDR_SIZE + IP_HDR_PROTOCOL_OFST;
 667
 668         protocol = RTE_MBUF_METADATA_UINT8_PTR(pkt, prot_offset);
 669
 670         if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_ARP) ||
 671                         ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV4)
 672                         && (*protocol == IP_PROTOCOL_ICMP))) {
 673                         process_arpicmp_pkt(pkt, ifm_get_port(in_port_id));
 674                         arp_pkts[in_port_id]++;
 675                         return 0;
 676         }
 677
 678         return 1;
 679
 680 }
 681
 682 static inline int check_arpicmpv6(struct rte_mbuf *pkt)
 683 {
 684
 685         struct ether_hdr *eth_h;
 686         struct ipv6_hdr *ipv6_h;
 687
 688         uint8_t in_port_id = pkt->port;
 689         uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
 690
 691         uint16_t *eth_proto =
 692                         RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
 693
 694         eth_h = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 695         ipv6_h = (struct ipv6_hdr *)((char *)eth_h + sizeof(struct ether_hdr));
 696
 697         if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV6)
 698                                         && (ipv6_h->proto == ICMPV6_PROTOCOL_ID)) {
 699                         process_icmpv6_pkt(pkt, ifm_get_port(in_port_id));
 700                         return 0;
 701         }
 702
 703         return 1;
 704
 705 }
 706
 707 static inline uint32_t
 708 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val)
 709 {
 710         const union ipv6_5tuple_host *k;
 711         uint32_t t;
 712         const uint32_t *p;
 713 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 714         const uint32_t  *ip_src0, *ip_src1, *ip_src2, *ip_src3;
 715         const uint32_t  *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
 716 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 717
 718         k = data;
 719         t = k->proto;
 720         p = (const uint32_t *)&k->port_src;
 721
 722 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 723         ip_src0 = (const uint32_t *) k->ip_src;
 724         ip_src1 = (const uint32_t *)(k->ip_src+4);
 725         ip_src2 = (const uint32_t *)(k->ip_src+8);
 726         ip_src3 = (const uint32_t *)(k->ip_src+12);
 727         ip_dst0 = (const uint32_t *) k->ip_dst;
 728         ip_dst1 = (const uint32_t *)(k->ip_dst+4);
 729         ip_dst2 = (const uint32_t *)(k->ip_dst+8);
 730         ip_dst3 = (const uint32_t *)(k->ip_dst+12);
 731         init_val = rte_hash_crc_4byte(t, init_val);
 732         init_val = rte_hash_crc_4byte(*ip_src0, init_val);
 733         init_val = rte_hash_crc_4byte(*ip_src1, init_val);
 734         init_val = rte_hash_crc_4byte(*ip_src2, init_val);
 735         init_val = rte_hash_crc_4byte(*ip_src3, init_val);
 736         init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
 737         init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
 738         init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
 739         init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
 740         init_val = rte_hash_crc_4byte(*p, init_val);
 741 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 742         init_val = rte_jhash_1word(t, init_val);
 743         init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 744         init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 745         init_val = rte_jhash_1word(*p, init_val);
 746 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 747         return (init_val);
 748 }
 749
 750 #define IPV4_L3FWD_NUM_ROUTES \
 751         (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0]))
 752
 753 #define IPV6_L3FWD_NUM_ROUTES \
 754         (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0]))
 755
 756 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
 757 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
 758
 759 #endif
 760
 761 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 762 struct ipv4_l3fwd_route {
 763         uint32_t ip;
 764         uint8_t  depth;
 765         uint8_t  if_out;
 766 };
 767
 768 struct ipv6_l3fwd_route {
 769         uint8_t ip[16];
 770         uint8_t  depth;
 771         uint8_t  if_out;
 772 };
 773
 774 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
 775         {IPv4(1,1,1,0), 24, 0},
 776         {IPv4(2,1,1,0), 24, 1},
 777         {IPv4(3,1,1,0), 24, 2},
 778         {IPv4(4,1,1,0), 24, 3},
 779         {IPv4(5,1,1,0), 24, 4},
 780         {IPv4(6,1,1,0), 24, 5},
 781         {IPv4(7,1,1,0), 24, 6},
 782         {IPv4(8,1,1,0), 24, 7},
 783 };
 784
 785 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
 786         {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0},
 787         {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1},
 788         {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2},
 789         {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3},
 790         {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4},
 791         {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5},
 792         {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6},
 793         {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7},
 794 };
 795
 796 #define IPV4_L3FWD_NUM_ROUTES \
 797         (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0]))
 798 #define IPV6_L3FWD_NUM_ROUTES \
 799         (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0]))
 800
 801 #define IPV4_L3FWD_LPM_MAX_RULES         1024
 802 #define IPV6_L3FWD_LPM_MAX_RULES         1024
 803 #define IPV6_L3FWD_LPM_NUMBER_TBL8S (1 << 16)
 804
 805 typedef struct rte_lpm lookup_struct_t;
 806 typedef struct rte_lpm6 lookup6_struct_t;
 807 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS];
 808 static lookup6_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS];
 809 #endif
 810
 811 struct lcore_conf {
 812         uint16_t n_rx_queue;
 813         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 814         uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
 815         struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
 816         lookup_struct_t * ipv4_lookup_struct;
 817 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 818         lookup6_struct_t * ipv6_lookup_struct;
 819 #else
 820         lookup_struct_t * ipv6_lookup_struct;
 821 #endif
 822 } __rte_cache_aligned;
 823
 824 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
 825
 826 /* Send burst of packets on an output interface */
 827 static inline int
 828 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port)
 829 {
 830         struct rte_mbuf **m_table;
 831         int ret;
 832         uint16_t queueid;
 833
 834         queueid = qconf->tx_queue_id[port];
 835         m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 836
 837         ret = rte_eth_tx_burst(port, queueid, m_table, n);
 838         if (unlikely(ret < n)) {
 839                 do {
 840                         rte_pktmbuf_free(m_table[ret]);
 841                 } while (++ret < n);
 842         }
 843         /*Tx Pkt count*/
 844         tx_pkt_count[port] += ret;
 845         return 0;
 846 }
 847
 848 /* Enqueue a single packet, and send burst if queue is filled */
 849 static inline int
 850 send_single_packet(struct rte_mbuf *m, uint8_t port)
 851 {
 852         uint32_t lcore_id;
 853         uint16_t len;
 854         struct lcore_conf *qconf;
 855
 856         lcore_id = rte_lcore_id();
 857
 858         qconf = &lcore_conf[lcore_id];
 859         len = qconf->tx_mbufs[port].len;
 860         qconf->tx_mbufs[port].m_table[len] = m;
 861         len++;
 862
 863         /* enough pkts to be sent */
 864         if (unlikely(len == MAX_PKT_BURST)) {
 865                 send_burst(qconf, MAX_PKT_BURST, port);
 866                 len = 0;
 867         }
 868
 869         qconf->tx_mbufs[port].len = len;
 870         return 0;
 871 }
 872
 873 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 874 static inline __attribute__((always_inline)) void
 875 send_packetsx4(struct lcore_conf *qconf, uint8_t port,
 876         struct rte_mbuf *m[], uint32_t num)
 877 {
 878         uint32_t len, j, n;
 879
 880         len = qconf->tx_mbufs[port].len;
 881
 882         /*
 883          * If TX buffer for that queue is empty, and we have enough packets,
 884          * then send them straightway.
 885          */
 886         if (num >= MAX_TX_BURST && len == 0) {
 887                 n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num);
 888                 if (unlikely(n < num)) {
 889                         do {
 890                                 rte_pktmbuf_free(m[n]);
 891                         } while (++n < num);
 892                 }
 893                 return;
 894         }
 895
 896         /*
 897          * Put packets into TX buffer for that queue.
 898          */
 899
 900         n = len + num;
 901         n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num;
 902
 903         j = 0;
 904         switch (n % FWDSTEP) {
 905         while (j < n) {
 906         case 0:
 907                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 908                 j++;
 909         case 3:
 910                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 911                 j++;
 912         case 2:
 913                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 914                 j++;
 915         case 1:
 916                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 917                 j++;
 918         }
 919         }
 920
 921         len += n;
 922
 923         /* enough pkts to be sent */
 924         if (unlikely(len == MAX_PKT_BURST)) {
 925
 926                 send_burst(qconf, MAX_PKT_BURST, port);
 927
 928                 /* copy rest of the packets into the TX buffer. */
 929                 len = num - n;
 930                 j = 0;
 931                 switch (len % FWDSTEP) {
 932                 while (j < len) {
 933                 case 0:
 934                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 935                         j++;
 936                 case 3:
 937                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 938                         j++;
 939                 case 2:
 940                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 941                         j++;
 942                 case 1:
 943                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 944                         j++;
 945                 }
 946                 }
 947         }
 948
 949         qconf->tx_mbufs[port].len = len;
 950 }
 951 #endif /* APP_LOOKUP_LPM */
 952
 953 #ifdef DO_RFC_1812_CHECKS
 954 static inline int
 955 is_valid_pkt_ipv4(struct ipv4_hdr *pkt, uint32_t link_len)
 956 {
 957         /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 958         /*
 959          * 1. The packet length reported by the Link Layer must be large
 960          * enough to hold the minimum length legal IP datagram (20 bytes).
 961          */
 962         if (link_len < sizeof(struct ipv4_hdr))
 963                 return -1;
 964
 965         /* 2. The IP checksum must be correct. */
 966         /* this is checked in H/W */
 967
 968         /*
 969          * 3. The IP version number must be 4. If the version number is not 4
 970          * then the packet may be another version of IP, such as IPng or
 971          * ST-II.
 972          */
 973         if (((pkt->version_ihl) >> 4) != 4)
 974                 return -3;
 975         /*
 976          * 4. The IP header length field must be large enough to hold the
 977          * minimum length legal IP datagram (20 bytes = 5 words).
 978          */
 979         if ((pkt->version_ihl & 0xf) < 5)
 980                 return -4;
 981
 982         /*
 983          * 5. The IP total length field must be large enough to hold the IP
 984          * datagram header, whose length is specified in the IP header length
 985          * field.
 986          */
 987         if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr))
 988                 return -5;
 989
 990         return 0;
 991 }
 992 #endif
 993
 994 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 995
 996 static __m128i mask0;
 997 static __m128i mask1;
 998 static __m128i mask2;
 999 static inline uint8_t
1000 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct)
1001 {
1002         int ret = 0;
1003         union ipv4_5tuple_host key;
1004
1005         ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live);
1006         __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr));
1007         /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */
1008         key.xmm = _mm_and_si128(data, mask0);
1009         /* Find destination port */
1010         ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key);
1011         return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]);
1012 }
1013
1014 static inline uint8_t
1015 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup_struct_t * ipv6_l3fwd_lookup_struct)
1016 {
1017         int ret = 0;
1018         union ipv6_5tuple_host key;
1019
1020         ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len);
1021         __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr));
1022         __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)));
1023         __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i)));
1024         /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
1025         key.xmm[0] = _mm_and_si128(data0, mask1);
1026         /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */
1027         key.xmm[1] = data1;
1028         /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */
1029         key.xmm[2] = _mm_and_si128(data2, mask2);
1030
1031         /* Find destination port */
1032         ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key);
1033         return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]);
1034 }
1035 #endif
1036
1037 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1038
1039 static inline uint8_t
1040 get_ipv4_dst_port(void *ipv4_hdr,  uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct)
1041 {
1042         uint8_t next_hop;
1043
1044         return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct,
1045                 rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr),
1046                 &next_hop) == 0) ? next_hop : portid);
1047 }
1048
1049 static inline uint8_t
1050 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup6_struct_t * ipv6_l3fwd_lookup_struct)
1051 {
1052         uint8_t next_hop;
1053         return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct,
1054                         ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)?
1055                         next_hop : portid);
1056 }
1057 #endif
1058
1059 static inline void l3fwd_simple_replay(struct rte_mbuf *m, uint8_t portid,
1060         struct lcore_conf *qconf)  __attribute__((unused));
1061
1062 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \
1063         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1064
1065 #define MASK_ALL_PKTS    0xff
1066 #define EXCLUDE_1ST_PKT 0xfe
1067 #define EXCLUDE_2ND_PKT 0xfd
1068 #define EXCLUDE_3RD_PKT 0xfb
1069 #define EXCLUDE_4TH_PKT 0xf7
1070 #define EXCLUDE_5TH_PKT 0xef
1071 #define EXCLUDE_6TH_PKT 0xdf
1072 #define EXCLUDE_7TH_PKT 0xbf
1073 #define EXCLUDE_8TH_PKT 0x7f
1074
1075 static inline void
1076 simple_ipv4_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
1077 {
1078         struct ether_hdr *eth_hdr[8];
1079         struct ether_hdr tmp;
1080         struct ipv4_hdr *ipv4_hdr[8];
1081         struct udp_hdr *udp_hdr[8];
1082         int i, a[8];
1083         l2_phy_interface_t *port = ifm_get_port(portid);
1084
1085         if (port == NULL) {
1086                 printf("port may be un initialized\n");
1087                 return;
1088         }
1089
1090         for ( i = 0; i < 8; i++)
1091         {
1092                 if (m[i])
1093                         a[i] = check_arpicmp(m[i]);
1094                 else {
1095                         printf("null packet received\n");
1096                         return;
1097                 }
1098         }
1099
1100         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
1101         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
1102         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
1103         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
1104         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
1105         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
1106         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
1107         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
1108
1109
1110         memset(&tmp,0,sizeof (struct ether_hdr));
1111
1112         for(i=0;i<8;i++)
1113         {
1114         ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
1115         ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
1116         ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
1117         }
1118
1119         /* Handle IPv4 headers.*/
1120         ipv4_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv4_hdr *,
1121                                               sizeof(struct ether_hdr));
1122         ipv4_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv4_hdr *,
1123                                               sizeof(struct ether_hdr));
1124         ipv4_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv4_hdr *,
1125                                               sizeof(struct ether_hdr));
1126         ipv4_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv4_hdr *,
1127                                               sizeof(struct ether_hdr));
1128         ipv4_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv4_hdr *,
1129                                               sizeof(struct ether_hdr));
1130         ipv4_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv4_hdr *,
1131                                               sizeof(struct ether_hdr));
1132         ipv4_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv4_hdr *,
1133                                               sizeof(struct ether_hdr));
1134         ipv4_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv4_hdr *,
1135                                               sizeof(struct ether_hdr));
1136         struct ipv4_hdr temp_ipv4;
1137         for(i=0;i<8;i++)
1138         {
1139         temp_ipv4.dst_addr = ipv4_hdr[i]->dst_addr;
1140         ipv4_hdr[i]->dst_addr = ipv4_hdr[i]->src_addr;
1141         ipv4_hdr[i]->src_addr = temp_ipv4.dst_addr;
1142         }
1143
1144         /* Handle UDP headers.*/
1145         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1146                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1147
1148         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1149                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1150         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1151                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1152         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1153                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1154         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1155                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1156         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1157                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1158         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1159                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1160         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1161                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1162        /*1) memcpy or assignment.*/
1163
1164         struct udp_hdr temp_udp;
1165         for(i=0;i<8;i++)
1166         {
1167         temp_udp.dst_port = udp_hdr[i]->dst_port;
1168         udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1169         udp_hdr[i]->src_port = temp_udp.dst_port;
1170         }
1171 #ifdef DO_RFC_1812_CHECKS
1172         /* Check to make sure the packet is valid (RFC1812) */
1173         uint8_t valid_mask = MASK_ALL_PKTS;
1174         if (is_valid_pkt_ipv4(ipv4_hdr[0], m[0]->pkt_len) < 0) {
1175                 rte_pktmbuf_free(m[0]);
1176                 valid_mask &= EXCLUDE_1ST_PKT;
1177         }
1178         if (is_valid_pkt_ipv4(ipv4_hdr[1], m[1]->pkt_len) < 0) {
1179                 rte_pktmbuf_free(m[1]);
1180                 valid_mask &= EXCLUDE_2ND_PKT;
1181         }
1182         if (is_valid_pkt_ipv4(ipv4_hdr[2], m[2]->pkt_len) < 0) {
1183                 rte_pktmbuf_free(m[2]);
1184                 valid_mask &= EXCLUDE_3RD_PKT;
1185         }
1186         if (is_valid_pkt_ipv4(ipv4_hdr[3], m[3]->pkt_len) < 0) {
1187                 rte_pktmbuf_free(m[3]);
1188                 valid_mask &= EXCLUDE_4TH_PKT;
1189         }
1190         if (is_valid_pkt_ipv4(ipv4_hdr[4], m[4]->pkt_len) < 0) {
1191                 rte_pktmbuf_free(m[4]);
1192                 valid_mask &= EXCLUDE_5TH_PKT;
1193         }
1194         if (is_valid_pkt_ipv4(ipv4_hdr[5], m[5]->pkt_len) < 0) {
1195                 rte_pktmbuf_free(m[5]);
1196                 valid_mask &= EXCLUDE_6TH_PKT;
1197         }
1198         if (is_valid_pkt_ipv4(ipv4_hdr[6], m[6]->pkt_len) < 0) {
1199                 rte_pktmbuf_free(m[6]);
1200                 valid_mask &= EXCLUDE_7TH_PKT;
1201         }
1202         if (is_valid_pkt_ipv4(ipv4_hdr[7], m[7]->pkt_len) < 0) {
1203                 rte_pktmbuf_free(m[7]);
1204                 valid_mask &= EXCLUDE_8TH_PKT;
1205         }
1206         if (unlikely(valid_mask != MASK_ALL_PKTS)) {
1207                 if (valid_mask == 0){
1208                         return;
1209                 } else {
1210                         uint8_t i = 0;
1211                         for (i = 0; i < 8; i++) {
1212                                 if ((0x1 << i) & valid_mask) {
1213                                         l3fwd_simple_replay(m[i], portid, qconf);
1214                                 }
1215                         }
1216                         return;
1217                 }
1218         }
1219 #endif // End of #ifdef DO_RFC_1812_CHECKS
1220
1221 #ifdef DO_RFC_1812_CHECKS
1222         /* Update time to live and header checksum */
1223         --(ipv4_hdr[0]->time_to_live);
1224         --(ipv4_hdr[1]->time_to_live);
1225         --(ipv4_hdr[2]->time_to_live);
1226         --(ipv4_hdr[3]->time_to_live);
1227         ++(ipv4_hdr[0]->hdr_checksum);
1228         ++(ipv4_hdr[1]->hdr_checksum);
1229         ++(ipv4_hdr[2]->hdr_checksum);
1230         ++(ipv4_hdr[3]->hdr_checksum);
1231         --(ipv4_hdr[4]->time_to_live);
1232         --(ipv4_hdr[5]->time_to_live);
1233         --(ipv4_hdr[6]->time_to_live);
1234         --(ipv4_hdr[7]->time_to_live);
1235         ++(ipv4_hdr[4]->hdr_checksum);
1236         ++(ipv4_hdr[5]->hdr_checksum);
1237         ++(ipv4_hdr[6]->hdr_checksum);
1238         ++(ipv4_hdr[7]->hdr_checksum);
1239 #endif
1240
1241         for ( i = 0; i < 8; i++)
1242         {
1243                 /* if not already processed as a arp/icmp pkt */
1244                 if (a[i]) {
1245                         port->transmit_single_pkt(port, m[i]);
1246                         tx_pkt_count[(uint64_t)port]++;
1247                 }
1248         }
1249
1250 }
1251
1252 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1,
1253                                  union ipv6_5tuple_host * key)
1254 {
1255         __m128i tmpdata0 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
1256         __m128i tmpdata1 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i)));
1257         __m128i tmpdata2 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i) + sizeof(__m128i)));
1258         key->xmm[0] = _mm_and_si128(tmpdata0, mask0);
1259         key->xmm[1] = tmpdata1;
1260         key->xmm[2] = _mm_and_si128(tmpdata2, mask1);
1261         return;
1262 }
1263
1264 static inline void
1265 simple_ipv6_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
1266 {
1267         struct ether_hdr *eth_hdr[8],tmp;
1268         int i, a[8];
1269
1270         __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[8], temp_ipv6;
1271         int32_t ret[8];
1272         union ipv6_5tuple_host key[8];
1273         struct udp_hdr *udp_hdr[8];
1274         l2_phy_interface_t *port = ifm_get_port(portid);
1275
1276         if (port == NULL) {
1277                 printf("port may be un initialized\n");
1278                 return;
1279         }
1280
1281         for ( i = 0; i < 8; i++)
1282         {
1283                 a[i] = check_arpicmpv6(m[i]);
1284         }
1285
1286         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
1287         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
1288         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
1289         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
1290         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
1291         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
1292         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
1293         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
1294
1295         memset(&tmp,0,sizeof (struct ether_hdr));
1296
1297         for(i=0;i<8;i++)
1298         {
1299
1300         ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
1301         ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
1302         ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
1303         }
1304         /* Handle IPv6 headers.*/
1305         ipv6_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv6_hdr *,
1306                                               sizeof(struct ether_hdr));
1307         ipv6_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv6_hdr *,
1308                                               sizeof(struct ether_hdr));
1309         ipv6_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv6_hdr *,
1310                                               sizeof(struct ether_hdr));
1311         ipv6_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv6_hdr *,
1312                                               sizeof(struct ether_hdr));
1313         ipv6_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv6_hdr *,
1314                                               sizeof(struct ether_hdr));
1315         ipv6_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv6_hdr *,
1316                                               sizeof(struct ether_hdr));
1317         ipv6_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv6_hdr *,
1318                                               sizeof(struct ether_hdr));
1319         ipv6_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv6_hdr *,
1320                                               sizeof(struct ether_hdr));
1321         for(i=0;i<8;i++)
1322         {
1323                 memcpy(temp_ipv6.dst_addr,ipv6_hdr[i]->dst_addr,16);
1324                 memcpy(ipv6_hdr[i]->dst_addr,ipv6_hdr[i]->src_addr,16);
1325                 memcpy(ipv6_hdr[i]->src_addr,temp_ipv6.dst_addr,16);
1326         }
1327
1328         /* Handle UDP headers.*/
1329         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1330                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1331
1332         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1333                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1334         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1335                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1336         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1337                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1338         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1339                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1340         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1341                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1342         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1343                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1344         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1345                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1346        /*1) memcpy or assignment.*/
1347
1348         struct udp_hdr temp_udp;
1349         for(i=0;i<8;i++)
1350         {
1351         temp_udp.dst_port = udp_hdr[i]->dst_port;
1352         udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1353         udp_hdr[i]->src_port = temp_udp.dst_port;
1354         }
1355         const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3],
1356                                 &key[4], &key[5], &key[6], &key[7]};
1357         rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1358
1359         for ( i = 0; i < 8; i++)
1360         {
1361                 /* if not already processed as a arp/icmp pkt */
1362                 if (a[i]){
1363                         port->transmit_single_pkt(port, m[i]);
1364                         tx_pkt_count[(uint64_t)portid]++;
1365                 }
1366         }
1367
1368 }
1369 #endif /* APP_LOOKUP_METHOD */
1370
1371 static inline __attribute__((always_inline)) void
1372 l3fwd_simple_replay(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf)
1373 {
1374         struct ether_hdr *eth_hdr,tmp;
1375         struct ipv4_hdr *ipv4_hdr,temp_ipv4;
1376         struct udp_hdr *udp_hdr,temp_udp;
1377         l2_phy_interface_t *port = ifm_get_port(portid);
1378
1379         if (port == NULL) {
1380                 printf("port may be un initialized\n");
1381                 return;
1382         }
1383
1384         if (m == NULL) {
1385                 printf("Null packet received\n");
1386                 return;
1387         }
1388
1389         /* arp packet already processed return back */
1390         if (!check_arpicmp(m))
1391                 return;
1392
1393         if (qconf == NULL)
1394                 printf("qconf configuration is NULL\n");
1395
1396         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1397         ether_addr_copy(&eth_hdr->s_addr, &tmp.s_addr);
1398         ether_addr_copy(&eth_hdr->d_addr, &eth_hdr->s_addr);
1399         ether_addr_copy(&tmp.s_addr, &eth_hdr->d_addr);
1400         struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1401
1402         if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1403                 /* Handle IPv4 headers.*/
1404                 ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
1405                                                    sizeof(struct ether_hdr));
1406                 temp_ipv4.dst_addr = ipv4_hdr->dst_addr;
1407                 ipv4_hdr->dst_addr = ipv4_hdr->src_addr;
1408                 ipv4_hdr->src_addr = temp_ipv4.dst_addr;
1409
1410 #ifdef DO_RFC_1812_CHECKS
1411                 /* Check to make sure the packet is valid (RFC1812) */
1412                 if (is_valid_pkt_ipv4(ipv4_hdr, m->pkt_len) < 0) {
1413                         printf("not of type 1812\n");
1414                         rte_pktmbuf_free(m);
1415                         return;
1416                 }
1417 #endif
1418
1419 #ifdef DO_RFC_1812_CHECKS
1420                 /* Update time to live and header checksum */
1421                 --(ipv4_hdr->time_to_live);
1422                 ++(ipv4_hdr->hdr_checksum);
1423 #endif
1424                 /* Handle UDP headers.*/
1425                 udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1426                          (sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr)));
1427                 /*Swapping Src and Dst Port*/
1428                 temp_udp.dst_port = udp_hdr->dst_port;
1429                 udp_hdr->dst_port = udp_hdr->src_port;
1430                 udp_hdr->src_port = temp_udp.dst_port;
1431
1432                 if (m) {
1433                         port->transmit_single_pkt(port, m);
1434                         tx_pkt_count[portid]++;
1435                 }
1436         } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1437                 /* Handle IPv6 headers.*/
1438                 struct ipv6_hdr *ipv6_hdr,temp_ipv6;
1439                 /* Handle IPv4 headers.*/
1440
1441                 ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
1442                                                    sizeof(struct ether_hdr));
1443
1444         /*Swapping of Src and Dst IP address*/
1445         memcpy(temp_ipv6.dst_addr,ipv6_hdr->dst_addr,16);
1446         memcpy(ipv6_hdr->dst_addr,ipv6_hdr->src_addr,16);
1447         memcpy(ipv6_hdr->src_addr,temp_ipv6.dst_addr,16);
1448
1449         /* Handle UDP headers.*/
1450         udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1451                          (sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr)));
1452         /*Swapping Src and Dst Port*/
1453         temp_udp.dst_port = udp_hdr->dst_port;
1454         udp_hdr->dst_port = udp_hdr->src_port;
1455         udp_hdr->src_port = temp_udp.dst_port;
1456                 send_single_packet(m, portid);
1457         } else
1458                 /* Free the mbuf that contains non-IPV4/IPV6 packet */
1459                 rte_pktmbuf_free(m);
1460 }
1461
1462 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1463         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1464 #ifdef DO_RFC_1812_CHECKS
1465
1466 #define IPV4_MIN_VER_IHL        0x45
1467 #define IPV4_MAX_VER_IHL        0x4f
1468 #define IPV4_MAX_VER_IHL_DIFF   (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL)
1469
1470 /* Minimum value of IPV4 total length (20B) in network byte order. */
1471 #define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8)
1472
1473 /*
1474  * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2:
1475  * - The IP version number must be 4.
1476  * - The IP header length field must be large enough to hold the
1477  *    minimum length legal IP datagram (20 bytes = 5 words).
1478  * - The IP total length field must be large enough to hold the IP
1479  *   datagram header, whose length is specified in the IP header length
1480  *   field.
1481  * If we encounter invalid IPV4 packet, then set destination port for it
1482  * to BAD_PORT value.
1483  */
1484 static inline __attribute__((always_inline)) void
1485 rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype)
1486 {
1487         uint8_t ihl;
1488
1489         if (RTE_ETH_IS_IPV4_HDR(ptype)) {
1490                 ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL;
1491
1492                 ipv4_hdr->time_to_live--;
1493                 ipv4_hdr->hdr_checksum++;
1494
1495                 if (ihl > IPV4_MAX_VER_IHL_DIFF ||
1496                                 ((uint8_t)ipv4_hdr->total_length == 0 &&
1497                                 ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) {
1498                         dp[0] = BAD_PORT;
1499                 }
1500         }
1501 }
1502
1503 #else
1504 #define rfc1812_process(mb, dp) do { } while (0)
1505 #endif /* DO_RFC_1812_CHECKS */
1506 #endif /* APP_LOOKUP_LPM && ENABLE_MULTI_BUFFER_OPTIMIZE */
1507
1508
1509 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1510         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1511
1512 static inline __attribute__((always_inline)) uint16_t
1513 get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt,
1514         uint32_t dst_ipv4, uint8_t portid)
1515 {
1516         uint8_t next_hop;
1517         struct ipv6_hdr *ipv6_hdr;
1518         struct ether_hdr *eth_hdr;
1519
1520         if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) {
1521                 if (rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4,
1522                                 &next_hop) != 0)
1523                         next_hop = portid;
1524         } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) {
1525                 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1526                 ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1);
1527                 if (rte_lpm6_lookup(qconf->ipv6_lookup_struct,
1528                                 ipv6_hdr->dst_addr, &next_hop) != 0)
1529                         next_hop = portid;
1530         } else {
1531                 next_hop = portid;
1532         }
1533
1534         return next_hop;
1535 }
1536
1537 static inline void
1538 process_packet(struct lcore_conf *qconf, struct rte_mbuf *pkt,
1539         uint16_t *dst_port, uint8_t portid)
1540 {
1541         struct ether_hdr *eth_hdr;
1542         struct ipv4_hdr *ipv4_hdr;
1543         uint32_t dst_ipv4;
1544         uint16_t dp;
1545         __m128i te, ve;
1546
1547         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1548         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1549
1550         /*Add swap*/
1551         dst_ipv4 = ipv4_hdr->dst_addr;
1552         dst_ipv4 = rte_be_to_cpu_32(dst_ipv4);
1553
1554         /*Changing the dp to incoming port*/
1555         dp = get_dst_port(qconf, pkt, dst_ipv4, portid);
1556         dp = portid;
1557
1558         te = _mm_loadu_si128((__m128i *)eth_hdr);
1559         ve = val_eth[dp];
1560
1561         dst_port[0] = dp;
1562         rfc1812_process(ipv4_hdr, dst_port, pkt->packet_type);
1563
1564         te =  _mm_blend_epi16(te, ve, MASK_ETH);
1565         _mm_storeu_si128((__m128i *)eth_hdr, te);
1566 }
1567 /* Wont be using the following fucntion*/
1568
1569 /*
1570  * Read packet_type and destination IPV4 addresses from 4 mbufs.
1571  */
1572 static inline void
1573 processx4_step1(struct rte_mbuf *pkt[FWDSTEP],
1574                 __m128i *dip,
1575                 uint32_t *ipv4_flag)
1576 {
1577         struct ipv4_hdr *ipv4_hdr;
1578         struct ether_hdr *eth_hdr;
1579         uint32_t x0, x1, x2, x3;
1580
1581         eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *);
1582         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1583         x0 = ipv4_hdr->dst_addr;
1584         ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4;
1585
1586         eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *);
1587         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1588         x1 = ipv4_hdr->dst_addr;
1589         ipv4_flag[0] &= pkt[1]->packet_type;
1590
1591         eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *);
1592         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1593         x2 = ipv4_hdr->dst_addr;
1594         ipv4_flag[0] &= pkt[2]->packet_type;
1595
1596         eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *);
1597         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1598         x3 = ipv4_hdr->dst_addr;
1599         ipv4_flag[0] &= pkt[3]->packet_type;
1600
1601         dip[0] = _mm_set_epi32(x3, x2, x1, x0);
1602 }
1603
1604 /*
1605  * Lookup into LPM for destination port.
1606  * If lookup fails, use incoming port (portid) as destination port.
1607  */
1608 static inline void
1609 processx4_step2(const struct lcore_conf *qconf,
1610                 __m128i dip,
1611                 uint32_t ipv4_flag,
1612                 uint8_t portid,
1613                 struct rte_mbuf *pkt[FWDSTEP],
1614                 uint16_t dprt[FWDSTEP])
1615 {
1616         rte_xmm_t dst;
1617         const  __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11,
1618                                                 4, 5, 6, 7, 0, 1, 2, 3);
1619
1620         /* Byte swap 4 IPV4 addresses. */
1621         dip = _mm_shuffle_epi8(dip, bswap_mask);
1622
1623         /* if all 4 packets are IPV4. */
1624         if (likely(ipv4_flag)) {
1625                 rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid);
1626         } else {
1627                 dst.x = dip;
1628                 dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid);
1629                 dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid);
1630                 dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid);
1631                 dprt[3] = get_dst_port(qconf, pkt[3], dst.u32[3], portid);
1632         }
1633 }
1634
1635 /*
1636  * Update source and destination MAC addresses in the ethernet header.
1637  * Perform RFC1812 checks and updates for IPV4 packets.
1638  */
1639 static inline void
1640 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP])
1641 {
1642         __m128i te[FWDSTEP];
1643         __m128i ve[FWDSTEP];
1644         __m128i *p[FWDSTEP];
1645
1646         p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *);
1647         p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *);
1648         p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *);
1649         p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *);
1650
1651         ve[0] = val_eth[dst_port[0]];
1652         te[0] = _mm_loadu_si128(p[0]);
1653
1654         ve[1] = val_eth[dst_port[1]];
1655         te[1] = _mm_loadu_si128(p[1]);
1656
1657         ve[2] = val_eth[dst_port[2]];
1658         te[2] = _mm_loadu_si128(p[2]);
1659
1660         ve[3] = val_eth[dst_port[3]];
1661         te[3] = _mm_loadu_si128(p[3]);
1662
1663         /* Update first 12 bytes, keep rest bytes intact. */
1664         te[0] =  _mm_blend_epi16(te[0], ve[0], MASK_ETH);
1665         te[1] =  _mm_blend_epi16(te[1], ve[1], MASK_ETH);
1666         te[2] =  _mm_blend_epi16(te[2], ve[2], MASK_ETH);
1667         te[3] =  _mm_blend_epi16(te[3], ve[3], MASK_ETH);
1668
1669         _mm_storeu_si128(p[0], te[0]);
1670         _mm_storeu_si128(p[1], te[1]);
1671         _mm_storeu_si128(p[2], te[2]);
1672         _mm_storeu_si128(p[3], te[3]);
1673
1674         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1),
1675                 &dst_port[0], pkt[0]->packet_type);
1676         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1),
1677                 &dst_port[1], pkt[1]->packet_type);
1678         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1),
1679                 &dst_port[2], pkt[2]->packet_type);
1680         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1),
1681                 &dst_port[3], pkt[3]->packet_type);
1682 }
1683
1684 /*
1685  * We group consecutive packets with the same destionation port into one burst.
1686  * To avoid extra latency this is done together with some other packet
1687  * processing, but after we made a final decision about packet's destination.
1688  * To do this we maintain:
1689  * pnum - array of number of consecutive packets with the same dest port for
1690  * each packet in the input burst.
1691  * lp - pointer to the last updated element in the pnum.
1692  * dlp - dest port value lp corresponds to.
1693  */
1694
1695 #define GRPSZ   (1 << FWDSTEP)
1696 #define GRPMSK  (GRPSZ - 1)
1697
1698 #define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx)  do { \
1699         if (likely((dlp) == (dcp)[(idx)])) {         \
1700                 (lp)[0]++;                           \
1701         } else {                                     \
1702                 (dlp) = (dcp)[idx];                  \
1703                 (lp) = (pn) + (idx);                 \
1704                 (lp)[0] = 1;                         \
1705         }                                            \
1706 } while (0)
1707
1708 /*
1709  * Group consecutive packets with the same destination port in bursts of 4.
1710  * Suppose we have array of destionation ports:
1711  * dst_port[] = {a, b, c, d,, e, ... }
1712  * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
1713  * We doing 4 comparisions at once and the result is 4 bit mask.
1714  * This mask is used as an index into prebuild array of pnum values.
1715  */
1716 static inline uint16_t *
1717 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2)
1718 {
1719         static const struct {
1720                 uint64_t pnum; /* prebuild 4 values for pnum[]. */
1721                 int32_t  idx;  /* index for new last updated elemnet. */
1722                 uint16_t lpv;  /* add value to the last updated element. */
1723         } gptbl[GRPSZ] = {
1724         {
1725                 /* 0: a != b, b != c, c != d, d != e */
1726                 .pnum = UINT64_C(0x0001000100010001),
1727                 .idx = 4,
1728                 .lpv = 0,
1729         },
1730         {
1731                 /* 1: a == b, b != c, c != d, d != e */
1732                 .pnum = UINT64_C(0x0001000100010002),
1733                 .idx = 4,
1734                 .lpv = 1,
1735         },
1736         {
1737                 /* 2: a != b, b == c, c != d, d != e */
1738                 .pnum = UINT64_C(0x0001000100020001),
1739                 .idx = 4,
1740                 .lpv = 0,
1741         },
1742         {
1743                 /* 3: a == b, b == c, c != d, d != e */
1744                 .pnum = UINT64_C(0x0001000100020003),
1745                 .idx = 4,
1746                 .lpv = 2,
1747         },
1748         {
1749                 /* 4: a != b, b != c, c == d, d != e */
1750                 .pnum = UINT64_C(0x0001000200010001),
1751                 .idx = 4,
1752                 .lpv = 0,
1753         },
1754         {
1755                 /* 5: a == b, b != c, c == d, d != e */
1756                 .pnum = UINT64_C(0x0001000200010002),
1757                 .idx = 4,
1758                 .lpv = 1,
1759         },
1760         {
1761                 /* 6: a != b, b == c, c == d, d != e */
1762                 .pnum = UINT64_C(0x0001000200030001),
1763                 .idx = 4,
1764                 .lpv = 0,
1765         },
1766         {
1767                 /* 7: a == b, b == c, c == d, d != e */
1768                 .pnum = UINT64_C(0x0001000200030004),
1769                 .idx = 4,
1770                 .lpv = 3,
1771         },
1772         {
1773                 /* 8: a != b, b != c, c != d, d == e */
1774                 .pnum = UINT64_C(0x0002000100010001),
1775                 .idx = 3,
1776                 .lpv = 0,
1777         },
1778         {
1779                 /* 9: a == b, b != c, c != d, d == e */
1780                 .pnum = UINT64_C(0x0002000100010002),
1781                 .idx = 3,
1782                 .lpv = 1,
1783         },
1784         {
1785                 /* 0xa: a != b, b == c, c != d, d == e */
1786                 .pnum = UINT64_C(0x0002000100020001),
1787                 .idx = 3,
1788                 .lpv = 0,
1789         },
1790         {
1791                 /* 0xb: a == b, b == c, c != d, d == e */
1792                 .pnum = UINT64_C(0x0002000100020003),
1793                 .idx = 3,
1794                 .lpv = 2,
1795         },
1796         {
1797                 /* 0xc: a != b, b != c, c == d, d == e */
1798                 .pnum = UINT64_C(0x0002000300010001),
1799                 .idx = 2,
1800                 .lpv = 0,
1801         },
1802         {
1803                 /* 0xd: a == b, b != c, c == d, d == e */
1804                 .pnum = UINT64_C(0x0002000300010002),
1805                 .idx = 2,
1806                 .lpv = 1,
1807         },
1808         {
1809                 /* 0xe: a != b, b == c, c == d, d == e */
1810                 .pnum = UINT64_C(0x0002000300040001),
1811                 .idx = 1,
1812                 .lpv = 0,
1813         },
1814         {
1815                 /* 0xf: a == b, b == c, c == d, d == e */
1816                 .pnum = UINT64_C(0x0002000300040005),
1817                 .idx = 0,
1818                 .lpv = 4,
1819         },
1820         };
1821
1822         union {
1823                 uint16_t u16[FWDSTEP + 1];
1824                 uint64_t u64;
1825         } *pnum = (void *)pn;
1826
1827         int32_t v;
1828
1829         dp1 = _mm_cmpeq_epi16(dp1, dp2);
1830         dp1 = _mm_unpacklo_epi16(dp1, dp1);
1831         v = _mm_movemask_ps((__m128)dp1);
1832
1833         /* update last port counter. */
1834         lp[0] += gptbl[v].lpv;
1835
1836         /* if dest port value has changed. */
1837         if (v != GRPMSK) {
1838                 lp = pnum->u16 + gptbl[v].idx;
1839                 lp[0] = 1;
1840                 pnum->u64 = gptbl[v].pnum;
1841         }
1842
1843         return lp;
1844 }
1845
1846 #endif /* APP_LOOKUP_METHOD */
1847
1848 /* main processing loop */
1849 static int
1850 main_loop(__attribute__((unused)) void *dummy)
1851 {
1852         unsigned lcore_id;
1853         uint8_t portid, queueid;
1854         struct lcore_conf *qconf;
1855
1856         l2_phy_interface_t *port;
1857         struct rte_mbuf *pkts_burst[IFM_BURST_SIZE];
1858         uint32_t nb_tx = 0, nb_rx, j, i;
1859         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /US_PER_S * BURST_TX_DRAIN_US;
1860         uint64_t prev_tsc = 0, cur_tsc, diff_tsc;
1861
1862 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1863         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1864         int32_t k;
1865         uint16_t dlp;
1866         uint16_t *lp;
1867         uint16_t dst_port[MAX_PKT_BURST];
1868         __m128i dip[MAX_PKT_BURST / FWDSTEP];
1869         uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP];
1870         uint16_t pnum[MAX_PKT_BURST + 1];
1871 #endif
1872
1873         prev_tsc = 0;
1874
1875         lcore_id = rte_lcore_id();
1876         qconf = &lcore_conf[lcore_id];
1877
1878         if (qconf->n_rx_queue == 0) {
1879                 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id);
1880                 return 0;
1881         }
1882
1883         RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id);
1884         for (i = 0; i < qconf->n_rx_queue; i++) {
1885                 portid = qconf->rx_queue_list[i].port_id;
1886                 queueid = qconf->rx_queue_list[i].queue_id;
1887                 RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id,
1888                 portid, queueid);
1889         }
1890
1891         while (exit_loop) {
1892
1893                 cur_tsc = rte_rdtsc();
1894
1895                 /*
1896                  * TX burst queue drain
1897                  */
1898                 diff_tsc = cur_tsc - prev_tsc;
1899                 if (unlikely(diff_tsc > drain_tsc)) {
1900
1901                         /*
1902                          * This could be optimized (use queueid instead of
1903                          * portid), but it is not called so often
1904                          */
1905                         for (portid = 0; portid < num_ports; portid++) {
1906                                 port = ifm_get_port(portid);
1907                                 if (!port)
1908                                         continue;
1909
1910                                 if(port->tx_buf_len > 0){
1911                                         RTE_SET_USED(nb_tx);
1912                                         port->tx_buf_len = 0;
1913                                 }
1914                                 prev_tsc = cur_tsc;
1915                         }
1916
1917                         prev_tsc = cur_tsc;
1918                 }
1919
1920
1921                 /*
1922                  * Read packet from RX queues
1923                  */
1924                 for (i = 0; i < qconf->n_rx_queue; ++i) {
1925                         portid = qconf->rx_queue_list[i].port_id;
1926                         queueid = qconf->rx_queue_list[i].queue_id;
1927                         port = ifm_get_port(portid);
1928                         if (port != NULL) {
1929                                 nb_rx = port->retrieve_bulk_pkts(portid,
1930                                                  queueid, pkts_burst);
1931                                 port->n_rxpkts += nb_rx;
1932                         } else {
1933                                 printf("port may be un initialized\n");
1934                                 return 0;
1935                         }
1936
1937                         if(nb_rx)
1938                                 rcv_pkt_count[portid] += nb_rx;
1939
1940                         if (nb_rx == 0)
1941                                 continue;
1942
1943 #if (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)
1944 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1945                         {
1946                                 /*
1947                                  * Send nb_rx - nb_rx%8 packets
1948                                  * in groups of 8.
1949                                  */
1950                                 uint32_t n = RTE_ALIGN_FLOOR(nb_rx, 8);
1951                                 for (j = 0; j < n; j += 8) {
1952                                         uint32_t pkt_type =
1953                                                 pkts_burst[j]->packet_type &
1954                                                 pkts_burst[j+1]->packet_type &
1955                                                 pkts_burst[j+2]->packet_type &
1956                                                 pkts_burst[j+3]->packet_type &
1957                                                 pkts_burst[j+4]->packet_type &
1958                                                 pkts_burst[j+5]->packet_type &
1959                                                 pkts_burst[j+6]->packet_type &
1960                                                 pkts_burst[j+7]->packet_type;
1961                                         if (pkt_type & RTE_PTYPE_L3_IPV4) {
1962                                                 simple_ipv4_replay_8pkts(
1963                                                 &pkts_burst[j], portid, qconf);
1964                                         } else if (pkt_type &
1965                                                 RTE_PTYPE_L3_IPV6) {
1966                                                 simple_ipv6_replay_8pkts(&pkts_burst[j],
1967                                                                         portid, qconf);
1968                                         } else {
1969                                                 for (i = j; i < j + 8; i++) {
1970                                                         l3fwd_simple_replay(pkts_burst[i],
1971                                                                         portid, qconf);
1972                                                 }
1973                                         }
1974                                 }
1975                                 for (; j < nb_rx ; j++) {
1976                                         l3fwd_simple_replay(pkts_burst[j],
1977                                                                 portid, qconf);
1978                                 }
1979                         }
1980 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1981
1982                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1983                         for (j = 0; j != k; j += FWDSTEP) {
1984                                 processx4_step1(&pkts_burst[j],
1985                                         &dip[j / FWDSTEP],
1986                                         &ipv4_flag[j / FWDSTEP]);
1987                         }
1988
1989                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1990                         for (j = 0; j != k; j += FWDSTEP) {
1991                                 processx4_step2(qconf, dip[j / FWDSTEP],
1992                                         ipv4_flag[j / FWDSTEP], portid,
1993                                         &pkts_burst[j], &dst_port[j]);
1994                         }
1995
1996                         /*
1997                          * Finish packet processing and group consecutive
1998                          * packets with the same destination port.
1999                          */
2000                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
2001                         if (k != 0) {
2002                                 __m128i dp1, dp2;
2003
2004                                 lp = pnum;
2005                                 lp[0] = 1;
2006
2007                                 processx4_step3(pkts_burst, dst_port);
2008
2009                                 /* dp1: <d[0], d[1], d[2], d[3], ... > */
2010                                 dp1 = _mm_loadu_si128((__m128i *)dst_port);
2011
2012                                 for (j = FWDSTEP; j != k; j += FWDSTEP) {
2013                                         processx4_step3(&pkts_burst[j],
2014                                                 &dst_port[j]);
2015
2016                                         /*
2017                                          * dp2:
2018                                          * <d[j-3], d[j-2], d[j-1], d[j], ... >
2019                                          */
2020                                         dp2 = _mm_loadu_si128((__m128i *)
2021                                                 &dst_port[j - FWDSTEP + 1]);
2022                                         lp  = port_groupx4(&pnum[j - FWDSTEP],
2023                                                 lp, dp1, dp2);
2024
2025                                         /*
2026                                          * dp1:
2027                                          * <d[j], d[j+1], d[j+2], d[j+3], ... >
2028                                          */
2029                                         dp1 = _mm_srli_si128(dp2,
2030                                                 (FWDSTEP - 1) *
2031                                                 sizeof(dst_port[0]));
2032                                 }
2033
2034                                 /*
2035                                  * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
2036                                  */
2037                                 dp2 = _mm_shufflelo_epi16(dp1, 0xf9);
2038                                 lp  = port_groupx4(&pnum[j - FWDSTEP], lp,
2039                                         dp1, dp2);
2040
2041                                 /*
2042                                  * remove values added by the last repeated
2043                                  * dst port.
2044                                  */
2045                                 lp[0]--;
2046                                 dlp = dst_port[j - 1];
2047                         } else {
2048                                 /* set dlp and lp to the never used values. */
2049                                 dlp = BAD_PORT - 1;
2050                                 lp = pnum + MAX_PKT_BURST;
2051                         }
2052
2053                         /* Process up to last 3 packets one by one. */
2054                         switch (nb_rx % FWDSTEP) {
2055                         case 3:
2056                                 process_packet(qconf, pkts_burst[j],
2057                                         dst_port + j, portid);
2058                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2059                                 j++;
2060                         case 2:
2061                                 process_packet(qconf, pkts_burst[j],
2062                                         dst_port + j, portid);
2063                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2064                                 j++;
2065                         case 1:
2066                                 process_packet(qconf, pkts_burst[j],
2067                                         dst_port + j, portid);
2068                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2069                                 j++;
2070                         }
2071
2072                         /*
2073                          * Send packets out, through destination port.
2074                          * Consecuteve pacekts with the same destination port
2075                          * are already grouped together.
2076                          * If destination port for the packet equals BAD_PORT,
2077                          * then free the packet without sending it out.
2078                          */
2079                         for (j = 0; j < nb_rx; j += k) {
2080
2081                                 int32_t m;
2082                                 uint16_t pn;
2083
2084                                 pn = dst_port[j];
2085                                 k = pnum[j];
2086
2087                                 if (likely(pn != BAD_PORT)) {
2088                                         send_packetsx4(qconf, pn,
2089                                                 pkts_burst + j, k);
2090                                 } else {
2091                                         for (m = j; m != j + k; m++)
2092                                                 rte_pktmbuf_free(pkts_burst[m]);
2093                                 }
2094                         }
2095
2096 #endif /* APP_LOOKUP_METHOD */
2097 #else /* ENABLE_MULTI_BUFFER_OPTIMIZE == 0 */
2098
2099                         /* Prefetch first packets */
2100                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
2101                                 rte_prefetch0(rte_pktmbuf_mtod(
2102                                                 pkts_burst[j], void *));
2103                         }
2104
2105                         /* Prefetch and forward already prefetched packets */
2106                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
2107                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
2108                                                 j + PREFETCH_OFFSET], void *));
2109                                 l3fwd_simple_replay(pkts_burst[j], portid,
2110                                         qconf);
2111                         }
2112
2113                         /* Forward remaining prefetched packets */
2114                         for (; j < nb_rx; j++) {
2115                                 l3fwd_simple_replay(pkts_burst[j], portid,
2116                                         qconf);
2117                         }
2118 #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */
2119                 }
2120
2121         }
2122         return 0;
2123 }
2124
2125 /* display usage */
2126 int
2127 print_l4stats(void)
2128 {
2129         unsigned portid;
2130         uint16_t i;
2131         printf ("\n");
2132         printf ("UDP_Replay stats:\n");
2133         printf ("--------------\n");
2134         printf (" Port          Rx Packet               Tx Packet               Rx Pkt Drop             Tx Pkt Drop             arp_pkts\n");
2135         for (i = 0; i < nb_lcore_params; ++i) {
2136                 portid = lcore_params[i].port_id;
2137                 printf (" %u            %"PRId64"                       %"PRId64"                       0                       0                       %"PRId64"",portid, rcv_pkt_count[(uint64_t)portid], tx_pkt_count[(uint64_t)portid], arp_pkts[(uint64_t)portid]);
2138                 printf ("\n");
2139         }
2140         printf ("\n");
2141         return 0;
2142
2143 }
2144
2145 int
2146 clear_stats(void)
2147 {
2148         uint64_t i;
2149         for (i = 0; i < 32; i++) {
2150             rcv_pkt_count[i] = 0;
2151             tx_pkt_count[i] = 0;
2152         }
2153         print_l4stats();
2154         return 0;
2155 }
2156
2157 static int
2158 check_lcore_params(void)
2159 {
2160         uint8_t queue, lcore;
2161         uint16_t i;
2162         int socketid;
2163
2164         for (i = 0; i < nb_lcore_params; ++i) {
2165                 queue = lcore_params[i].queue_id;
2166                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
2167                         printf("invalid queue number: %hhu\n", queue);
2168                         return -1;
2169                 }
2170                 lcore = lcore_params[i].lcore_id;
2171                 if (!rte_lcore_is_enabled(lcore)) {
2172                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
2173                         return -1;
2174                 }
2175                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
2176                         (numa_on == 0)) {
2177                         printf("warning: lcore %hhu is on socket %d with numa off \n",
2178                                 lcore, socketid);
2179                 }
2180         }
2181         return 0;
2182 }
2183
2184 static int
2185 check_port_config(const unsigned nb_ports)
2186 {
2187         unsigned portid;
2188         uint16_t i;
2189
2190         for (i = 0; i < nb_lcore_params; ++i) {
2191                 portid = lcore_params[i].port_id;
2192                 if ((enabled_port_mask & (1 << portid)) == 0) {
2193                         printf("port %u is not enabled in port mask\n", portid);
2194                         return -1;
2195                 }
2196                 if (portid >= nb_ports) {
2197                         printf("port %u is not present on the board\n", portid);
2198                         return -1;
2199                 }
2200         }
2201         return 0;
2202 }
2203
2204 static uint8_t
2205 get_port_n_rx_queues(const uint8_t port)
2206 {
2207         int queue = -1;
2208         uint16_t i;
2209
2210         for (i = 0; i < nb_lcore_params; ++i) {
2211                 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue)
2212                         queue = lcore_params[i].queue_id;
2213         }
2214         return (uint8_t)(++queue);
2215 }
2216
2217 static int
2218 init_lcore_rx_queues(void)
2219 {
2220         uint16_t i, nb_rx_queue;
2221         uint8_t lcore;
2222
2223         for (i = 0; i < nb_lcore_params; ++i) {
2224                 lcore = lcore_params[i].lcore_id;
2225                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
2226                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
2227                         printf("error: too many queues (%u) for lcore: %u\n",
2228                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
2229                         return -1;
2230                 } else {
2231                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
2232                                 lcore_params[i].port_id;
2233                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
2234                                 lcore_params[i].queue_id;
2235                         lcore_conf[lcore].n_rx_queue++;
2236                 }
2237         }
2238         return 0;
2239 }
2240
2241 /* display usage */
2242 static void
2243 print_usage(const char *prgname)
2244 {
2245         printf ("%s [EAL options] -- -p PORTMASK -P"
2246                 "  [--config (port,queue,lcore)[,(port,queue,lcore]]"
2247                 "  [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
2248                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
2249                 "  -P : enable promiscuous mode\n"
2250                 "  --config (port,queue,lcore): rx queues configuration\n"
2251                 "  --eth-dest=X,MM:MM:MM:MM:MM:MM: optional, ethernet destination for port X\n"
2252                 "  --no-numa: optional, disable numa awareness\n"
2253                 "  --no-hw-csum: optional, disable hw ip checksum\n"
2254                 "  --ipv6: optional, specify it if running ipv6 packets\n"
2255                 "  --enable-jumbo: enable jumbo frame"
2256                 " which max packet len is PKTLEN in decimal (64-9600)\n"
2257                 "  --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n",
2258                 prgname);
2259 }
2260
2261 static int parse_max_pkt_len(const char *pktlen)
2262 {
2263         char *end = NULL;
2264         unsigned long len;
2265
2266         /* parse decimal string */
2267         len = strtoul(pktlen, &end, 10);
2268         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
2269                 return -1;
2270
2271         if (len == 0)
2272                 return -1;
2273
2274         return len;
2275 }
2276
2277 static int
2278 parse_link_ip(const char *file_name)
2279 {
2280
2281         uint32_t i, type;
2282         struct rte_cfgfile *file;
2283         const char *entry;
2284         char buf[256];
2285
2286         file = rte_cfgfile_load(file_name, 0);
2287
2288         entry = rte_cfgfile_get_entry(file, "linkip", "num_ports");
2289         numports = (uint32_t)atoi(entry);
2290         if (numports <= 0 || numports > 32)
2291                 rte_panic("numports is not valid\n");
2292
2293         entry = rte_cfgfile_get_entry(file, "linkip", "ip_type");
2294         type = (uint32_t)atoi(entry);
2295
2296         for (i = 0;i < numports; i++) {
2297                 sprintf(buf, "port%d", i);
2298                 entry = rte_cfgfile_get_entry(file, "linkip", buf);
2299
2300                 if (entry == NULL)
2301                         continue;
2302
2303                 if (!type)
2304                         ipv4[i] = strdup(entry);
2305                 else if (type)
2306                         my_inet_pton_ipv6(AF_INET6, entry, &link_ipv6[i][0]);
2307         }
2308
2309         return 0;
2310 }
2311
2312 static int
2313 parse_portmask(const char *portmask)
2314 {
2315         char *end = NULL;
2316         unsigned long pm;
2317
2318         /* parse hexadecimal string */
2319         pm = strtoul(portmask, &end, 16);
2320         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
2321                 return -1;
2322
2323         if (pm == 0)
2324                 return -1;
2325
2326         return pm;
2327 }
2328
2329 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2330 static int
2331 parse_hash_entry_number(const char *hash_entry_num)
2332 {
2333         char *end = NULL;
2334         unsigned long hash_en;
2335         /* parse hexadecimal string */
2336         hash_en = strtoul(hash_entry_num, &end, 16);
2337         if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
2338                 return -1;
2339
2340         if (hash_en == 0)
2341                 return -1;
2342
2343         return hash_en;
2344 }
2345 #endif
2346
2347 static int
2348 parse_config(const char *q_arg)
2349 {
2350         char s[256];
2351         const char *p, *p0 = q_arg;
2352         char *end;
2353         enum fieldnames {
2354                 FLD_PORT = 0,
2355                 FLD_QUEUE,
2356                 FLD_LCORE,
2357                 _NUM_FLD
2358         };
2359         unsigned long int_fld[_NUM_FLD];
2360         char *str_fld[_NUM_FLD];
2361         int i;
2362         unsigned size;
2363
2364         nb_lcore_params = 0;
2365
2366         while ((p = strchr(p0,'(')) != NULL) {
2367                 ++p;
2368                 if((p0 = strchr(p,')')) == NULL)
2369                         return -1;
2370
2371                 size = p0 - p;
2372                 if(size >= sizeof(s))
2373                         return -1;
2374
2375                 snprintf(s, sizeof(s), "%.*s", size, p);
2376                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
2377                         return -1;
2378                 for (i = 0; i < _NUM_FLD; i++){
2379                         errno = 0;
2380                         int_fld[i] = strtoul(str_fld[i], &end, 0);
2381                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
2382                                 return -1;
2383                 }
2384                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
2385                         printf("exceeded max number of lcore params: %hu\n",
2386                                 nb_lcore_params);
2387                         return -1;
2388                 }
2389                 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT];
2390                 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE];
2391                 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE];
2392                 ++nb_lcore_params;
2393         }
2394         lcore_params = lcore_params_array;
2395         return 0;
2396 }
2397
2398 static void
2399 parse_eth_dest(const char *optarg)
2400 {
2401         uint8_t portid;
2402         char *port_end;
2403         uint8_t c, *dest, peer_addr[6];
2404
2405         errno = 0;
2406         portid = strtoul(optarg, &port_end, 10);
2407         if (errno != 0 || port_end == optarg || *port_end++ != ',')
2408                 rte_exit(EXIT_FAILURE,
2409                 "Invalid eth-dest: %s", optarg);
2410         if (portid >= RTE_MAX_ETHPORTS)
2411                 rte_exit(EXIT_FAILURE,
2412                 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
2413                 portid, RTE_MAX_ETHPORTS);
2414
2415         if (cmdline_parse_etheraddr(NULL, port_end,
2416                 &peer_addr, sizeof(peer_addr)) < 0)
2417                 rte_exit(EXIT_FAILURE,
2418                 "Invalid ethernet address: %s\n",
2419                 port_end);
2420         dest = (uint8_t *)&dest_eth_addr[portid];
2421         for (c = 0; c < 6; c++)
2422                 dest[c] = peer_addr[c];
2423         *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
2424 }
2425
2426 #define CMD_LINE_OPT_CONFIG "config"
2427 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
2428 #define CMD_LINE_OPT_NO_NUMA "no-numa"
2429 #define CMD_LINE_OPT_NO_HW_CSUM "no-hw-csum"
2430 #define CMD_LINE_OPT_IPV6 "ipv6"
2431 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
2432 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
2433
2434 /* Parse the argument given in the command line of the application */
2435 static int
2436 parse_args(int argc, char **argv)
2437 {
2438         int opt, ret;
2439         char **argvopt;
2440         int option_index;
2441         char *prgname = argv[0];
2442         static struct option lgopts[] = {
2443                 {CMD_LINE_OPT_CONFIG, 1, 0, 0},
2444                 {CMD_LINE_OPT_ETH_DEST, 1, 0, 0},
2445                 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0},
2446                 {CMD_LINE_OPT_NO_HW_CSUM, 0, 0, 0},
2447                 {CMD_LINE_OPT_IPV6, 0, 0, 0},
2448                 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0},
2449                 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0},
2450                 {NULL, 0, 0, 0}
2451         };
2452
2453         argvopt = argv;
2454
2455         while ((opt = getopt_long(argc, argvopt, "s:p:P",
2456                                 lgopts, &option_index)) != EOF) {
2457
2458                 switch (opt) {
2459                 case 's':
2460                         parse_link_ip(optarg);
2461                         break;
2462                 /* portmask */
2463                 case 'p':
2464                         enabled_port_mask = parse_portmask(optarg);
2465                         if (enabled_port_mask == 0) {
2466                                 printf("invalid portmask\n");
2467                                 print_usage(prgname);
2468                                 return -1;
2469                         }
2470                         break;
2471                 case 'P':
2472                         printf("Promiscuous mode selected\n");
2473                         promiscuous_on = 1;
2474                         break;
2475
2476                 /* long options */
2477                 case 0:
2478                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG,
2479                                 sizeof (CMD_LINE_OPT_CONFIG))) {
2480                                 ret = parse_config(optarg);
2481                                 if (ret) {
2482                                         printf("invalid config\n");
2483                                         print_usage(prgname);
2484                                         return -1;
2485                                 }
2486                         }
2487
2488                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ETH_DEST,
2489                                 sizeof(CMD_LINE_OPT_ETH_DEST))) {
2490                                         parse_eth_dest(optarg);
2491                         }
2492
2493                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA,
2494                                 sizeof(CMD_LINE_OPT_NO_NUMA))) {
2495                                 printf("numa is disabled \n");
2496                                 numa_on = 0;
2497                         }
2498
2499                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_HW_CSUM,
2500                                 sizeof(CMD_LINE_OPT_NO_HW_CSUM))) {
2501                                 printf("numa is hw ip checksum \n");
2502                                 port_conf.rxmode.hw_ip_checksum = 0;
2503                                 rx_conf.rx_free_thresh = 30;
2504                                 csum_on = 0;
2505                         }
2506
2507 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2508                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6,
2509                                 sizeof(CMD_LINE_OPT_IPV6))) {
2510                                 printf("ipv6 is specified \n");
2511                                 ipv6 = 1;
2512                         }
2513 #endif
2514
2515                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO,
2516                                 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) {
2517                                 struct option lenopts = {"max-pkt-len", required_argument, 0, 0};
2518
2519                                 printf("jumbo frame is enabled - disabling simple TX path\n");
2520                                 port_conf.rxmode.jumbo_frame = 1;
2521
2522                                 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */
2523                                 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) {
2524                                         ret = parse_max_pkt_len(optarg);
2525                                         if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){
2526                                                 printf("invalid packet length\n");
2527                                                 print_usage(prgname);
2528                                                 return -1;
2529                                         }
2530                                         port_conf.rxmode.max_rx_pkt_len = ret;
2531                                 }
2532                                 printf("set jumbo frame max packet length to %u\n",
2533                                                 (unsigned int)port_conf.rxmode.max_rx_pkt_len);
2534                         }
2535 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2536                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM,
2537                                 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) {
2538                                 ret = parse_hash_entry_number(optarg);
2539                                 if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) {
2540                                         hash_entry_number = ret;
2541                                 } else {
2542                                         printf("invalid hash entry number\n");
2543                                         print_usage(prgname);
2544                                         return -1;
2545                                 }
2546                         }
2547 #endif
2548                         break;
2549
2550                 default:
2551                         print_usage(prgname);
2552                         return -1;
2553                 }
2554         }
2555
2556         if (optind >= 0)
2557                 argv[optind-1] = prgname;
2558
2559         ret = optind-1;
2560         optind = 0; /* reset getopt lib */
2561         return ret;
2562 }
2563
2564 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2565
2566 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1,
2567                 union ipv4_5tuple_host* key2)
2568 {
2569         key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
2570         key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
2571         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2572         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2573         key2->proto = key1->proto;
2574         key2->pad0 = 0;
2575         key2->pad1 = 0;
2576         return;
2577 }
2578
2579 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1,
2580                 union ipv6_5tuple_host* key2)
2581 {
2582         uint32_t i;
2583         for (i = 0; i < 16; i++)
2584         {
2585                 key2->ip_dst[i] = key1->ip_dst[i];
2586                 key2->ip_src[i] = key1->ip_src[i];
2587         }
2588         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2589         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2590         key2->proto = key1->proto;
2591         key2->pad0 = 0;
2592         key2->pad1 = 0;
2593         key2->reserve = 0;
2594         return;
2595 }
2596
2597 #define BYTE_VALUE_MAX 256
2598 #define ALL_32_BITS 0xffffffff
2599 #define BIT_8_TO_15 0x0000ff00
2600 static inline void
2601 populate_ipv4_few_flow_into_table(const struct rte_hash* h)
2602 {
2603         uint32_t i;
2604         int32_t ret;
2605         uint32_t array_len = sizeof(ipv4_l3fwd_route_array)/sizeof(ipv4_l3fwd_route_array[0]);
2606
2607         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2608         for (i = 0; i < array_len; i++) {
2609                 struct ipv4_l3fwd_route  entry;
2610                 union ipv4_5tuple_host newkey;
2611                 entry = ipv4_l3fwd_route_array[i];
2612                 convert_ipv4_5tuple(&entry.key, &newkey);
2613                 ret = rte_hash_add_key (h,(void *) &newkey);
2614                 if (ret < 0) {
2615                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2616                                 " to the l3fwd hash.\n", i);
2617                 }
2618                 ipv4_l3fwd_out_if[ret] = entry.if_out;
2619         }
2620         printf("Hash: Adding 0x%" PRIx32 " keys\n", array_len);
2621 }
2622
2623 #define BIT_16_TO_23 0x00ff0000
2624 static inline void
2625 populate_ipv6_few_flow_into_table(const struct rte_hash* h)
2626 {
2627         uint32_t i;
2628         int32_t ret;
2629         uint32_t array_len = sizeof(ipv6_l3fwd_route_array)/sizeof(ipv6_l3fwd_route_array[0]);
2630
2631         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2632         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2633         for (i = 0; i < array_len; i++) {
2634                 struct ipv6_l3fwd_route entry;
2635                 union ipv6_5tuple_host newkey;
2636                 entry = ipv6_l3fwd_route_array[i];
2637                 convert_ipv6_5tuple(&entry.key, &newkey);
2638                 ret = rte_hash_add_key (h, (void *) &newkey);
2639                 if (ret < 0) {
2640                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2641                                 " to the l3fwd hash.\n", i);
2642                 }
2643                 ipv6_l3fwd_out_if[ret] = entry.if_out;
2644         }
2645         printf("Hash: Adding 0x%" PRIx32 "keys\n", array_len);
2646 }
2647
2648 #define NUMBER_PORT_USED 4
2649 static inline void
2650 populate_ipv4_many_flow_into_table(const struct rte_hash* h,
2651                 unsigned int nr_flow)
2652 {
2653         unsigned i;
2654         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2655         for (i = 0; i < nr_flow; i++) {
2656                 struct ipv4_l3fwd_route entry;
2657                 union ipv4_5tuple_host newkey;
2658                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2659                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2660                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2661                 /* Create the ipv4 exact match flow */
2662                 memset(&entry, 0, sizeof(entry));
2663                 switch (i & (NUMBER_PORT_USED -1)) {
2664                 case 0:
2665                         entry = ipv4_l3fwd_route_array[0];
2666                         entry.key.ip_dst = IPv4(101,c,b,a);
2667                         break;
2668                 case 1:
2669                         entry = ipv4_l3fwd_route_array[1];
2670                         entry.key.ip_dst = IPv4(201,c,b,a);
2671                         break;
2672                 case 2:
2673                         entry = ipv4_l3fwd_route_array[2];
2674                         entry.key.ip_dst = IPv4(111,c,b,a);
2675                         break;
2676                 case 3:
2677                         entry = ipv4_l3fwd_route_array[3];
2678                         entry.key.ip_dst = IPv4(211,c,b,a);
2679                         break;
2680                 };
2681                 convert_ipv4_5tuple(&entry.key, &newkey);
2682                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2683                 if (ret < 0) {
2684                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2685                 }
2686                 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
2687
2688         }
2689         printf("Hash: Adding 0x%x keys\n", nr_flow);
2690 }
2691
2692 static inline void
2693 populate_ipv6_many_flow_into_table(const struct rte_hash* h,
2694                 unsigned int nr_flow)
2695 {
2696         unsigned i;
2697         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2698         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2699         for (i = 0; i < nr_flow; i++) {
2700                 struct ipv6_l3fwd_route entry;
2701                 union ipv6_5tuple_host newkey;
2702                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2703                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2704                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2705                 /* Create the ipv6 exact match flow */
2706                 memset(&entry, 0, sizeof(entry));
2707                 switch (i & (NUMBER_PORT_USED - 1)) {
2708                 case 0: entry = ipv6_l3fwd_route_array[0]; break;
2709                 case 1: entry = ipv6_l3fwd_route_array[1]; break;
2710                 case 2: entry = ipv6_l3fwd_route_array[2]; break;
2711                 case 3: entry = ipv6_l3fwd_route_array[3]; break;
2712                 };
2713                 entry.key.ip_dst[13] = c;
2714                 entry.key.ip_dst[14] = b;
2715                 entry.key.ip_dst[15] = a;
2716                 convert_ipv6_5tuple(&entry.key, &newkey);
2717                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2718                 if (ret < 0) {
2719                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2720                 }
2721                 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
2722
2723         }
2724         printf("Hash: Adding 0x%x keys\n", nr_flow);
2725 }
2726
2727 #endif
2728
2729 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2730 static void
2731 setup_lpm(int socketid)
2732 {
2733         struct rte_lpm6_config config;
2734         unsigned i;
2735         int ret;
2736         char s[64];
2737
2738         /* create the LPM table */
2739         snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid);
2740         ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid,
2741                                 IPV4_L3FWD_LPM_MAX_RULES, 0);
2742         if (ipv4_l3fwd_lookup_struct[socketid] == NULL)
2743                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
2744                                 " on socket %d\n", socketid);
2745
2746         /* populate the LPM table */
2747         for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) {
2748
2749                 /* skip unused ports */
2750                 if ((1 << ipv4_l3fwd_route_array[i].if_out &
2751                                 enabled_port_mask) == 0)
2752                         continue;
2753
2754                 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid],
2755                         ipv4_l3fwd_route_array[i].ip,
2756                         ipv4_l3fwd_route_array[i].depth,
2757                         ipv4_l3fwd_route_array[i].if_out);
2758
2759                 if (ret < 0) {
2760                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2761                                 "l3fwd LPM table on socket %d\n",
2762                                 i, socketid);
2763                 }
2764
2765                 printf("LPM: Adding route 0x%08x / %d (%d)\n",
2766                         (unsigned)ipv4_l3fwd_route_array[i].ip,
2767                         ipv4_l3fwd_route_array[i].depth,
2768                         ipv4_l3fwd_route_array[i].if_out);
2769         }
2770
2771         /* create the LPM6 table */
2772         snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid);
2773
2774         config.max_rules = IPV6_L3FWD_LPM_MAX_RULES;
2775         config.number_tbl8s = IPV6_L3FWD_LPM_NUMBER_TBL8S;
2776         config.flags = 0;
2777         ipv6_l3fwd_lookup_struct[socketid] = rte_lpm6_create(s, socketid,
2778                                 &config);
2779         if (ipv6_l3fwd_lookup_struct[socketid] == NULL)
2780                 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
2781                                 " on socket %d\n", socketid);
2782
2783         /* populate the LPM table */
2784         for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) {
2785
2786                 /* skip unused ports */
2787                 if ((1 << ipv6_l3fwd_route_array[i].if_out &
2788                                 enabled_port_mask) == 0)
2789                         continue;
2790
2791                 ret = rte_lpm6_add(ipv6_l3fwd_lookup_struct[socketid],
2792                         ipv6_l3fwd_route_array[i].ip,
2793                         ipv6_l3fwd_route_array[i].depth,
2794                         ipv6_l3fwd_route_array[i].if_out);
2795
2796                 if (ret < 0) {
2797                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2798                                 "l3fwd LPM table on socket %d\n",
2799                                 i, socketid);
2800                 }
2801
2802                 printf("LPM: Adding route %s / %d (%d)\n",
2803                         "IPV6",
2804                         ipv6_l3fwd_route_array[i].depth,
2805                         ipv6_l3fwd_route_array[i].if_out);
2806         }
2807 }
2808 #endif
2809
2810 /* Check the link status of all ports in up to 9s, and print them finally */
2811 static void
2812 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
2813 {
2814 #define CHECK_INTERVAL 100 /* 100ms */
2815 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2816         uint8_t portid, count, all_ports_up, print_flag = 0;
2817         struct rte_eth_link link;
2818
2819         printf("\nChecking link status");
2820         fflush(stdout);
2821         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2822                 all_ports_up = 1;
2823                 for (portid = 0; portid < port_num; portid++) {
2824                         if ((port_mask & (1 << portid)) == 0)
2825                                 continue;
2826                         memset(&link, 0, sizeof(link));
2827                         rte_eth_link_get_nowait(portid, &link);
2828                         /* print link status if flag set */
2829                         if (print_flag == 1) {
2830                                 if (link.link_status)
2831                                         printf("Port %d Link Up - speed %u "
2832                                                 "Mbps - %s\n", (uint8_t)portid,
2833                                                 (unsigned)link.link_speed,
2834                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2835                                         ("full-duplex") : ("half-duplex\n"));
2836                                 else
2837                                         printf("Port %d Link Down\n",
2838                                                 (uint8_t)portid);
2839                                 continue;
2840                         }
2841                         /* clear all_ports_up flag if any link down */
2842                         if (link.link_status == 0) {
2843                                 all_ports_up = 0;
2844                                 break;
2845                         }
2846                 }
2847                 /* after finally printing all link status, get out */
2848                 if (print_flag == 1)
2849                         break;
2850
2851                 if (all_ports_up == 0) {
2852                         printf(".");
2853                         fflush(stdout);
2854                         rte_delay_ms(CHECK_INTERVAL);
2855                 }
2856
2857                 /* set the print_flag if all ports up or timeout */
2858                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2859                         print_flag = 1;
2860                         printf("done\n");
2861                 }
2862         }
2863 }
2864
2865 int
2866 main(int argc, char **argv)
2867 {
2868         int ret;
2869         unsigned nb_ports;
2870         unsigned lcore_id;
2871         uint32_t n_tx_queue, nb_lcores;
2872         uint8_t portid, nb_rx_queue;
2873         struct cmdline *cl;
2874         uint32_t size;
2875         struct pipeline_params *params;
2876
2877         /* init EAL */
2878         ret = rte_eal_init(argc, argv);
2879         if (ret < 0)
2880                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2881         argc -= ret;
2882         argv += ret;
2883
2884         timer_lcore = rte_lcore_id();
2885
2886         /* parse application arguments (after the EAL ones) */
2887         ret = parse_args(argc, argv);
2888         if (ret < 0)
2889                 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
2890
2891         if (check_lcore_params() < 0)
2892                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
2893
2894         ret = init_lcore_rx_queues();
2895         if (ret < 0)
2896                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2897
2898         params = rte_malloc(NULL, sizeof(*params), RTE_CACHE_LINE_SIZE);
2899
2900         memcpy(params, &def_pipeline_params, sizeof(def_pipeline_params));
2901         lib_arp_init(params, NULL);
2902
2903         /* configure the interface manager */
2904         ifm_init();
2905
2906         nb_ports = rte_eth_dev_count();
2907         num_ports = nb_ports;
2908         if (nb_ports > RTE_MAX_ETHPORTS)
2909                 nb_ports = RTE_MAX_ETHPORTS;
2910
2911         if (check_port_config(nb_ports) < 0)
2912                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
2913
2914         nb_lcores = rte_lcore_count();
2915
2916         /*
2917          *Configuring port_config_t structure for interface manager initialization
2918          */
2919         size = RTE_CACHE_LINE_ROUNDUP(sizeof(port_config_t));
2920         port_config = rte_zmalloc(NULL, (RTE_MAX_ETHPORTS * size), RTE_CACHE_LINE_SIZE);
2921         if (port_config == NULL)
2922                 rte_panic("port_config is NULL: Memory Allocation failure\n");
2923
2924         /* initialize all ports */
2925         for (portid = 0; portid < nb_ports; portid++) {
2926                 /* skip ports that are not enabled */
2927                 if ((enabled_port_mask & (1 << portid)) == 0) {
2928                         printf("\nSkipping disabled port %d\n", portid);
2929                         num_ports--;
2930                         continue;
2931                 }
2932
2933                 /* init port */
2934                 printf("Initializing port %d ... ", portid );
2935                 fflush(stdout);
2936
2937                 nb_rx_queue = get_port_n_rx_queues(portid);
2938                 n_tx_queue = nb_lcores;
2939                 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
2940                         n_tx_queue = MAX_TX_QUEUE_PER_PORT;
2941
2942                 port_config[portid].port_id = portid;
2943                 port_config[portid].nrx_queue = nb_rx_queue;
2944                 port_config[portid].ntx_queue = n_tx_queue;
2945                 port_config[portid].state = 1;
2946                 port_config[portid].promisc = promiscuous_on;
2947                 port_config[portid].mempool.pool_size = MEMPOOL_SIZE;
2948                 port_config[portid].mempool.buffer_size = BUFFER_SIZE + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
2949                 port_config[portid].mempool.cache_size = CACHE_SIZE;
2950                 port_config[portid].mempool.cpu_socket_id = rte_socket_id();
2951                 memcpy (&port_config[portid].port_conf, &port_conf, sizeof(struct rte_eth_conf));
2952                 memcpy (&port_config[portid].rx_conf, &rx_conf, sizeof(struct rte_eth_rxconf));
2953                 memcpy (&port_config[portid].tx_conf, &tx_conf, sizeof(struct rte_eth_txconf));
2954
2955                 /* Enable TCP and UDP HW Checksum , when required */
2956                 if (csum_on) {
2957                     port_config[portid].tx_conf.txq_flags &=
2958                         ~(ETH_TXQ_FLAGS_NOXSUMTCP|ETH_TXQ_FLAGS_NOXSUMUDP);
2959                 }
2960
2961                 if (ifm_port_setup (portid, &port_config[portid]))
2962                        rte_panic("Port Setup Failed:  %"PRIu32"\n. Try running by disabling checksum with (--no-hw-csum)", portid);
2963         }
2964
2965         check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask);
2966
2967         l3fwd_init();
2968         create_arp_table();
2969         create_nd_table();
2970         populate_lpm_routes();
2971
2972         convert_ipstr_to_numeric();
2973
2974         /* launch per-lcore init on every lcore */
2975         rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
2976         cl = cmdline_stdin_new(main_ctx, "Replay> ");
2977         if (cl == NULL)
2978                 rte_panic("Cannot create cmdline instance\n");
2979         cmdline_interact(cl);
2980         cmdline_stdin_exit(cl);
2981         exit_loop = 0;
2982         rte_exit(0, "Bye!\n");
2983         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2984                 if (rte_eal_wait_lcore(lcore_id) < 0)
2985                         return -1;
2986         }
2987
2988         return 0;
2989 }
2990
2991 /**********************************************************/
2992
2993 struct cmd_obj_clear_result {
2994         cmdline_fixed_string_t clear;
2995         cmdline_fixed_string_t l3fwd;
2996         cmdline_fixed_string_t stats;
2997 };
2998
2999 static void cmd_clear_l3fwd_stats_parsed(
3000         __rte_unused void *parsed_result,
3001         __rte_unused struct cmdline *cl,
3002    __attribute__((unused)) void *data)
3003 {
3004
3005         clear_stats();
3006 }
3007
3008 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_l3fwd_string =
3009         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, l3fwd, "UDP_Replay");
3010 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_clear_string =
3011         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, clear, "clear");
3012 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_stats_string =
3013         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, stats, "stats");
3014
3015 cmdline_parse_inst_t cmd_clear_l3fwd_stats = {
3016         .f = cmd_clear_l3fwd_stats_parsed,  /* function to call */
3017         .data = NULL,      /* 2nd arg of func */
3018         .help_str = "clears UDP_Replay stats for rx/tx",
3019         .tokens = {        /* token list, NULL terminated */
3020                 (void *)&cmd_clear_l3fwd_stats_l3fwd_string,
3021                 (void *)&cmd_clear_l3fwd_stats_clear_string,
3022                 (void *)&cmd_clear_l3fwd_stats_stats_string,
3023                 NULL,
3024         },
3025 };
3026 /**********************************************************/
3027 struct cmd_obj_add_result {
3028         cmdline_fixed_string_t action;
3029         cmdline_fixed_string_t name;
3030 };
3031
3032 static void cmd_l3fwd_stats_parsed(
3033         __rte_unused void *parsed_result,
3034         __rte_unused struct cmdline *cl,
3035    __attribute__((unused)) void *data)
3036 {
3037
3038         /*printf("\n Rx value :     Tx Value :  \n");*/
3039         print_l4stats();
3040         /*cmdline_printf(cl, "Object %s added, ip=%s\n",
3041                        o->name, ip_str);*/
3042 }
3043
3044 cmdline_parse_token_string_t cmd_l3fwd_stats_l3fwd_string =
3045         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, action, "UDP_Replay");
3046 cmdline_parse_token_string_t cmd_l3fwd_stats_stats_string =
3047         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, name, "stats");
3048
3049 cmdline_parse_inst_t cmd_l3fwd_stats = {
3050         .f = cmd_l3fwd_stats_parsed,  /* function to call */
3051         .data = NULL,      /* 2nd arg of func */
3052         .help_str = "UDP_Replay stats for rx/tx",
3053         .tokens = {        /* token list, NULL terminated */
3054                 (void *)&cmd_l3fwd_stats_l3fwd_string,
3055                 (void *)&cmd_l3fwd_stats_stats_string,
3056                 NULL,
3057         },
3058 };
3059 /* quit*/
3060 struct cmd_quit_result {
3061         cmdline_fixed_string_t quit;
3062 };
3063
3064 static void
3065 cmd_quit_parsed(
3066         __rte_unused void *parsed_result,
3067         struct cmdline *cl,
3068         __rte_unused void *data)
3069 {
3070         cmdline_quit(cl);
3071 }
3072
3073 static cmdline_parse_token_string_t cmd_quit_quit =
3074         TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
3075
3076 static cmdline_parse_inst_t cmd_quit = {
3077         .f = cmd_quit_parsed,
3078         .data = NULL,
3079         .help_str = "Quit",
3080         .tokens = {
3081                   (void *) &cmd_quit_quit,
3082                   NULL,
3083         },
3084 };
3085
3086 /**********************************************************/
3087 /****** CONTEXT (list of instruction) */
3088
3089 cmdline_parse_ctx_t main_ctx[] = {
3090         (cmdline_parse_inst_t *)&cmd_l3fwd_stats,
3091         (cmdline_parse_inst_t *)&cmd_clear_l3fwd_stats,
3092         (cmdline_parse_inst_t *)&cmd_quit,
3093         NULL,
3094 };