VNFs/UDP_Replay/main.c

   1 /*
   2 // Copyright (c) 2016-2017 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //      http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 /*
  18 Correlated traffic VNF :
  19 ------------------------
  20 1. Receive UDP packet
  21 2. Modify received packet
  22      a. exchange src mac and destination mac
  23      b. exchange src ip and destination IP for both IPv4 and IPv6 cases
  24      c. exchange UDP src port and UDP destination port
  25      d. change the len of the response according to the IMIX definition (
  26         option to make traffic more realistic to emulate some IoT payloads)
  27 3. send modified packet to the port where it was received.
  28
  29 Such VNF does not need LPM and routing table implementations.
  30 As the packet modification is very minimal  and there is no memory access as the packet is stored in L3 cache the
  31 performance of the solution should be sufficient for testing the UDP NAT performance.
  32 */
  33
  34 #include <stdlib.h>
  35 #include <stdint.h>
  36 #include <inttypes.h>
  37 #include <sys/types.h>
  38 #include <string.h>
  39 #include <sys/queue.h>
  40 #include <stdarg.h>
  41 #include <errno.h>
  42 #include <getopt.h>
  43
  44 #include <rte_common.h>
  45 #include <rte_vect.h>
  46 #include <rte_byteorder.h>
  47 #include <rte_log.h>
  48 #include <rte_memory.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_memzone.h>
  51 #include <rte_eal.h>
  52 #include <rte_per_lcore.h>
  53 #include <rte_launch.h>
  54 #include <rte_atomic.h>
  55 #include <rte_cycles.h>
  56 #include <rte_prefetch.h>
  57 #include <rte_lcore.h>
  58 #include <rte_per_lcore.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_interrupts.h>
  61 #include <rte_pci.h>
  62 #include <rte_random.h>
  63 #include <rte_debug.h>
  64 #include <rte_ether.h>
  65 #include <rte_ethdev.h>
  66 #include <rte_ring.h>
  67 #include <rte_mempool.h>
  68 #include <rte_mbuf.h>
  69 #include <rte_ip.h>
  70 #include <rte_tcp.h>
  71 #include <rte_udp.h>
  72 #include <rte_string_fns.h>
  73 #include <rte_version.h>
  74
  75 #include <cmdline_parse.h>
  76 #include <cmdline_parse_etheraddr.h>
  77 #include <cmdline_rdline.h>
  78 #include <cmdline_socket.h>
  79 #include <cmdline.h>
  80 #include <cmdline_parse_num.h>
  81 #include <cmdline_parse_string.h>
  82 #include <cmdline_parse_ipaddr.h>
  83 #include <rte_errno.h>
  84 #include <rte_cfgfile.h>
  85
  86 #include "parse_obj_list.h"
  87
  88 #include <lib_arp.h>
  89 #include "l2_proto.h"
  90 #include "interface.h"
  91 #include "l3fwd_common.h"
  92 #include "l3fwd_lpm4.h"
  93 #include "l3fwd_lpm6.h"
  94 #include "lib_icmpv6.h"
  95 #include "app.h"
  96 #include "vnf_common.h"
  97 #include "gateway.h"
  98 #define IN6ADDRSZ 16
  99 #define INADDRSZ 4
 100 #define APP_LOOKUP_EXACT_MATCH          0
 101 #define APP_LOOKUP_LPM                  1
 102 #define DO_RFC_1812_CHECKS
 103 #if 1
 104 #ifndef APP_LOOKUP_METHOD
 105 #define APP_LOOKUP_METHOD             APP_LOOKUP_EXACT_MATCH
 106 #endif
 107 #endif
 108
 109 #include <stdio.h>
 110 #include <netinet/in.h>
 111 #include <termios.h>
 112
 113 /*
 114  *  When set to zero, simple forwaring path is eanbled.
 115  *  When set to one, optimized forwarding path is enabled.
 116  *  Note that LPM optimisation path uses SSE4.1 instructions.
 117  */
 118 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__))
 119 #define ENABLE_MULTI_BUFFER_OPTIMIZE    0
 120 #else
 121 #define ENABLE_MULTI_BUFFER_OPTIMIZE    1
 122 #endif
 123
 124 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 125 #include <rte_hash.h>
 126 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 127 #include <rte_lpm.h>
 128 #include <rte_lpm6.h>
 129 #else
 130 #error "APP_LOOKUP_METHOD set to incorrect value"
 131 #endif
 132
 133 #ifndef IPv6_BYTES
 134 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
 135                        "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
 136 #define IPv6_BYTES(addr) \
 137         addr[0],  addr[1], addr[2],  addr[3], \
 138         addr[4],  addr[5], addr[6],  addr[7], \
 139         addr[8],  addr[9], addr[10], addr[11],\
 140         addr[12], addr[13],addr[14], addr[15]
 141 #endif
 142
 143
 144 #define RTE_LOGTYPE_UDP_Replay RTE_LOGTYPE_USER1
 145
 146 #define MAX_JUMBO_PKT_LEN  9600
 147
 148 #define IPV6_ADDR_LEN 16
 149
 150 #define MEMPOOL_CACHE_SIZE 256
 151
 152 /*
 153  * This expression is used to calculate the number of mbufs needed depending on user input, taking
 154  *  into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore.
 155  *  RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192
 156  */
 157
 158 #define NB_MBUF RTE_MAX (                                                                                                                                       \
 159                                 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT +                                                        \
 160                                 nb_ports*nb_lcores*MAX_PKT_BURST +                                                                                      \
 161                                 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT +                                                          \
 162                                 nb_lcores*MEMPOOL_CACHE_SIZE),                                                                                          \
 163                                 (unsigned)8192)
 164
 165 #define MAX_PKT_BURST     32
 166 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
 167
 168 /*
 169  * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send.
 170  */
 171 #define MAX_TX_BURST    (MAX_PKT_BURST / 2)
 172
 173 #define NB_SOCKETS 8
 174
 175 /* Configure how many packets ahead to prefetch, when reading packets */
 176 #define PREFETCH_OFFSET 3
 177
 178 /* Used to mark destination port as 'invalid'. */
 179 #define BAD_PORT        ((uint16_t)-1)
 180
 181 #define FWDSTEP 4
 182
 183 /*
 184  * Configurable number of RX/TX ring descriptors
 185  */
 186 #define RTE_TEST_RX_DESC_DEFAULT 128
 187 #define RTE_TEST_TX_DESC_DEFAULT 512
 188 static uint64_t rcv_pkt_count[32] = {0};
 189 static uint64_t tx_pkt_count[32] = {0};
 190 static uint32_t arp_support;
 191
 192 unsigned num_ports;
 193 struct sockaddr_in ipaddr1, ipaddr2;
 194 /* ethernet addresses of ports */
 195 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
 196
 197 static __m128i val_eth[RTE_MAX_ETHPORTS];
 198
 199 cmdline_parse_ctx_t main_ctx[];
 200
 201 uint32_t timer_lcore;
 202 uint32_t exit_loop = 1;
 203 port_config_t *port_config;
 204
 205 #define MEMPOOL_SIZE    32 * 1024
 206 #define BUFFER_SIZE             2048
 207 #define CACHE_SIZE              256
 208 /* replace first 12B of the ethernet header. */
 209 #define MASK_ETH        0x3f
 210
 211 #define IP_TYPE_IPV4    0
 212 #define IP_TYPE_IPV6    1
 213 #define MAX_IP          32
 214 const char* ipv4[MAX_IP];
 215 uint8_t link_ipv6[MAX_IP][16];
 216 uint32_t        type, numports;
 217 /* mask of enabled ports */
 218 static uint32_t enabled_port_mask = 0;
 219 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */
 220 static int numa_on = 1; /**< NUMA is enabled by default. */
 221 static int csum_on = 1; /**< NUMA is enabled by default. */
 222 struct pipeline_params def_pipeline_params = {
 223         .n_ports_in = 0,
 224         .n_ports_out = 0,
 225         .n_msgq = 0,
 226         .socket_id = 0,
 227         .n_args = 0,
 228         .log_level = 0,
 229 };
 230
 231 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 232 static int ipv6 = 0; /**< ipv6 is false by default. */
 233 #endif
 234
 235 void convert_ipstr_to_numeric(void);
 236
 237 int print_l4stats(void);
 238 int clear_stats(void);
 239
 240 struct mbuf_table {
 241         uint16_t len;
 242         struct rte_mbuf *m_table[MAX_PKT_BURST];
 243 };
 244
 245 struct lcore_rx_queue {
 246         uint8_t port_id;
 247         uint8_t queue_id;
 248 } __rte_cache_aligned;
 249
 250 #define MAX_RX_QUEUE_PER_LCORE 16
 251 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
 252 #define MAX_RX_QUEUE_PER_PORT 128
 253
 254 #define MAX_LCORE_PARAMS 1024
 255 struct lcore_params {
 256         uint8_t port_id;
 257         uint8_t queue_id;
 258         uint8_t lcore_id;
 259 } __rte_cache_aligned;
 260
 261 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 262 static struct lcore_params lcore_params_array_default[] = {
 263         {0, 0, 2},
 264         {0, 1, 2},
 265         {0, 2, 2},
 266         {1, 0, 2},
 267         {1, 1, 2},
 268         {1, 2, 2},
 269         {2, 0, 2},
 270         {3, 0, 3},
 271         {3, 1, 3},
 272 };
 273
 274 static struct lcore_params * lcore_params = lcore_params_array_default;
 275 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 276                                 sizeof(lcore_params_array_default[0]);
 277
 278 static struct rte_eth_conf port_conf = {
 279         .rxmode = {
 280                 .mq_mode = ETH_MQ_RX_RSS,
 281                 .max_rx_pkt_len = ETHER_MAX_LEN,
 282                 .split_hdr_size = 0,
 283                 .header_split   = 0, /**< Header Split disabled */
 284                 .hw_ip_checksum = 1, /**< IP checksum offload enabled */
 285                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 286                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 287                 .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 288         },
 289         .rx_adv_conf = {
 290                 .rss_conf = {
 291                         .rss_key = NULL,
 292                         .rss_hf = ETH_RSS_IP,
 293                 },
 294         },
 295         .txmode = {
 296                 .mq_mode = ETH_MQ_TX_NONE,
 297         },
 298 };
 299
 300 /* empty vmdq configuration structure. Filled in programatically */
 301 static struct rte_eth_rxconf rx_conf = {
 302                 .rx_thresh = {
 303                         .pthresh = 8,
 304                         .hthresh = 8,
 305                         .wthresh = 4,
 306                 },
 307                 .rx_free_thresh = 64,
 308                 .rx_drop_en = 0,
 309                 .rx_deferred_start = 0,
 310 };
 311 static struct rte_eth_txconf tx_conf = {
 312                 .tx_thresh = {
 313                         .pthresh = 36,
 314                         .hthresh = 0,
 315                         .wthresh = 0,
 316                 },
 317                 .tx_rs_thresh = 0,
 318                 .tx_free_thresh = 0,
 319                 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
 320                         ETH_TXQ_FLAGS_NOOFFLOADS,
 321                 .tx_deferred_start = 0,
 322 };
 323
 324 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 325
 326 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 327 #include <rte_hash_crc.h>
 328 #define DEFAULT_HASH_FUNC       rte_hash_crc
 329 #else
 330 #include <rte_jhash.h>
 331 #define DEFAULT_HASH_FUNC       rte_jhash
 332 #endif
 333
 334 struct ipv4_5tuple {
 335         uint32_t ip_dst;
 336         uint32_t ip_src;
 337         uint16_t port_dst;
 338         uint16_t port_src;
 339         uint8_t  proto;
 340 } __attribute__((__packed__));
 341
 342 union ipv4_5tuple_host {
 343         struct {
 344                 uint8_t  pad0;
 345                 uint8_t  proto;
 346                 uint16_t pad1;
 347                 uint32_t ip_src;
 348                 uint32_t ip_dst;
 349                 uint16_t port_src;
 350                 uint16_t port_dst;
 351         };
 352         __m128i xmm;
 353 };
 354
 355 #define XMM_NUM_IN_IPV6_5TUPLE 3
 356
 357 struct ipv6_5tuple {
 358         uint8_t  ip_dst[IPV6_ADDR_LEN];
 359         uint8_t  ip_src[IPV6_ADDR_LEN];
 360         uint16_t port_dst;
 361         uint16_t port_src;
 362         uint8_t  proto;
 363 } __attribute__((__packed__));
 364
 365 union ipv6_5tuple_host {
 366         struct {
 367                 uint16_t pad0;
 368                 uint8_t  proto;
 369                 uint8_t  pad1;
 370                 uint8_t  ip_src[IPV6_ADDR_LEN];
 371                 uint8_t  ip_dst[IPV6_ADDR_LEN];
 372                 uint16_t port_src;
 373                 uint16_t port_dst;
 374                 uint64_t reserve;
 375         };
 376         __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE];
 377 };
 378
 379 struct ipv4_udp_replay_route {
 380         struct ipv4_5tuple key;
 381         uint8_t if_out;
 382 };
 383
 384 struct ipv6_udp_replay_route {
 385         struct ipv6_5tuple key;
 386         uint8_t if_out;
 387 };
 388
 389 static struct ipv4_udp_replay_route ipv4_udp_replay_route_array[] = {
 390         {{IPv4(101,0,0,0), IPv4(100,10,0,1),  101, 11, IPPROTO_TCP}, 0},
 391         {{IPv4(201,0,0,0), IPv4(200,20,0,1),  102, 12, IPPROTO_TCP}, 1},
 392         {{IPv4(111,0,0,0), IPv4(100,30,0,1),  101, 11, IPPROTO_TCP}, 2},
 393         {{IPv4(211,0,0,0), IPv4(200,40,0,1),  102, 12, IPPROTO_TCP}, 3},
 394 };
 395
 396 static struct ipv6_udp_replay_route ipv6_udp_replay_route_array[] = {
 397         {{
 398         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 399         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 400         101, 11, IPPROTO_TCP}, 0},
 401
 402         {{
 403         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 404         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 405         102, 12, IPPROTO_TCP}, 1},
 406
 407         {{
 408         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 409         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 410         101, 11, IPPROTO_TCP}, 2},
 411
 412         {{
 413         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 414         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 415         102, 12, IPPROTO_TCP}, 3},
 416 };
 417
 418 typedef struct rte_hash lookup_struct_t;
 419
 420 #ifdef RTE_ARCH_X86_64
 421 /* default to 4 million hash entries (approx) */
 422 #define UDP_Replay_HASH_ENTRIES         1024*1024*4
 423 #else
 424 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */
 425 #define UDP_Replay_HASH_ENTRIES         1024*1024*1
 426 #endif
 427 #define HASH_ENTRY_NUMBER_DEFAULT       4
 428
 429 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
 430 void
 431 app_link_up_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
 432 {
 433         cp->state = 1;
 434 }
 435 void
 436 app_link_down_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
 437 {
 438         cp->state = 0;
 439 }
 440
 441 void convert_ipstr_to_numeric(void)
 442 {
 443         uint32_t i;
 444         for (i = 0; i < numports; i++)
 445         {
 446                 if (type == IP_TYPE_IPV4) {
 447                         memset(&ipaddr1, '\0', sizeof(struct sockaddr_in));
 448                         ipaddr1.sin_addr.s_addr = inet_addr(ipv4[i]);
 449                         ifm_add_ipv4_port(i, ipaddr1.sin_addr.s_addr, 24);
 450                 } else if (type == IP_TYPE_IPV6) {
 451                         ifm_add_ipv6_port(i, &link_ipv6[i][0], 128);
 452                 }
 453         }
 454 }
 455
 456 static inline uint32_t
 457 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
 458         uint32_t init_val)
 459 {
 460         const union ipv4_5tuple_host *k;
 461         uint32_t t;
 462         const uint32_t *p;
 463
 464         k = data;
 465         t = k->proto;
 466         p = (const uint32_t *)&k->port_src;
 467
 468 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 469         init_val = rte_hash_crc_4byte(t, init_val);
 470         init_val = rte_hash_crc_4byte(k->ip_src, init_val);
 471         init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
 472         init_val = rte_hash_crc_4byte(*p, init_val);
 473 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 474         init_val = rte_jhash_1word(t, init_val);
 475         init_val = rte_jhash_1word(k->ip_src, init_val);
 476         init_val = rte_jhash_1word(k->ip_dst, init_val);
 477         init_val = rte_jhash_1word(*p, init_val);
 478 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 479         return (init_val);
 480 }
 481 static int arp_pkts;
 482 static inline int check_arpicmp(struct rte_mbuf *pkt)
 483 {
 484         uint8_t in_port_id = pkt->port;
 485         uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
 486         uint16_t *eth_proto =
 487                         RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
 488         uint8_t *protocol;
 489         uint32_t prot_offset =
 490                         MBUF_HDR_ROOM + ETH_HDR_SIZE + IP_HDR_PROTOCOL_OFST;
 491         protocol = RTE_MBUF_METADATA_UINT8_PTR(pkt, prot_offset);
 492         if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_ARP) ||
 493                         ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV4)
 494                         && (*protocol == IP_PROTOCOL_ICMP))) {
 495                         process_arpicmp_pkt(pkt, ifm_get_port(in_port_id));
 496                         arp_pkts++;
 497                         return 0;
 498         }
 499         return 1;
 500 }
 501 static inline int check_arpicmpv6(struct rte_mbuf *pkt)
 502 {
 503         struct ether_hdr *eth_h;
 504         struct ipv6_hdr *ipv6_h;
 505         uint8_t in_port_id = pkt->port;
 506         uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
 507         uint16_t *eth_proto =
 508                         RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
 509         eth_h = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
 510         ipv6_h = (struct ipv6_hdr *)((char *)eth_h + sizeof(struct ether_hdr));
 511         if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV6)
 512                                         && (ipv6_h->proto == ICMPV6_PROTOCOL_ID)) {
 513                         process_icmpv6_pkt(pkt, ifm_get_port(in_port_id));
 514                         return 0;
 515         }
 516         return 1;
 517 }
 518
 519 static inline uint32_t
 520 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val)
 521 {
 522         const union ipv6_5tuple_host *k;
 523         uint32_t t;
 524         const uint32_t *p;
 525 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 526         const uint32_t  *ip_src0, *ip_src1, *ip_src2, *ip_src3;
 527         const uint32_t  *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
 528 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 529
 530         k = data;
 531         t = k->proto;
 532         p = (const uint32_t *)&k->port_src;
 533
 534 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 535         ip_src0 = (const uint32_t *) k->ip_src;
 536         ip_src1 = (const uint32_t *)(k->ip_src+4);
 537         ip_src2 = (const uint32_t *)(k->ip_src+8);
 538         ip_src3 = (const uint32_t *)(k->ip_src+12);
 539         ip_dst0 = (const uint32_t *) k->ip_dst;
 540         ip_dst1 = (const uint32_t *)(k->ip_dst+4);
 541         ip_dst2 = (const uint32_t *)(k->ip_dst+8);
 542         ip_dst3 = (const uint32_t *)(k->ip_dst+12);
 543         init_val = rte_hash_crc_4byte(t, init_val);
 544         init_val = rte_hash_crc_4byte(*ip_src0, init_val);
 545         init_val = rte_hash_crc_4byte(*ip_src1, init_val);
 546         init_val = rte_hash_crc_4byte(*ip_src2, init_val);
 547         init_val = rte_hash_crc_4byte(*ip_src3, init_val);
 548         init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
 549         init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
 550         init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
 551         init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
 552         init_val = rte_hash_crc_4byte(*p, init_val);
 553 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 554         init_val = rte_jhash_1word(t, init_val);
 555         init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 556         init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 557         init_val = rte_jhash_1word(*p, init_val);
 558 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 559         return (init_val);
 560 }
 561
 562 #define IPV4_UDP_Replay_NUM_ROUTES \
 563         (sizeof(ipv4_udp_replay_route_array) / sizeof(ipv4_udp_replay_route_array[0]))
 564
 565 #define IPV6_UDP_Replay_NUM_ROUTES \
 566         (sizeof(ipv6_udp_replay_route_array) / sizeof(ipv6_udp_replay_route_array[0]))
 567
 568 static uint8_t ipv4_udp_replay_out_if[UDP_Replay_HASH_ENTRIES] __rte_cache_aligned;
 569 static uint8_t ipv6_udp_replay_out_if[UDP_Replay_HASH_ENTRIES] __rte_cache_aligned;
 570
 571 #endif
 572
 573 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 574 struct ipv4_udp_replay_route {
 575         uint32_t ip;
 576         uint8_t  depth;
 577         uint8_t  if_out;
 578 };
 579
 580 struct ipv6_udp_replay_route {
 581         uint8_t ip[16];
 582         uint8_t  depth;
 583         uint8_t  if_out;
 584 };
 585
 586 static struct ipv4_udp_replay_route ipv4_udp_replay_route_array[] = {
 587         {IPv4(1,1,1,0), 24, 0},
 588         {IPv4(2,1,1,0), 24, 1},
 589         {IPv4(3,1,1,0), 24, 2},
 590         {IPv4(4,1,1,0), 24, 3},
 591         {IPv4(5,1,1,0), 24, 4},
 592         {IPv4(6,1,1,0), 24, 5},
 593         {IPv4(7,1,1,0), 24, 6},
 594         {IPv4(8,1,1,0), 24, 7},
 595 };
 596
 597 static struct ipv6_udp_replay_route ipv6_udp_replay_route_array[] = {
 598         {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0},
 599         {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1},
 600         {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2},
 601         {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3},
 602         {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4},
 603         {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5},
 604         {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6},
 605         {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7},
 606 };
 607
 608 #define IPV4_UDP_Replay_NUM_ROUTES \
 609         (sizeof(ipv4_udp_replay_route_array) / sizeof(ipv4_udp_replay_route_array[0]))
 610 #define IPV6_UDP_Replay_NUM_ROUTES \
 611         (sizeof(ipv6_udp_replay_route_array) / sizeof(ipv6_udp_replay_route_array[0]))
 612
 613 #define IPV4_UDP_Replay_LPM_MAX_RULES         1024
 614 #define IPV6_UDP_Replay_LPM_MAX_RULES         1024
 615 #define IPV6_UDP_Replay_LPM_NUMBER_TBL8S (1 << 16)
 616
 617 typedef struct rte_lpm lookup_struct_t;
 618 typedef struct rte_lpm6 lookup6_struct_t;
 619 static lookup_struct_t *ipv4_udp_replay_lookup_struct[NB_SOCKETS];
 620 static lookup6_struct_t *ipv6_udp_replay_lookup_struct[NB_SOCKETS];
 621 #endif
 622
 623 struct lcore_conf {
 624         uint16_t n_rx_queue;
 625         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 626         uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
 627         struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
 628         lookup_struct_t * ipv4_lookup_struct;
 629 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 630         lookup6_struct_t * ipv6_lookup_struct;
 631 #else
 632         lookup_struct_t * ipv6_lookup_struct;
 633 #endif
 634 } __rte_cache_aligned;
 635
 636 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
 637
 638 /* Send burst of packets on an output interface */
 639 static inline int
 640 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port)
 641 {
 642         struct rte_mbuf **m_table;
 643         int ret;
 644         uint16_t queueid;
 645
 646         queueid = qconf->tx_queue_id[port];
 647         m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 648
 649         ret = rte_eth_tx_burst(port, queueid, m_table, n);
 650         if (unlikely(ret < n)) {
 651                 do {
 652                         rte_pktmbuf_free(m_table[ret]);
 653                 } while (++ret < n);
 654         }
 655         /*Tx Pkt count*/
 656         tx_pkt_count[port] += ret;
 657         return 0;
 658 }
 659
 660 /* Enqueue a single packet, and send burst if queue is filled */
 661 static inline int
 662 send_single_packet(struct rte_mbuf *m, uint8_t port)
 663 {
 664         uint32_t lcore_id;
 665         uint16_t len;
 666         struct lcore_conf *qconf;
 667
 668         lcore_id = rte_lcore_id();
 669
 670         qconf = &lcore_conf[lcore_id];
 671         len = qconf->tx_mbufs[port].len;
 672         qconf->tx_mbufs[port].m_table[len] = m;
 673         len++;
 674
 675         /* enough pkts to be sent */
 676         if (unlikely(len == MAX_PKT_BURST)) {
 677                 send_burst(qconf, MAX_PKT_BURST, port);
 678                 len = 0;
 679         }
 680
 681         qconf->tx_mbufs[port].len = len;
 682         return 0;
 683 }
 684
 685 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 686 static inline __attribute__((always_inline)) void
 687 send_packetsx4(struct lcore_conf *qconf, uint8_t port,
 688         struct rte_mbuf *m[], uint32_t num)
 689 {
 690         uint32_t len, j, n;
 691
 692         len = qconf->tx_mbufs[port].len;
 693
 694         /*
 695          * If TX buffer for that queue is empty, and we have enough packets,
 696          * then send them straightway.
 697          */
 698         if (num >= MAX_TX_BURST && len == 0) {
 699                 n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num);
 700                 if (unlikely(n < num)) {
 701                         do {
 702                                 rte_pktmbuf_free(m[n]);
 703                         } while (++n < num);
 704                 }
 705                 return;
 706         }
 707
 708         /*
 709          * Put packets into TX buffer for that queue.
 710          */
 711
 712         n = len + num;
 713         n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num;
 714
 715         j = 0;
 716         switch (n % FWDSTEP) {
 717         while (j < n) {
 718         case 0:
 719                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 720                 j++;
 721         case 3:
 722                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 723                 j++;
 724         case 2:
 725                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 726                 j++;
 727         case 1:
 728                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 729                 j++;
 730         }
 731         }
 732
 733         len += n;
 734
 735         /* enough pkts to be sent */
 736         if (unlikely(len == MAX_PKT_BURST)) {
 737
 738                 send_burst(qconf, MAX_PKT_BURST, port);
 739
 740                 /* copy rest of the packets into the TX buffer. */
 741                 len = num - n;
 742                 j = 0;
 743                 switch (len % FWDSTEP) {
 744                 while (j < len) {
 745                 case 0:
 746                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 747                         j++;
 748                 case 3:
 749                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 750                         j++;
 751                 case 2:
 752                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 753                         j++;
 754                 case 1:
 755                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 756                         j++;
 757                 }
 758                 }
 759         }
 760
 761         qconf->tx_mbufs[port].len = len;
 762 }
 763 #endif /* APP_LOOKUP_LPM */
 764
 765 #ifdef DO_RFC_1812_CHECKS
 766 static inline int
 767 is_valid_pkt_ipv4(struct ipv4_hdr *pkt, uint32_t link_len)
 768 {
 769         /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 770         /*
 771          * 1. The packet length reported by the Link Layer must be large
 772          * enough to hold the minimum length legal IP datagram (20 bytes).
 773          */
 774         if (link_len < sizeof(struct ipv4_hdr))
 775                 return -1;
 776
 777         /* 2. The IP checksum must be correct. */
 778         /* this is checked in H/W */
 779
 780         /*
 781          * 3. The IP version number must be 4. If the version number is not 4
 782          * then the packet may be another version of IP, such as IPng or
 783          * ST-II.
 784          */
 785         if (((pkt->version_ihl) >> 4) != 4)
 786                 return -3;
 787         /*
 788          * 4. The IP header length field must be large enough to hold the
 789          * minimum length legal IP datagram (20 bytes = 5 words).
 790          */
 791         if ((pkt->version_ihl & 0xf) < 5)
 792                 return -4;
 793
 794         /*
 795          * 5. The IP total length field must be large enough to hold the IP
 796          * datagram header, whose length is specified in the IP header length
 797          * field.
 798          */
 799         if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr))
 800                 return -5;
 801
 802         return 0;
 803 }
 804 #endif
 805
 806 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 807
 808 static __m128i mask0;
 809 static __m128i mask1;
 810 static __m128i mask2;
 811 static inline uint8_t
 812 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_udp_replay_lookup_struct)
 813 {
 814         int ret = 0;
 815         union ipv4_5tuple_host key;
 816
 817         ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live);
 818         __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr));
 819         /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */
 820         key.xmm = _mm_and_si128(data, mask0);
 821         /* Find destination port */
 822         ret = rte_hash_lookup(ipv4_udp_replay_lookup_struct, (const void *)&key);
 823         return (uint8_t)((ret < 0)? portid : ipv4_udp_replay_out_if[ret]);
 824 }
 825
 826 static inline uint8_t
 827 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup_struct_t * ipv6_udp_replay_lookup_struct)
 828 {
 829         int ret = 0;
 830         union ipv6_5tuple_host key;
 831
 832         ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len);
 833         __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr));
 834         __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)));
 835         __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i)));
 836         /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
 837         key.xmm[0] = _mm_and_si128(data0, mask1);
 838         /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */
 839         key.xmm[1] = data1;
 840         /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */
 841         key.xmm[2] = _mm_and_si128(data2, mask2);
 842
 843         /* Find destination port */
 844         ret = rte_hash_lookup(ipv6_udp_replay_lookup_struct, (const void *)&key);
 845         return (uint8_t)((ret < 0)? portid : ipv6_udp_replay_out_if[ret]);
 846 }
 847 #endif
 848
 849 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 850
 851 static inline uint8_t
 852 get_ipv4_dst_port(void *ipv4_hdr,  uint8_t portid, lookup_struct_t * ipv4_udp_replay_lookup_struct)
 853 {
 854         uint8_t next_hop;
 855
 856         return (uint8_t) ((rte_lpm_lookup(ipv4_udp_replay_lookup_struct,
 857                 rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr),
 858                 &next_hop) == 0) ? next_hop : portid);
 859 }
 860
 861 static inline uint8_t
 862 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup6_struct_t * ipv6_udp_replay_lookup_struct)
 863 {
 864         uint8_t next_hop;
 865         return (uint8_t) ((rte_lpm6_lookup(ipv6_udp_replay_lookup_struct,
 866                         ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)?
 867                         next_hop : portid);
 868 }
 869 #endif
 870
 871 static inline void udp_replay_simple_replay(struct rte_mbuf *m, uint8_t portid,
 872         struct lcore_conf *qconf)  __attribute__((unused));
 873
 874 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \
 875         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
 876
 877 #define MASK_ALL_PKTS    0xff
 878 #define EXCLUDE_1ST_PKT 0xfe
 879 #define EXCLUDE_2ND_PKT 0xfd
 880 #define EXCLUDE_3RD_PKT 0xfb
 881 #define EXCLUDE_4TH_PKT 0xf7
 882 #define EXCLUDE_5TH_PKT 0xef
 883 #define EXCLUDE_6TH_PKT 0xdf
 884 #define EXCLUDE_7TH_PKT 0xbf
 885 #define EXCLUDE_8TH_PKT 0x7f
 886
 887 static inline void
 888 simple_ipv4_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
 889 {
 890         struct ether_hdr *eth_hdr[8];
 891         struct ether_hdr tmp;
 892         struct ipv4_hdr *ipv4_hdr[8];
 893         struct udp_hdr *udp_hdr[8];
 894         int i;
 895         l2_phy_interface_t *port = ifm_get_port(portid);
 896         if (port == NULL) {
 897                 printf("port may be un initialized\n");
 898                 return;
 899         }
 900         if (unlikely(arp_support)) {
 901                 check_arpicmp(m[0]);
 902                 check_arpicmp(m[1]);
 903                 check_arpicmp(m[2]);
 904                 check_arpicmp(m[3]);
 905                 check_arpicmp(m[4]);
 906                 check_arpicmp(m[5]);
 907                 check_arpicmp(m[6]);
 908                 check_arpicmp(m[7]);
 909         }
 910
 911         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
 912         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
 913         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
 914         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
 915         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
 916         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
 917         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
 918         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
 919
 920
 921         memset(&tmp,0,sizeof (struct ether_hdr));
 922
 923         for(i=0;i<8;i++)
 924         {
 925
 926         ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
 927         ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
 928         ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
 929         }
 930
 931         /* Handle IPv4 headers.*/
 932         ipv4_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv4_hdr *,
 933                                               sizeof(struct ether_hdr));
 934         ipv4_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv4_hdr *,
 935                                               sizeof(struct ether_hdr));
 936         ipv4_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv4_hdr *,
 937                                               sizeof(struct ether_hdr));
 938         ipv4_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv4_hdr *,
 939                                               sizeof(struct ether_hdr));
 940         ipv4_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv4_hdr *,
 941                                               sizeof(struct ether_hdr));
 942         ipv4_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv4_hdr *,
 943                                               sizeof(struct ether_hdr));
 944         ipv4_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv4_hdr *,
 945                                               sizeof(struct ether_hdr));
 946         ipv4_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv4_hdr *,
 947                                               sizeof(struct ether_hdr));
 948         struct ipv4_hdr temp_ipv4;
 949         temp_ipv4.dst_addr = ipv4_hdr[0]->dst_addr;
 950         ipv4_hdr[0]->dst_addr = ipv4_hdr[0]->src_addr;
 951         ipv4_hdr[0]->src_addr = temp_ipv4.dst_addr;
 952         temp_ipv4.dst_addr = ipv4_hdr[1]->dst_addr;
 953         ipv4_hdr[1]->dst_addr = ipv4_hdr[1]->src_addr;
 954         ipv4_hdr[1]->src_addr = temp_ipv4.dst_addr;
 955         temp_ipv4.dst_addr = ipv4_hdr[2]->dst_addr;
 956         ipv4_hdr[2]->dst_addr = ipv4_hdr[2]->src_addr;
 957         ipv4_hdr[2]->src_addr = temp_ipv4.dst_addr;
 958         temp_ipv4.dst_addr = ipv4_hdr[3]->dst_addr;
 959         ipv4_hdr[3]->dst_addr = ipv4_hdr[3]->src_addr;
 960         ipv4_hdr[3]->src_addr = temp_ipv4.dst_addr;
 961         temp_ipv4.dst_addr = ipv4_hdr[4]->dst_addr;
 962         ipv4_hdr[4]->dst_addr = ipv4_hdr[4]->src_addr;
 963         ipv4_hdr[4]->src_addr = temp_ipv4.dst_addr;
 964         temp_ipv4.dst_addr = ipv4_hdr[5]->dst_addr;
 965         ipv4_hdr[5]->dst_addr = ipv4_hdr[5]->src_addr;
 966         ipv4_hdr[5]->src_addr = temp_ipv4.dst_addr;
 967         temp_ipv4.dst_addr = ipv4_hdr[6]->dst_addr;
 968         ipv4_hdr[6]->dst_addr = ipv4_hdr[6]->src_addr;
 969         ipv4_hdr[6]->src_addr = temp_ipv4.dst_addr;
 970         temp_ipv4.dst_addr = ipv4_hdr[7]->dst_addr;
 971         ipv4_hdr[7]->dst_addr = ipv4_hdr[7]->src_addr;
 972         ipv4_hdr[7]->src_addr = temp_ipv4.dst_addr;
 973
 974         /* Handle UDP headers.*/
 975         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
 976                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 977
 978         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
 979                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 980         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
 981                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 982         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
 983                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 984         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
 985                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 986         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
 987                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 988         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
 989                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 990         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
 991                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 992        /*1) memcpy or assignment.*/
 993
 994         struct udp_hdr temp_udp;
 995         temp_udp.dst_port = udp_hdr[0]->dst_port;
 996         udp_hdr[0]->dst_port = udp_hdr[0]->src_port;
 997         udp_hdr[0]->src_port = temp_udp.dst_port;
 998         temp_udp.dst_port = udp_hdr[1]->dst_port;
 999         udp_hdr[1]->dst_port = udp_hdr[1]->src_port;
1000         udp_hdr[1]->src_port = temp_udp.dst_port;
1001         temp_udp.dst_port = udp_hdr[2]->dst_port;
1002         udp_hdr[2]->dst_port = udp_hdr[2]->src_port;
1003         udp_hdr[2]->src_port = temp_udp.dst_port;
1004         temp_udp.dst_port = udp_hdr[3]->dst_port;
1005         udp_hdr[3]->dst_port = udp_hdr[3]->src_port;
1006         udp_hdr[3]->src_port = temp_udp.dst_port;
1007         temp_udp.dst_port = udp_hdr[4]->dst_port;
1008         udp_hdr[4]->dst_port = udp_hdr[4]->src_port;
1009         udp_hdr[4]->src_port = temp_udp.dst_port;
1010         temp_udp.dst_port = udp_hdr[5]->dst_port;
1011         udp_hdr[5]->dst_port = udp_hdr[5]->src_port;
1012         udp_hdr[5]->src_port = temp_udp.dst_port;
1013         temp_udp.dst_port = udp_hdr[6]->dst_port;
1014         udp_hdr[6]->dst_port = udp_hdr[6]->src_port;
1015         udp_hdr[6]->src_port = temp_udp.dst_port;
1016         temp_udp.dst_port = udp_hdr[7]->dst_port;
1017         udp_hdr[7]->dst_port = udp_hdr[7]->src_port;
1018         udp_hdr[7]->src_port = temp_udp.dst_port;
1019 #ifdef DO_RFC_1812_CHECKS
1020         /* Check to make sure the packet is valid (RFC1812) */
1021         uint8_t valid_mask = MASK_ALL_PKTS;
1022         if (is_valid_pkt_ipv4(ipv4_hdr[0], m[0]->pkt_len) < 0) {
1023                 rte_pktmbuf_free(m[0]);
1024                 valid_mask &= EXCLUDE_1ST_PKT;
1025         }
1026         if (is_valid_pkt_ipv4(ipv4_hdr[1], m[1]->pkt_len) < 0) {
1027                 rte_pktmbuf_free(m[1]);
1028                 valid_mask &= EXCLUDE_2ND_PKT;
1029         }
1030         if (is_valid_pkt_ipv4(ipv4_hdr[2], m[2]->pkt_len) < 0) {
1031                 rte_pktmbuf_free(m[2]);
1032                 valid_mask &= EXCLUDE_3RD_PKT;
1033         }
1034         if (is_valid_pkt_ipv4(ipv4_hdr[3], m[3]->pkt_len) < 0) {
1035                 rte_pktmbuf_free(m[3]);
1036                 valid_mask &= EXCLUDE_4TH_PKT;
1037         }
1038         if (is_valid_pkt_ipv4(ipv4_hdr[4], m[4]->pkt_len) < 0) {
1039                 rte_pktmbuf_free(m[4]);
1040                 valid_mask &= EXCLUDE_5TH_PKT;
1041         }
1042         if (is_valid_pkt_ipv4(ipv4_hdr[5], m[5]->pkt_len) < 0) {
1043                 rte_pktmbuf_free(m[5]);
1044                 valid_mask &= EXCLUDE_6TH_PKT;
1045         }
1046         if (is_valid_pkt_ipv4(ipv4_hdr[6], m[6]->pkt_len) < 0) {
1047                 rte_pktmbuf_free(m[6]);
1048                 valid_mask &= EXCLUDE_7TH_PKT;
1049         }
1050         if (is_valid_pkt_ipv4(ipv4_hdr[7], m[7]->pkt_len) < 0) {
1051                 rte_pktmbuf_free(m[7]);
1052                 valid_mask &= EXCLUDE_8TH_PKT;
1053         }
1054         if (unlikely(valid_mask != MASK_ALL_PKTS)) {
1055                 if (valid_mask == 0){
1056                         return;
1057                 } else {
1058                         uint8_t i = 0;
1059                         for (i = 0; i < 8; i++) {
1060                                 if ((0x1 << i) & valid_mask) {
1061                                         udp_replay_simple_replay(m[i], portid, qconf);
1062                                 }
1063                         }
1064                         return;
1065                 }
1066         }
1067 #endif // End of #ifdef DO_RFC_1812_CHECKS
1068
1069 #ifdef DO_RFC_1812_CHECKS
1070         /* Update time to live and header checksum */
1071         --(ipv4_hdr[0]->time_to_live);
1072         --(ipv4_hdr[1]->time_to_live);
1073         --(ipv4_hdr[2]->time_to_live);
1074         --(ipv4_hdr[3]->time_to_live);
1075         ++(ipv4_hdr[0]->hdr_checksum);
1076         ++(ipv4_hdr[1]->hdr_checksum);
1077         ++(ipv4_hdr[2]->hdr_checksum);
1078         ++(ipv4_hdr[3]->hdr_checksum);
1079         --(ipv4_hdr[4]->time_to_live);
1080         --(ipv4_hdr[5]->time_to_live);
1081         --(ipv4_hdr[6]->time_to_live);
1082         --(ipv4_hdr[7]->time_to_live);
1083         ++(ipv4_hdr[4]->hdr_checksum);
1084         ++(ipv4_hdr[5]->hdr_checksum);
1085         ++(ipv4_hdr[6]->hdr_checksum);
1086         ++(ipv4_hdr[7]->hdr_checksum);
1087 #endif
1088
1089         send_single_packet(m[0],portid );
1090         send_single_packet(m[1],portid );
1091         send_single_packet(m[2],portid );
1092         send_single_packet(m[3],portid);
1093         send_single_packet(m[4],portid);
1094         send_single_packet(m[5],portid);
1095         send_single_packet(m[6],portid);
1096         send_single_packet(m[7],portid);
1097
1098 }
1099
1100 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1,
1101                                  union ipv6_5tuple_host * key)
1102 {
1103         __m128i tmpdata0 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
1104         __m128i tmpdata1 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i)));
1105         __m128i tmpdata2 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i) + sizeof(__m128i)));
1106         key->xmm[0] = _mm_and_si128(tmpdata0, mask0);
1107         key->xmm[1] = tmpdata1;
1108         key->xmm[2] = _mm_and_si128(tmpdata2, mask1);
1109         return;
1110 }
1111
1112 static inline void
1113 simple_ipv6_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
1114 {
1115         struct ether_hdr *eth_hdr[8],tmp;
1116         int i;
1117         __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[8], temp_ipv6;
1118         int32_t ret[8];
1119         union ipv6_5tuple_host key[8];
1120         struct udp_hdr *udp_hdr[8];
1121         l2_phy_interface_t *port = ifm_get_port(portid);
1122         if (port == NULL) {
1123                 printf("port may be un initialized\n");
1124                 return;
1125         }
1126
1127         if (unlikely(arp_support)) {
1128                 check_arpicmpv6(m[0]);
1129                 check_arpicmpv6(m[1]);
1130                 check_arpicmpv6(m[2]);
1131                 check_arpicmpv6(m[3]);
1132                 check_arpicmpv6(m[4]);
1133                 check_arpicmpv6(m[5]);
1134                 check_arpicmpv6(m[6]);
1135                 check_arpicmpv6(m[7]);
1136         }
1137
1138
1139         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
1140         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
1141         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
1142         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
1143         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
1144         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
1145         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
1146         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
1147
1148         memset(&tmp,0,sizeof (struct ether_hdr));
1149
1150         for(i=0;i<8;i++)
1151         {
1152             ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
1153             ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
1154             ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
1155         }
1156         /* Handle IPv6 headers.*/
1157         ipv6_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv6_hdr *,
1158                                               sizeof(struct ether_hdr));
1159         ipv6_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv6_hdr *,
1160                                               sizeof(struct ether_hdr));
1161         ipv6_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv6_hdr *,
1162                                               sizeof(struct ether_hdr));
1163         ipv6_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv6_hdr *,
1164                                               sizeof(struct ether_hdr));
1165         ipv6_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv6_hdr *,
1166                                               sizeof(struct ether_hdr));
1167         ipv6_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv6_hdr *,
1168                                               sizeof(struct ether_hdr));
1169         ipv6_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv6_hdr *,
1170                                               sizeof(struct ether_hdr));
1171         ipv6_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv6_hdr *,
1172                                               sizeof(struct ether_hdr));
1173         for(i=0;i<8;i++)
1174         {
1175            memcpy(temp_ipv6.dst_addr,ipv6_hdr[i]->dst_addr,16);
1176            memcpy(ipv6_hdr[i]->dst_addr,ipv6_hdr[i]->src_addr,16);
1177            memcpy(ipv6_hdr[i]->src_addr,temp_ipv6.dst_addr,16);
1178         }
1179
1180         /* Handle UDP headers.*/
1181         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1182                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1183
1184         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1185                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1186         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1187                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1188         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1189                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1190         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1191                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1192         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1193                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1194         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1195                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1196         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1197                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1198        /*1) memcpy or assignment.*/
1199
1200         struct udp_hdr temp_udp;
1201         for(i=0;i<8;i++)
1202         {
1203            temp_udp.dst_port = udp_hdr[i]->dst_port;
1204            udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1205            udp_hdr[i]->src_port = temp_udp.dst_port;
1206         }
1207         const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3],
1208                                 &key[4], &key[5], &key[6], &key[7]};
1209 #if RTE_VERSION < 0x100b0000
1210         rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1211 #else
1212         rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1213 #endif
1214         send_single_packet(m[0],portid);
1215         send_single_packet(m[1],portid);
1216         send_single_packet(m[2],portid);
1217         send_single_packet(m[3],portid);
1218         send_single_packet(m[4],portid);
1219         send_single_packet(m[5],portid);
1220         send_single_packet(m[6],portid);
1221         send_single_packet(m[7],portid);
1222
1223 }
1224 #endif /* APP_LOOKUP_METHOD */
1225
1226 static inline __attribute__((always_inline)) void
1227 udp_replay_simple_replay(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf)
1228 {
1229         struct ether_hdr *eth_hdr,tmp;
1230         struct ipv4_hdr *ipv4_hdr,temp_ipv4;
1231         struct udp_hdr *udp_hdr,temp_udp;
1232         l2_phy_interface_t *port = ifm_get_port(portid);
1233
1234         if (port == NULL) {
1235                 printf("port may be un initialized\n");
1236                 return;
1237         }
1238         if (m == NULL) {
1239                 printf("Null packet received\n");
1240                 return;
1241         }
1242         if (unlikely(arp_support)) {
1243         if (!check_arpicmp(m))
1244                 return;
1245         }
1246         if (qconf == NULL)
1247                 printf("qconf configuration is NULL\n");
1248         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1249         ether_addr_copy(&eth_hdr->s_addr, &tmp.s_addr);
1250         ether_addr_copy(&eth_hdr->d_addr, &eth_hdr->s_addr);
1251         ether_addr_copy(&tmp.s_addr, &eth_hdr->d_addr);
1252         struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1253
1254         if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1255                 /* Handle IPv4 headers.*/
1256                 ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
1257                                                    sizeof(struct ether_hdr));
1258         temp_ipv4.dst_addr = ipv4_hdr->dst_addr;
1259         ipv4_hdr->dst_addr = ipv4_hdr->src_addr;
1260         ipv4_hdr->src_addr = temp_ipv4.dst_addr;
1261 #ifdef DO_RFC_1812_CHECKS
1262                 /* Check to make sure the packet is valid (RFC1812) */
1263                 if (is_valid_pkt_ipv4(ipv4_hdr, m->pkt_len) < 0) {
1264                         rte_pktmbuf_free(m);
1265                         return;
1266                 }
1267 #endif
1268
1269
1270 #ifdef DO_RFC_1812_CHECKS
1271                 /* Update time to live and header checksum */
1272                 --(ipv4_hdr->time_to_live);
1273                 ++(ipv4_hdr->hdr_checksum);
1274 #endif
1275         /* Handle UDP headers.*/
1276         udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1277                          (sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr)));
1278
1279         /*Swapping Src and Dst Port*/
1280         temp_udp.dst_port = udp_hdr->dst_port;
1281         udp_hdr->dst_port = udp_hdr->src_port;
1282         udp_hdr->src_port = temp_udp.dst_port;
1283
1284                 send_single_packet(m, portid);
1285         } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1286                 /* Handle IPv6 headers.*/
1287                 struct ipv6_hdr *ipv6_hdr,temp_ipv6;
1288
1289                 ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
1290                                                    sizeof(struct ether_hdr));
1291
1292         /*Swapping of Src and Dst IP address*/
1293         memcpy(temp_ipv6.dst_addr,ipv6_hdr->dst_addr,16);
1294         memcpy(ipv6_hdr->dst_addr,ipv6_hdr->src_addr,16);
1295         memcpy(ipv6_hdr->src_addr,temp_ipv6.dst_addr,16);
1296
1297         /* Handle UDP headers.*/
1298         udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1299                          (sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr)));
1300         /*Swapping Src and Dst Port*/
1301         temp_udp.dst_port = udp_hdr->dst_port;
1302         udp_hdr->dst_port = udp_hdr->src_port;
1303         udp_hdr->src_port = temp_udp.dst_port;
1304                 send_single_packet(m, portid);
1305         } else
1306                 /* Free the mbuf that contains non-IPV4/IPV6 packet */
1307                 rte_pktmbuf_free(m);
1308 }
1309
1310 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1311         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1312 #ifdef DO_RFC_1812_CHECKS
1313
1314 #define IPV4_MIN_VER_IHL        0x45
1315 #define IPV4_MAX_VER_IHL        0x4f
1316 #define IPV4_MAX_VER_IHL_DIFF   (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL)
1317
1318 /* Minimum value of IPV4 total length (20B) in network byte order. */
1319 #define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8)
1320
1321 /*
1322  * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2:
1323  * - The IP version number must be 4.
1324  * - The IP header length field must be large enough to hold the
1325  *    minimum length legal IP datagram (20 bytes = 5 words).
1326  * - The IP total length field must be large enough to hold the IP
1327  *   datagram header, whose length is specified in the IP header length
1328  *   field.
1329  * If we encounter invalid IPV4 packet, then set destination port for it
1330  * to BAD_PORT value.
1331  */
1332 static inline __attribute__((always_inline)) void
1333 rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype)
1334 {
1335         uint8_t ihl;
1336
1337         if (RTE_ETH_IS_IPV4_HDR(ptype)) {
1338                 ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL;
1339
1340                 ipv4_hdr->time_to_live--;
1341                 ipv4_hdr->hdr_checksum++;
1342
1343                 if (ihl > IPV4_MAX_VER_IHL_DIFF ||
1344                                 ((uint8_t)ipv4_hdr->total_length == 0 &&
1345                                 ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) {
1346                         dp[0] = BAD_PORT;
1347                 }
1348         }
1349 }
1350
1351 #else
1352 #define rfc1812_process(mb, dp) do { } while (0)
1353 #endif /* DO_RFC_1812_CHECKS */
1354 #endif /* APP_LOOKUP_LPM && ENABLE_MULTI_BUFFER_OPTIMIZE */
1355
1356
1357 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1358         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1359
1360 static inline __attribute__((always_inline)) uint16_t
1361 get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt,
1362         uint32_t dst_ipv4, uint8_t portid)
1363 {
1364         uint8_t next_hop;
1365         struct ipv6_hdr *ipv6_hdr;
1366         struct ether_hdr *eth_hdr;
1367         struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1368
1369         if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1370                 if (rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4,
1371                                 &next_hop) != 0)
1372                         next_hop = portid;
1373         } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1374                 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1375                 ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1);
1376                 if (rte_lpm6_lookup(qconf->ipv6_lookup_struct,
1377                                 ipv6_hdr->dst_addr, &next_hop) != 0)
1378                         next_hop = portid;
1379         } else {
1380                 next_hop = portid;
1381         }
1382
1383         return next_hop;
1384 }
1385
1386 static inline void
1387 process_packet(struct lcore_conf *qconf, struct rte_mbuf *pkt,
1388         uint16_t *dst_port, uint8_t portid)
1389 {
1390         struct ether_hdr *eth_hdr;
1391         struct ipv4_hdr *ipv4_hdr;
1392         uint32_t dst_ipv4;
1393         uint16_t dp;
1394         __m128i te, ve;
1395
1396         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1397         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1398
1399         /*Add swap*/
1400         dst_ipv4 = ipv4_hdr->dst_addr;
1401         dst_ipv4 = rte_be_to_cpu_32(dst_ipv4);
1402
1403         /*Changing the dp to incoming port*/
1404         dp = get_dst_port(qconf, pkt, dst_ipv4, portid);
1405         dp = portid;
1406
1407         te = _mm_loadu_si128((__m128i *)eth_hdr);
1408         ve = val_eth[dp];
1409
1410         dst_port[0] = dp;
1411         rfc1812_process(ipv4_hdr, dst_port, pkt->packet_type);
1412
1413         te =  _mm_blend_epi16(te, ve, MASK_ETH);
1414         _mm_storeu_si128((__m128i *)eth_hdr, te);
1415 }
1416 /* Wont be using the following fucntion*/
1417
1418 /*
1419  * Read packet_type and destination IPV4 addresses from 4 mbufs.
1420  */
1421 static inline void
1422 processx4_step1(struct rte_mbuf *pkt[FWDSTEP],
1423                 __m128i *dip,
1424                 uint32_t *ipv4_flag)
1425 {
1426         struct ipv4_hdr *ipv4_hdr;
1427         struct ether_hdr *eth_hdr;
1428         uint32_t x0, x1, x2, x3;
1429
1430         eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *);
1431         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1432         x0 = ipv4_hdr->dst_addr;
1433         ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4;
1434
1435         eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *);
1436         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1437         x1 = ipv4_hdr->dst_addr;
1438         ipv4_flag[0] &= pkt[1]->packet_type;
1439
1440         eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *);
1441         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1442         x2 = ipv4_hdr->dst_addr;
1443         ipv4_flag[0] &= pkt[2]->packet_type;
1444
1445         eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *);
1446         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1447         x3 = ipv4_hdr->dst_addr;
1448         ipv4_flag[0] &= pkt[3]->packet_type;
1449
1450         dip[0] = _mm_set_epi32(x3, x2, x1, x0);
1451 }
1452
1453 /*
1454  * Lookup into LPM for destination port.
1455  * If lookup fails, use incoming port (portid) as destination port.
1456  */
1457 static inline void
1458 processx4_step2(const struct lcore_conf *qconf,
1459                 __m128i dip,
1460                 uint32_t ipv4_flag,
1461                 uint8_t portid,
1462                 struct rte_mbuf *pkt[FWDSTEP],
1463                 uint16_t dprt[FWDSTEP])
1464 {
1465         rte_xmm_t dst;
1466         const  __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11,
1467                                                 4, 5, 6, 7, 0, 1, 2, 3);
1468
1469         /* Byte swap 4 IPV4 addresses. */
1470         dip = _mm_shuffle_epi8(dip, bswap_mask);
1471
1472         /* if all 4 packets are IPV4. */
1473         if (likely(ipv4_flag)) {
1474                 rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid);
1475         } else {
1476                 dst.x = dip;
1477                 dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid);
1478                 dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid);
1479                 dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid);
1480                 dprt[3] = get_dst_port(qconf, pkt[3], dst.u32[3], portid);
1481         }
1482 }
1483
1484 /*
1485  * Update source and destination MAC addresses in the ethernet header.
1486  * Perform RFC1812 checks and updates for IPV4 packets.
1487  */
1488 static inline void
1489 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP])
1490 {
1491         __m128i te[FWDSTEP];
1492         __m128i ve[FWDSTEP];
1493         __m128i *p[FWDSTEP];
1494
1495         p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *);
1496         p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *);
1497         p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *);
1498         p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *);
1499
1500         ve[0] = val_eth[dst_port[0]];
1501         te[0] = _mm_loadu_si128(p[0]);
1502
1503         ve[1] = val_eth[dst_port[1]];
1504         te[1] = _mm_loadu_si128(p[1]);
1505
1506         ve[2] = val_eth[dst_port[2]];
1507         te[2] = _mm_loadu_si128(p[2]);
1508
1509         ve[3] = val_eth[dst_port[3]];
1510         te[3] = _mm_loadu_si128(p[3]);
1511
1512         /* Update first 12 bytes, keep rest bytes intact. */
1513         te[0] =  _mm_blend_epi16(te[0], ve[0], MASK_ETH);
1514         te[1] =  _mm_blend_epi16(te[1], ve[1], MASK_ETH);
1515         te[2] =  _mm_blend_epi16(te[2], ve[2], MASK_ETH);
1516         te[3] =  _mm_blend_epi16(te[3], ve[3], MASK_ETH);
1517
1518         _mm_storeu_si128(p[0], te[0]);
1519         _mm_storeu_si128(p[1], te[1]);
1520         _mm_storeu_si128(p[2], te[2]);
1521         _mm_storeu_si128(p[3], te[3]);
1522
1523         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1),
1524                 &dst_port[0], pkt[0]->packet_type);
1525         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1),
1526                 &dst_port[1], pkt[1]->packet_type);
1527         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1),
1528                 &dst_port[2], pkt[2]->packet_type);
1529         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1),
1530                 &dst_port[3], pkt[3]->packet_type);
1531 }
1532
1533 /*
1534  * We group consecutive packets with the same destionation port into one burst.
1535  * To avoid extra latency this is done together with some other packet
1536  * processing, but after we made a final decision about packet's destination.
1537  * To do this we maintain:
1538  * pnum - array of number of consecutive packets with the same dest port for
1539  * each packet in the input burst.
1540  * lp - pointer to the last updated element in the pnum.
1541  * dlp - dest port value lp corresponds to.
1542  */
1543
1544 #define GRPSZ   (1 << FWDSTEP)
1545 #define GRPMSK  (GRPSZ - 1)
1546
1547 #define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx)  do { \
1548         if (likely((dlp) == (dcp)[(idx)])) {         \
1549                 (lp)[0]++;                           \
1550         } else {                                     \
1551                 (dlp) = (dcp)[idx];                  \
1552                 (lp) = (pn) + (idx);                 \
1553                 (lp)[0] = 1;                         \
1554         }                                            \
1555 } while (0)
1556
1557 /*
1558  * Group consecutive packets with the same destination port in bursts of 4.
1559  * Suppose we have array of destionation ports:
1560  * dst_port[] = {a, b, c, d,, e, ... }
1561  * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
1562  * We doing 4 comparisions at once and the result is 4 bit mask.
1563  * This mask is used as an index into prebuild array of pnum values.
1564  */
1565 static inline uint16_t *
1566 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2)
1567 {
1568         static const struct {
1569                 uint64_t pnum; /* prebuild 4 values for pnum[]. */
1570                 int32_t  idx;  /* index for new last updated elemnet. */
1571                 uint16_t lpv;  /* add value to the last updated element. */
1572         } gptbl[GRPSZ] = {
1573         {
1574                 /* 0: a != b, b != c, c != d, d != e */
1575                 .pnum = UINT64_C(0x0001000100010001),
1576                 .idx = 4,
1577                 .lpv = 0,
1578         },
1579         {
1580                 /* 1: a == b, b != c, c != d, d != e */
1581                 .pnum = UINT64_C(0x0001000100010002),
1582                 .idx = 4,
1583                 .lpv = 1,
1584         },
1585         {
1586                 /* 2: a != b, b == c, c != d, d != e */
1587                 .pnum = UINT64_C(0x0001000100020001),
1588                 .idx = 4,
1589                 .lpv = 0,
1590         },
1591         {
1592                 /* 3: a == b, b == c, c != d, d != e */
1593                 .pnum = UINT64_C(0x0001000100020003),
1594                 .idx = 4,
1595                 .lpv = 2,
1596         },
1597         {
1598                 /* 4: a != b, b != c, c == d, d != e */
1599                 .pnum = UINT64_C(0x0001000200010001),
1600                 .idx = 4,
1601                 .lpv = 0,
1602         },
1603         {
1604                 /* 5: a == b, b != c, c == d, d != e */
1605                 .pnum = UINT64_C(0x0001000200010002),
1606                 .idx = 4,
1607                 .lpv = 1,
1608         },
1609         {
1610                 /* 6: a != b, b == c, c == d, d != e */
1611                 .pnum = UINT64_C(0x0001000200030001),
1612                 .idx = 4,
1613                 .lpv = 0,
1614         },
1615         {
1616                 /* 7: a == b, b == c, c == d, d != e */
1617                 .pnum = UINT64_C(0x0001000200030004),
1618                 .idx = 4,
1619                 .lpv = 3,
1620         },
1621         {
1622                 /* 8: a != b, b != c, c != d, d == e */
1623                 .pnum = UINT64_C(0x0002000100010001),
1624                 .idx = 3,
1625                 .lpv = 0,
1626         },
1627         {
1628                 /* 9: a == b, b != c, c != d, d == e */
1629                 .pnum = UINT64_C(0x0002000100010002),
1630                 .idx = 3,
1631                 .lpv = 1,
1632         },
1633         {
1634                 /* 0xa: a != b, b == c, c != d, d == e */
1635                 .pnum = UINT64_C(0x0002000100020001),
1636                 .idx = 3,
1637                 .lpv = 0,
1638         },
1639         {
1640                 /* 0xb: a == b, b == c, c != d, d == e */
1641                 .pnum = UINT64_C(0x0002000100020003),
1642                 .idx = 3,
1643                 .lpv = 2,
1644         },
1645         {
1646                 /* 0xc: a != b, b != c, c == d, d == e */
1647                 .pnum = UINT64_C(0x0002000300010001),
1648                 .idx = 2,
1649                 .lpv = 0,
1650         },
1651         {
1652                 /* 0xd: a == b, b != c, c == d, d == e */
1653                 .pnum = UINT64_C(0x0002000300010002),
1654                 .idx = 2,
1655                 .lpv = 1,
1656         },
1657         {
1658                 /* 0xe: a != b, b == c, c == d, d == e */
1659                 .pnum = UINT64_C(0x0002000300040001),
1660                 .idx = 1,
1661                 .lpv = 0,
1662         },
1663         {
1664                 /* 0xf: a == b, b == c, c == d, d == e */
1665                 .pnum = UINT64_C(0x0002000300040005),
1666                 .idx = 0,
1667                 .lpv = 4,
1668         },
1669         };
1670
1671         union {
1672                 uint16_t u16[FWDSTEP + 1];
1673                 uint64_t u64;
1674         } *pnum = (void *)pn;
1675
1676         int32_t v;
1677
1678         dp1 = _mm_cmpeq_epi16(dp1, dp2);
1679         dp1 = _mm_unpacklo_epi16(dp1, dp1);
1680         v = _mm_movemask_ps((__m128)dp1);
1681
1682         /* update last port counter. */
1683         lp[0] += gptbl[v].lpv;
1684
1685         /* if dest port value has changed. */
1686         if (v != GRPMSK) {
1687                 lp = pnum->u16 + gptbl[v].idx;
1688                 lp[0] = 1;
1689                 pnum->u64 = gptbl[v].pnum;
1690         }
1691
1692         return lp;
1693 }
1694
1695 #endif /* APP_LOOKUP_METHOD */
1696
1697 /* main processing loop */
1698 static int
1699 main_loop(__attribute__((unused)) void *dummy)
1700 {
1701         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1702         unsigned lcore_id;
1703         uint64_t prev_tsc, diff_tsc, cur_tsc;
1704         int i, j, nb_rx;
1705         uint8_t portid, queueid;
1706         struct lcore_conf *qconf;
1707         l2_phy_interface_t *port;
1708         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
1709                 US_PER_S * BURST_TX_DRAIN_US;
1710
1711 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1712         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1713         int32_t k;
1714         uint16_t dlp;
1715         uint16_t *lp;
1716         uint16_t dst_port[MAX_PKT_BURST];
1717         __m128i dip[MAX_PKT_BURST / FWDSTEP];
1718         uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP];
1719         uint16_t pnum[MAX_PKT_BURST + 1];
1720 #endif
1721
1722         prev_tsc = 0;
1723
1724         lcore_id = rte_lcore_id();
1725         qconf = &lcore_conf[lcore_id];
1726
1727         if (qconf->n_rx_queue == 0) {
1728                 RTE_LOG(INFO, UDP_Replay, "lcore %u has nothing to do\n", lcore_id);
1729                 return 0;
1730         }
1731
1732         RTE_LOG(INFO, UDP_Replay, "entering main loop on lcore %u\n", lcore_id);
1733
1734         for (i = 0; i < qconf->n_rx_queue; i++) {
1735
1736                 portid = qconf->rx_queue_list[i].port_id;
1737                 queueid = qconf->rx_queue_list[i].queue_id;
1738                 RTE_LOG(INFO, UDP_Replay, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id,
1739                         portid, queueid);
1740         }
1741
1742         while (exit_loop) {
1743
1744                 cur_tsc = rte_rdtsc();
1745
1746                 /*
1747                  * TX burst queue drain
1748                  */
1749                 diff_tsc = cur_tsc - prev_tsc;
1750                 if (unlikely(diff_tsc > drain_tsc)) {
1751
1752                         /*
1753                          * This could be optimized (use queueid instead of
1754                          * portid), but it is not called so often
1755                          */
1756                         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
1757                                 if (qconf->tx_mbufs[portid].len == 0)
1758                                         continue;
1759                                 send_burst(qconf,
1760                                         qconf->tx_mbufs[portid].len,
1761                                         portid);
1762                                 qconf->tx_mbufs[portid].len = 0;
1763                         }
1764
1765                         prev_tsc = cur_tsc;
1766                 }
1767
1768                 /*
1769                  * Read packet from RX queues
1770                  */
1771                 for (i = 0; i < qconf->n_rx_queue; ++i) {
1772                         portid = qconf->rx_queue_list[i].port_id;
1773                         queueid = qconf->rx_queue_list[i].queue_id;
1774                         port = ifm_get_port(portid);
1775                         if (port != NULL) {
1776                                 nb_rx = port->retrieve_bulk_pkts(portid,
1777                                                  queueid, pkts_burst);
1778                                 port->n_rxpkts += nb_rx;
1779                         } else {
1780                                 printf("port may be un initialized\n");
1781                                 return 0;
1782                         }
1783                         if(nb_rx)
1784                             rcv_pkt_count[portid] += nb_rx;
1785                         if (nb_rx == 0)
1786                                 continue;
1787
1788 #if (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)
1789 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1790                         {
1791                                 /*
1792                                  * Send nb_rx - nb_rx%8 packets
1793                                  * in groups of 8.
1794                                  */
1795                                 int32_t n = RTE_ALIGN_FLOOR(nb_rx, 8);
1796                                 for (j = 0; j < n; j += 8) {
1797                                         struct ether_hdr *eth_h0 =
1798                                          rte_pktmbuf_mtod(pkts_burst[j], struct ether_hdr *);
1799                                         struct ether_hdr *eth_h1 =
1800                                          rte_pktmbuf_mtod(pkts_burst[j+1], struct ether_hdr *);
1801                                         struct ether_hdr *eth_h2 =
1802                                          rte_pktmbuf_mtod(pkts_burst[j+2], struct ether_hdr *);
1803                                         struct ether_hdr *eth_h3 =
1804                                          rte_pktmbuf_mtod(pkts_burst[j+3], struct ether_hdr *);
1805                                         struct ether_hdr *eth_h4 =
1806                                          rte_pktmbuf_mtod(pkts_burst[j+4], struct ether_hdr *);
1807                                         struct ether_hdr *eth_h5 =
1808                                          rte_pktmbuf_mtod(pkts_burst[j+5], struct ether_hdr *);
1809                                         struct ether_hdr *eth_h6 =
1810                                          rte_pktmbuf_mtod(pkts_burst[j+6], struct ether_hdr *);
1811                                         struct ether_hdr *eth_h7 =
1812                                          rte_pktmbuf_mtod(pkts_burst[j+7], struct ether_hdr *);
1813
1814                                         uint16_t ether_type;
1815                                         ether_type =    (rte_cpu_to_be_16(eth_h0->ether_type) &
1816                                                          rte_cpu_to_be_16(eth_h1->ether_type) &
1817                                                          rte_cpu_to_be_16(eth_h2->ether_type) &
1818                                                          rte_cpu_to_be_16(eth_h3->ether_type) &
1819                                                          rte_cpu_to_be_16(eth_h4->ether_type) &
1820                                                          rte_cpu_to_be_16(eth_h5->ether_type) &
1821                                                          rte_cpu_to_be_16(eth_h6->ether_type) &
1822                                                          rte_cpu_to_be_16(eth_h7->ether_type));
1823
1824                                         if (ether_type == ETHER_TYPE_IPv4) {
1825                                                 simple_ipv4_replay_8pkts(
1826                                                 &pkts_burst[j], portid, qconf);
1827                                         } else if (ether_type == ETHER_TYPE_IPv6) {
1828                                                 simple_ipv6_replay_8pkts(&pkts_burst[j],
1829                                                                         portid, qconf);
1830                                         } else {
1831                                                 udp_replay_simple_replay(pkts_burst[j],
1832                                                                         portid, qconf);
1833                                                 udp_replay_simple_replay(pkts_burst[j+1],
1834                                                                         portid, qconf);
1835                                                 udp_replay_simple_replay(pkts_burst[j+2],
1836                                                                         portid, qconf);
1837                                                 udp_replay_simple_replay(pkts_burst[j+3],
1838                                                                         portid, qconf);
1839                                                 udp_replay_simple_replay(pkts_burst[j+4],
1840                                                                         portid, qconf);
1841                                                 udp_replay_simple_replay(pkts_burst[j+5],
1842                                                                         portid, qconf);
1843                                                 udp_replay_simple_replay(pkts_burst[j+6],
1844                                                                         portid, qconf);
1845                                                 udp_replay_simple_replay(pkts_burst[j+7],
1846                                                                         portid, qconf);
1847                                         }
1848                                 }
1849
1850                                 for (; j < nb_rx ; j++) {
1851                                         udp_replay_simple_replay(pkts_burst[j],
1852                                                                 portid, qconf);
1853                                 }
1854                         }
1855 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1856
1857                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1858                         for (j = 0; j != k; j += FWDSTEP) {
1859                                 processx4_step1(&pkts_burst[j],
1860                                         &dip[j / FWDSTEP],
1861                                         &ipv4_flag[j / FWDSTEP]);
1862                         }
1863
1864                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1865                         for (j = 0; j != k; j += FWDSTEP) {
1866                                 processx4_step2(qconf, dip[j / FWDSTEP],
1867                                         ipv4_flag[j / FWDSTEP], portid,
1868                                         &pkts_burst[j], &dst_port[j]);
1869                         }
1870
1871                         /*
1872                          * Finish packet processing and group consecutive
1873                          * packets with the same destination port.
1874                          */
1875                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1876                         if (k != 0) {
1877                                 __m128i dp1, dp2;
1878
1879                                 lp = pnum;
1880                                 lp[0] = 1;
1881
1882                                 processx4_step3(pkts_burst, dst_port);
1883
1884                                 /* dp1: <d[0], d[1], d[2], d[3], ... > */
1885                                 dp1 = _mm_loadu_si128((__m128i *)dst_port);
1886
1887                                 for (j = FWDSTEP; j != k; j += FWDSTEP) {
1888                                         processx4_step3(&pkts_burst[j],
1889                                                 &dst_port[j]);
1890
1891                                         /*
1892                                          * dp2:
1893                                          * <d[j-3], d[j-2], d[j-1], d[j], ... >
1894                                          */
1895                                         dp2 = _mm_loadu_si128((__m128i *)
1896                                                 &dst_port[j - FWDSTEP + 1]);
1897                                         lp  = port_groupx4(&pnum[j - FWDSTEP],
1898                                                 lp, dp1, dp2);
1899
1900                                         /*
1901                                          * dp1:
1902                                          * <d[j], d[j+1], d[j+2], d[j+3], ... >
1903                                          */
1904                                         dp1 = _mm_srli_si128(dp2,
1905                                                 (FWDSTEP - 1) *
1906                                                 sizeof(dst_port[0]));
1907                                 }
1908
1909                                 /*
1910                                  * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
1911                                  */
1912                                 dp2 = _mm_shufflelo_epi16(dp1, 0xf9);
1913                                 lp  = port_groupx4(&pnum[j - FWDSTEP], lp,
1914                                         dp1, dp2);
1915
1916                                 /*
1917                                  * remove values added by the last repeated
1918                                  * dst port.
1919                                  */
1920                                 lp[0]--;
1921                                 dlp = dst_port[j - 1];
1922                         } else {
1923                                 /* set dlp and lp to the never used values. */
1924                                 dlp = BAD_PORT - 1;
1925                                 lp = pnum + MAX_PKT_BURST;
1926                         }
1927
1928                         /* Process up to last 3 packets one by one. */
1929                         switch (nb_rx % FWDSTEP) {
1930                         case 3:
1931                                 process_packet(qconf, pkts_burst[j],
1932                                         dst_port + j, portid);
1933                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1934                                 j++;
1935                         case 2:
1936                                 process_packet(qconf, pkts_burst[j],
1937                                         dst_port + j, portid);
1938                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1939                                 j++;
1940                         case 1:
1941                                 process_packet(qconf, pkts_burst[j],
1942                                         dst_port + j, portid);
1943                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1944                                 j++;
1945                         }
1946
1947                         /*
1948                          * Send packets out, through destination port.
1949                          * Consecuteve pacekts with the same destination port
1950                          * are already grouped together.
1951                          * If destination port for the packet equals BAD_PORT,
1952                          * then free the packet without sending it out.
1953                          */
1954                         for (j = 0; j < nb_rx; j += k) {
1955
1956                                 int32_t m;
1957                                 uint16_t pn;
1958
1959                                 pn = dst_port[j];
1960                                 k = pnum[j];
1961
1962                                 if (likely(pn != BAD_PORT)) {
1963                                         send_packetsx4(qconf, pn,
1964                                                 pkts_burst + j, k);
1965                                 } else {
1966                                         for (m = j; m != j + k; m++)
1967                                                 rte_pktmbuf_free(pkts_burst[m]);
1968                                 }
1969                         }
1970
1971 #endif /* APP_LOOKUP_METHOD */
1972 #else /* ENABLE_MULTI_BUFFER_OPTIMIZE == 0 */
1973
1974                         /* Prefetch first packets */
1975                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
1976                                 rte_prefetch0(rte_pktmbuf_mtod(
1977                                                 pkts_burst[j], void *));
1978                         }
1979
1980                         /* Prefetch and forward already prefetched packets */
1981                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
1982                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
1983                                                 j + PREFETCH_OFFSET], void *));
1984                                 udp_replay_simple_replay(pkts_burst[j], portid,
1985                                         qconf);
1986                         }
1987
1988                         /* Forward remaining prefetched packets */
1989                         for (; j < nb_rx; j++) {
1990                                 udp_replay_simple_replay(pkts_burst[j], portid,
1991                                         qconf);
1992                         }
1993 #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */
1994                 }
1995         }
1996 }
1997
1998 /* display usage */
1999 int
2000 print_l4stats(void)
2001 {
2002         unsigned portid;
2003         uint16_t i, j=0;
2004         printf ("\n");
2005         printf ("UDP_Replay stats:\n");
2006         printf ("--------------\n");
2007         printf (" Port      Rx Packet      Tx Packet      Rx Pkt Drop      Tx Pkt Drop      arp_pkts\n");
2008         for (i = 0; i < nb_lcore_params; ++i) {
2009                 portid = lcore_params[i].port_id;
2010                 printf ("%5u%15lu%15lu%17d%17d%14u",portid, rcv_pkt_count[portid], tx_pkt_count[portid],j,j, arp_pkts);
2011                 printf ("\n");
2012         }
2013         printf ("\n");
2014         return 0;
2015
2016 }
2017
2018 int
2019 clear_stats(void)
2020 {
2021         uint64_t i;
2022         for (i = 0; i < 32; i++) {
2023             rcv_pkt_count[i] = 0;
2024             tx_pkt_count[i] = 0;
2025         }
2026         print_l4stats();
2027         return 0;
2028 }
2029
2030 static int
2031 check_lcore_params(void)
2032 {
2033         uint8_t queue, lcore;
2034         uint16_t i;
2035         int socketid;
2036
2037         for (i = 0; i < nb_lcore_params; ++i) {
2038                 queue = lcore_params[i].queue_id;
2039                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
2040                         printf("invalid queue number: %hhu\n", queue);
2041                         return -1;
2042                 }
2043                 lcore = lcore_params[i].lcore_id;
2044                 if (!rte_lcore_is_enabled(lcore)) {
2045                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
2046                         return -1;
2047                 }
2048                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
2049                         (numa_on == 0)) {
2050                         printf("warning: lcore %hhu is on socket %d with numa off \n",
2051                                 lcore, socketid);
2052                 }
2053         }
2054         return 0;
2055 }
2056
2057 static int
2058 check_port_config(const unsigned nb_ports)
2059 {
2060         unsigned portid;
2061         uint16_t i;
2062
2063         for (i = 0; i < nb_lcore_params; ++i) {
2064                 portid = lcore_params[i].port_id;
2065                 if ((enabled_port_mask & (1 << portid)) == 0) {
2066                         printf("port %u is not enabled in port mask\n", portid);
2067                         return -1;
2068                 }
2069                 if (portid >= nb_ports) {
2070                         printf("port %u is not present on the board\n", portid);
2071                         return -1;
2072                 }
2073         }
2074         return 0;
2075 }
2076
2077 static uint8_t
2078 get_port_n_rx_queues(const uint8_t port)
2079 {
2080         int queue = -1;
2081         uint16_t i;
2082
2083         for (i = 0; i < nb_lcore_params; ++i) {
2084                 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue)
2085                         queue = lcore_params[i].queue_id;
2086         }
2087         return (uint8_t)(++queue);
2088 }
2089
2090 static int
2091 init_lcore_rx_queues(void)
2092 {
2093         uint16_t i, nb_rx_queue;
2094         uint8_t lcore;
2095
2096         for (i = 0; i < nb_lcore_params; ++i) {
2097                 lcore = lcore_params[i].lcore_id;
2098                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
2099                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
2100                         printf("error: too many queues (%u) for lcore: %u\n",
2101                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
2102                         return -1;
2103                 } else {
2104                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
2105                                 lcore_params[i].port_id;
2106                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
2107                                 lcore_params[i].queue_id;
2108                         lcore_conf[lcore].n_rx_queue++;
2109                 }
2110         }
2111         return 0;
2112 }
2113
2114 /* display usage */
2115 static void
2116 print_usage(const char *prgname)
2117 {
2118         printf ("%s [EAL options] -- -p PORTMASK -P"
2119                 "  [--config (port,queue,lcore)[,(port,queue,lcore]]"
2120                 "  [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
2121                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
2122                 "  -P : enable promiscuous mode\n"
2123                 "  --config (port,queue,lcore): rx queues configuration\n"
2124                 "  --eth-dest=X,MM:MM:MM:MM:MM:MM: optional, ethernet destination for port X\n"
2125                 "  --no-numa: optional, disable numa awareness\n"
2126                 "  --no-hw-csum: optional, disable hw ip checksum\n"
2127                 "  --ipv6: optional, specify it if running ipv6 packets\n"
2128                 "  --enable-jumbo: enable jumbo frame"
2129                 " which max packet len is PKTLEN in decimal (64-9600)\n"
2130                 "  --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n",
2131                 prgname);
2132 }
2133
2134 static int parse_max_pkt_len(const char *pktlen)
2135 {
2136         char *end = NULL;
2137         unsigned long len;
2138
2139         /* parse decimal string */
2140         len = strtoul(pktlen, &end, 10);
2141         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
2142                 return -1;
2143
2144         if (len == 0)
2145                 return -1;
2146
2147         return len;
2148 }
2149
2150 static int
2151 parse_link_ip(const char *file_name)
2152 {
2153         uint32_t i, type;
2154         struct rte_cfgfile *file;
2155         const char *entry;
2156         char buf[256];
2157         file = rte_cfgfile_load(file_name, 0);
2158         entry = rte_cfgfile_get_entry(file, "linkip", "num_ports");
2159         numports = (uint32_t)atoi(entry);
2160         if (numports <= 0 || numports > 32)
2161                 rte_panic("numports is not valid\n");
2162         entry = rte_cfgfile_get_entry(file, "linkip", "ip_type");
2163         type = (uint32_t)atoi(entry);
2164         for (i = 0;i < numports; i++) {
2165                 sprintf(buf, "port%d", i);
2166                 entry = rte_cfgfile_get_entry(file, "linkip", buf);
2167                 if (entry == NULL)
2168                         continue;
2169                 if (!type)
2170                         ipv4[i] = strdup(entry);
2171                 else if (type)
2172                         my_inet_pton_ipv6(AF_INET6, entry, &link_ipv6[i][0]);
2173         }
2174         return 0;
2175 }
2176 static int
2177 parse_portmask(const char *portmask)
2178 {
2179         char *end = NULL;
2180         unsigned long pm;
2181
2182         /* parse hexadecimal string */
2183         pm = strtoul(portmask, &end, 16);
2184         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
2185                 return -1;
2186
2187         if (pm == 0)
2188                 return -1;
2189
2190         return pm;
2191 }
2192
2193 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2194 static int
2195 parse_hash_entry_number(const char *hash_entry_num)
2196 {
2197         char *end = NULL;
2198         unsigned long hash_en;
2199         /* parse hexadecimal string */
2200         hash_en = strtoul(hash_entry_num, &end, 16);
2201         if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
2202                 return -1;
2203
2204         if (hash_en == 0)
2205                 return -1;
2206
2207         return hash_en;
2208 }
2209 #endif
2210
2211 static int
2212 parse_config(const char *q_arg)
2213 {
2214         char s[256];
2215         const char *p, *p0 = q_arg;
2216         char *end;
2217         enum fieldnames {
2218                 FLD_PORT = 0,
2219                 FLD_QUEUE,
2220                 FLD_LCORE,
2221                 _NUM_FLD
2222         };
2223         unsigned long int_fld[_NUM_FLD];
2224         char *str_fld[_NUM_FLD];
2225         int i;
2226         unsigned size;
2227
2228         nb_lcore_params = 0;
2229
2230         while ((p = strchr(p0,'(')) != NULL) {
2231                 ++p;
2232                 if((p0 = strchr(p,')')) == NULL)
2233                         return -1;
2234
2235                 size = p0 - p;
2236                 if(size >= sizeof(s))
2237                         return -1;
2238
2239                 snprintf(s, sizeof(s), "%.*s", size, p);
2240                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
2241                         return -1;
2242                 for (i = 0; i < _NUM_FLD; i++){
2243                         errno = 0;
2244                         int_fld[i] = strtoul(str_fld[i], &end, 0);
2245                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
2246                                 return -1;
2247                 }
2248                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
2249                         printf("exceeded max number of lcore params: %hu\n",
2250                                 nb_lcore_params);
2251                         return -1;
2252                 }
2253                 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT];
2254                 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE];
2255                 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE];
2256                 ++nb_lcore_params;
2257         }
2258         lcore_params = lcore_params_array;
2259         return 0;
2260 }
2261
2262 static void
2263 parse_eth_dest(const char *optarg)
2264 {
2265         uint8_t portid;
2266         char *port_end;
2267         uint8_t c, *dest, peer_addr[6];
2268
2269         errno = 0;
2270         portid = strtoul(optarg, &port_end, 10);
2271         if (errno != 0 || port_end == optarg || *port_end++ != ',')
2272                 rte_exit(EXIT_FAILURE,
2273                 "Invalid eth-dest: %s", optarg);
2274         if (portid >= RTE_MAX_ETHPORTS)
2275                 rte_exit(EXIT_FAILURE,
2276                 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
2277                 portid, RTE_MAX_ETHPORTS);
2278
2279         if (cmdline_parse_etheraddr(NULL, port_end,
2280                 &peer_addr, sizeof(peer_addr)) < 0)
2281                 rte_exit(EXIT_FAILURE,
2282                 "Invalid ethernet address: %s\n",
2283                 port_end);
2284         dest = (uint8_t *)&dest_eth_addr[portid];
2285         for (c = 0; c < 6; c++)
2286                 dest[c] = peer_addr[c];
2287         *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
2288 }
2289
2290 #define CMD_LINE_OPT_CONFIG "config"
2291 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
2292 #define CMD_LINE_OPT_NO_NUMA "no-numa"
2293 #define CMD_LINE_OPT_NO_HW_CSUM "no-hw-csum"
2294 #define CMD_LINE_OPT_IPV6 "ipv6"
2295 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
2296 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
2297
2298 /* Parse the argument given in the command line of the application */
2299 static int
2300 parse_args(int argc, char **argv)
2301 {
2302         int opt, ret;
2303         char **argvopt;
2304         int option_index;
2305         char *prgname = argv[0];
2306         static struct option lgopts[] = {
2307                 {CMD_LINE_OPT_CONFIG, 1, 0, 0},
2308                 {CMD_LINE_OPT_ETH_DEST, 1, 0, 0},
2309                 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0},
2310                 {CMD_LINE_OPT_NO_HW_CSUM, 0, 0, 0},
2311                 {CMD_LINE_OPT_IPV6, 0, 0, 0},
2312                 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0},
2313                 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0},
2314                 {NULL, 0, 0, 0}
2315         };
2316
2317         argvopt = argv;
2318
2319         while ((opt = getopt_long(argc, argvopt, "s:p:P",
2320                                 lgopts, &option_index)) != EOF) {
2321
2322                 switch (opt) {
2323                 case 's':
2324                         parse_link_ip(optarg);
2325                         arp_support = 1;
2326                         break;
2327                 /* portmask */
2328                 case 'p':
2329                         enabled_port_mask = parse_portmask(optarg);
2330                         if (enabled_port_mask == 0) {
2331                                 printf("invalid portmask\n");
2332                                 print_usage(prgname);
2333                                 return -1;
2334                         }
2335                         break;
2336                 case 'P':
2337                         printf("Promiscuous mode selected\n");
2338                         promiscuous_on = 1;
2339                         break;
2340
2341                 /* long options */
2342                 case 0:
2343                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG,
2344                                 sizeof (CMD_LINE_OPT_CONFIG))) {
2345                                 ret = parse_config(optarg);
2346                                 if (ret) {
2347                                         printf("invalid config\n");
2348                                         print_usage(prgname);
2349                                         return -1;
2350                                 }
2351                         }
2352
2353                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ETH_DEST,
2354                                 sizeof(CMD_LINE_OPT_ETH_DEST))) {
2355                                         parse_eth_dest(optarg);
2356                         }
2357
2358                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA,
2359                                 sizeof(CMD_LINE_OPT_NO_NUMA))) {
2360                                 printf("numa is disabled \n");
2361                                 numa_on = 0;
2362                         }
2363
2364                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_HW_CSUM,
2365                                 sizeof(CMD_LINE_OPT_NO_HW_CSUM))) {
2366                                 printf("numa is hw ip checksum \n");
2367                                 port_conf.rxmode.hw_ip_checksum = 0;
2368                                 rx_conf.rx_free_thresh = 30;
2369                                 csum_on = 0;
2370                         }
2371
2372 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2373                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6,
2374                                 sizeof(CMD_LINE_OPT_IPV6))) {
2375                                 printf("ipv6 is specified \n");
2376                                 ipv6 = 1;
2377                         }
2378 #endif
2379
2380                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO,
2381                                 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) {
2382                                 struct option lenopts = {"max-pkt-len", required_argument, 0, 0};
2383
2384                                 printf("jumbo frame is enabled - disabling simple TX path\n");
2385                                 port_conf.rxmode.jumbo_frame = 1;
2386
2387                                 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */
2388                                 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) {
2389                                         ret = parse_max_pkt_len(optarg);
2390                                         if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){
2391                                                 printf("invalid packet length\n");
2392                                                 print_usage(prgname);
2393                                                 return -1;
2394                                         }
2395                                         port_conf.rxmode.max_rx_pkt_len = ret;
2396                                 }
2397                                 printf("set jumbo frame max packet length to %u\n",
2398                                                 (unsigned int)port_conf.rxmode.max_rx_pkt_len);
2399                         }
2400 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2401                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM,
2402                                 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) {
2403                                 ret = parse_hash_entry_number(optarg);
2404                                 if ((ret > 0) && (ret <= UDP_Replay_HASH_ENTRIES)) {
2405                                         hash_entry_number = ret;
2406                                 } else {
2407                                         printf("invalid hash entry number\n");
2408                                         print_usage(prgname);
2409                                         return -1;
2410                                 }
2411                         }
2412 #endif
2413                         break;
2414
2415                 default:
2416                         print_usage(prgname);
2417                         return -1;
2418                 }
2419         }
2420
2421         if (optind >= 0)
2422                 argv[optind-1] = prgname;
2423
2424         ret = optind-1;
2425         optind = 0; /* reset getopt lib */
2426         return ret;
2427 }
2428
2429 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2430
2431 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1,
2432                 union ipv4_5tuple_host* key2)
2433 {
2434         key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
2435         key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
2436         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2437         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2438         key2->proto = key1->proto;
2439         key2->pad0 = 0;
2440         key2->pad1 = 0;
2441         return;
2442 }
2443
2444 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1,
2445                 union ipv6_5tuple_host* key2)
2446 {
2447         uint32_t i;
2448         for (i = 0; i < 16; i++)
2449         {
2450                 key2->ip_dst[i] = key1->ip_dst[i];
2451                 key2->ip_src[i] = key1->ip_src[i];
2452         }
2453         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2454         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2455         key2->proto = key1->proto;
2456         key2->pad0 = 0;
2457         key2->pad1 = 0;
2458         key2->reserve = 0;
2459         return;
2460 }
2461
2462 #define BYTE_VALUE_MAX 256
2463 #define ALL_32_BITS 0xffffffff
2464 #define BIT_8_TO_15 0x0000ff00
2465 static inline void
2466 populate_ipv4_few_flow_into_table(const struct rte_hash* h)
2467 {
2468         uint32_t i;
2469         int32_t ret;
2470         uint32_t array_len = sizeof(ipv4_udp_replay_route_array)/sizeof(ipv4_udp_replay_route_array[0]);
2471
2472         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2473         for (i = 0; i < array_len; i++) {
2474                 struct ipv4_udp_replay_route  entry;
2475                 union ipv4_5tuple_host newkey;
2476                 entry = ipv4_udp_replay_route_array[i];
2477                 convert_ipv4_5tuple(&entry.key, &newkey);
2478                 ret = rte_hash_add_key (h,(void *) &newkey);
2479                 if (ret < 0) {
2480                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2481                                 " to the udp_replay hash.\n", i);
2482                 }
2483                 ipv4_udp_replay_out_if[ret] = entry.if_out;
2484         }
2485         printf("Hash: Adding 0x%" PRIx32 " keys\n", array_len);
2486 }
2487
2488 #define BIT_16_TO_23 0x00ff0000
2489 static inline void
2490 populate_ipv6_few_flow_into_table(const struct rte_hash* h)
2491 {
2492         uint32_t i;
2493         int32_t ret;
2494         uint32_t array_len = sizeof(ipv6_udp_replay_route_array)/sizeof(ipv6_udp_replay_route_array[0]);
2495
2496         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2497         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2498         for (i = 0; i < array_len; i++) {
2499                 struct ipv6_udp_replay_route entry;
2500                 union ipv6_5tuple_host newkey;
2501                 entry = ipv6_udp_replay_route_array[i];
2502                 convert_ipv6_5tuple(&entry.key, &newkey);
2503                 ret = rte_hash_add_key (h, (void *) &newkey);
2504                 if (ret < 0) {
2505                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2506                                 " to the udp_replay hash.\n", i);
2507                 }
2508                 ipv6_udp_replay_out_if[ret] = entry.if_out;
2509         }
2510         printf("Hash: Adding 0x%" PRIx32 "keys\n", array_len);
2511 }
2512
2513 #define NUMBER_PORT_USED 4
2514 static inline void
2515 populate_ipv4_many_flow_into_table(const struct rte_hash* h,
2516                 unsigned int nr_flow)
2517 {
2518         unsigned i;
2519         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2520         for (i = 0; i < nr_flow; i++) {
2521                 struct ipv4_udp_replay_route entry;
2522                 union ipv4_5tuple_host newkey;
2523                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2524                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2525                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2526                 /* Create the ipv4 exact match flow */
2527                 memset(&entry, 0, sizeof(entry));
2528                 switch (i & (NUMBER_PORT_USED -1)) {
2529                 case 0:
2530                         entry = ipv4_udp_replay_route_array[0];
2531                         entry.key.ip_dst = IPv4(101,c,b,a);
2532                         break;
2533                 case 1:
2534                         entry = ipv4_udp_replay_route_array[1];
2535                         entry.key.ip_dst = IPv4(201,c,b,a);
2536                         break;
2537                 case 2:
2538                         entry = ipv4_udp_replay_route_array[2];
2539                         entry.key.ip_dst = IPv4(111,c,b,a);
2540                         break;
2541                 case 3:
2542                         entry = ipv4_udp_replay_route_array[3];
2543                         entry.key.ip_dst = IPv4(211,c,b,a);
2544                         break;
2545                 };
2546                 convert_ipv4_5tuple(&entry.key, &newkey);
2547                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2548                 if (ret < 0) {
2549                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2550                 }
2551                 ipv4_udp_replay_out_if[ret] = (uint8_t) entry.if_out;
2552
2553         }
2554         printf("Hash: Adding 0x%x keys\n", nr_flow);
2555 }
2556
2557 static inline void
2558 populate_ipv6_many_flow_into_table(const struct rte_hash* h,
2559                 unsigned int nr_flow)
2560 {
2561         unsigned i;
2562         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2563         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2564         for (i = 0; i < nr_flow; i++) {
2565                 struct ipv6_udp_replay_route entry;
2566                 union ipv6_5tuple_host newkey;
2567                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2568                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2569                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2570                 /* Create the ipv6 exact match flow */
2571                 memset(&entry, 0, sizeof(entry));
2572                 switch (i & (NUMBER_PORT_USED - 1)) {
2573                 case 0: entry = ipv6_udp_replay_route_array[0]; break;
2574                 case 1: entry = ipv6_udp_replay_route_array[1]; break;
2575                 case 2: entry = ipv6_udp_replay_route_array[2]; break;
2576                 case 3: entry = ipv6_udp_replay_route_array[3]; break;
2577                 };
2578                 entry.key.ip_dst[13] = c;
2579                 entry.key.ip_dst[14] = b;
2580                 entry.key.ip_dst[15] = a;
2581                 convert_ipv6_5tuple(&entry.key, &newkey);
2582                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2583                 if (ret < 0) {
2584                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2585                 }
2586                 ipv6_udp_replay_out_if[ret] = (uint8_t) entry.if_out;
2587
2588         }
2589         printf("Hash: Adding 0x%x keys\n", nr_flow);
2590 }
2591
2592 #endif
2593
2594 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2595 static void
2596 setup_lpm(int socketid)
2597 {
2598         struct rte_lpm6_config config;
2599         unsigned i;
2600         int ret;
2601         char s[64];
2602
2603         /* create the LPM table */
2604         snprintf(s, sizeof(s), "IPV4_UDP_Replay_LPM_%d", socketid);
2605         ipv4_udp_replay_lookup_struct[socketid] = rte_lpm_create(s, socketid,
2606                                 IPV4_UDP_Replay_LPM_MAX_RULES, 0);
2607         if (ipv4_udp_replay_lookup_struct[socketid] == NULL)
2608                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay LPM table"
2609                                 " on socket %d\n", socketid);
2610
2611         /* populate the LPM table */
2612         for (i = 0; i < IPV4_UDP_Replay_NUM_ROUTES; i++) {
2613
2614                 /* skip unused ports */
2615                 if ((1 << ipv4_udp_replay_route_array[i].if_out &
2616                                 enabled_port_mask) == 0)
2617                         continue;
2618
2619                 ret = rte_lpm_add(ipv4_udp_replay_lookup_struct[socketid],
2620                         ipv4_udp_replay_route_array[i].ip,
2621                         ipv4_udp_replay_route_array[i].depth,
2622                         ipv4_udp_replay_route_array[i].if_out);
2623
2624                 if (ret < 0) {
2625                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2626                                 "udp_replay LPM table on socket %d\n",
2627                                 i, socketid);
2628                 }
2629
2630                 printf("LPM: Adding route 0x%08x / %d (%d)\n",
2631                         (unsigned)ipv4_udp_replay_route_array[i].ip,
2632                         ipv4_udp_replay_route_array[i].depth,
2633                         ipv4_udp_replay_route_array[i].if_out);
2634         }
2635
2636         /* create the LPM6 table */
2637         snprintf(s, sizeof(s), "IPV6_UDP_Replay_LPM_%d", socketid);
2638
2639         config.max_rules = IPV6_UDP_Replay_LPM_MAX_RULES;
2640         config.number_tbl8s = IPV6_UDP_Replay_LPM_NUMBER_TBL8S;
2641         config.flags = 0;
2642         ipv6_udp_replay_lookup_struct[socketid] = rte_lpm6_create(s, socketid,
2643                                 &config);
2644         if (ipv6_udp_replay_lookup_struct[socketid] == NULL)
2645                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay LPM table"
2646                                 " on socket %d\n", socketid);
2647
2648         /* populate the LPM table */
2649         for (i = 0; i < IPV6_UDP_Replay_NUM_ROUTES; i++) {
2650
2651                 /* skip unused ports */
2652                 if ((1 << ipv6_udp_replay_route_array[i].if_out &
2653                                 enabled_port_mask) == 0)
2654                         continue;
2655
2656                 ret = rte_lpm6_add(ipv6_udp_replay_lookup_struct[socketid],
2657                         ipv6_udp_replay_route_array[i].ip,
2658                         ipv6_udp_replay_route_array[i].depth,
2659                         ipv6_udp_replay_route_array[i].if_out);
2660
2661                 if (ret < 0) {
2662                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2663                                 "udp_replay LPM table on socket %d\n",
2664                                 i, socketid);
2665                 }
2666
2667                 printf("LPM: Adding route %s / %d (%d)\n",
2668                         "IPV6",
2669                         ipv6_udp_replay_route_array[i].depth,
2670                         ipv6_udp_replay_route_array[i].if_out);
2671         }
2672 }
2673 #endif
2674
2675
2676
2677
2678
2679
2680 /* Check the link status of all ports in up to 9s, and print them finally */
2681 static void
2682 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
2683 {
2684 #define CHECK_INTERVAL 100 /* 100ms */
2685 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2686         uint8_t portid, count, all_ports_up, print_flag = 0;
2687         struct rte_eth_link link;
2688
2689         printf("\nChecking link status");
2690         fflush(stdout);
2691         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2692                 all_ports_up = 1;
2693                 for (portid = 0; portid < port_num; portid++) {
2694                         if ((port_mask & (1 << portid)) == 0)
2695                                 continue;
2696                         memset(&link, 0, sizeof(link));
2697                         rte_eth_link_get_nowait(portid, &link);
2698                         /* print link status if flag set */
2699                         if (print_flag == 1) {
2700                                 if (link.link_status)
2701                                         printf("Port %d Link Up - speed %u "
2702                                                 "Mbps - %s\n", (uint8_t)portid,
2703                                                 (unsigned)link.link_speed,
2704                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2705                                         ("full-duplex") : ("half-duplex\n"));
2706                                 else
2707                                         printf("Port %d Link Down\n",
2708                                                 (uint8_t)portid);
2709                                 continue;
2710                         }
2711                         /* clear all_ports_up flag if any link down */
2712                         if (link.link_status == 0) {
2713                                 all_ports_up = 0;
2714                                 break;
2715                         }
2716                 }
2717                 /* after finally printing all link status, get out */
2718                 if (print_flag == 1)
2719                         break;
2720
2721                 if (all_ports_up == 0) {
2722                         printf(".");
2723                         fflush(stdout);
2724                         rte_delay_ms(CHECK_INTERVAL);
2725                 }
2726
2727                 /* set the print_flag if all ports up or timeout */
2728                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2729                         print_flag = 1;
2730                         printf("done\n");
2731                 }
2732         }
2733 }
2734
2735 int
2736 main(int argc, char **argv)
2737 {
2738         int ret;
2739         unsigned nb_ports;
2740         unsigned lcore_id;
2741         uint32_t n_tx_queue;
2742         uint8_t portid, nb_rx_queue;
2743         struct cmdline *cl;
2744         uint32_t size;
2745         struct pipeline_params *params;
2746
2747         /* init EAL */
2748         ret = rte_eal_init(argc, argv);
2749         if (ret < 0)
2750                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2751         argc -= ret;
2752         argv += ret;
2753         timer_lcore = rte_lcore_id();
2754         /* parse application arguments (after the EAL ones) */
2755         ret = parse_args(argc, argv);
2756         if (ret < 0)
2757                 rte_exit(EXIT_FAILURE, "Invalid UDP_Replay parameters\n");
2758
2759         if (check_lcore_params() < 0)
2760                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
2761
2762         ret = init_lcore_rx_queues();
2763         if (ret < 0)
2764                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2765
2766         params = rte_malloc(NULL, sizeof(*params), RTE_CACHE_LINE_SIZE);
2767         memcpy(params, &def_pipeline_params, sizeof(def_pipeline_params));
2768         lib_arp_init(params, NULL);
2769         ifm_init();
2770         nb_ports = rte_eth_dev_count();
2771         num_ports = nb_ports;
2772         gw_init(num_ports);
2773         if (nb_ports > RTE_MAX_ETHPORTS)
2774                 nb_ports = RTE_MAX_ETHPORTS;
2775
2776         if (check_port_config(nb_ports) < 0)
2777                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
2778
2779         /*
2780          *Configuring port_config_t structure for interface manager initialization
2781          */
2782         size = RTE_CACHE_LINE_ROUNDUP(sizeof(port_config_t));
2783         port_config = rte_zmalloc(NULL, (RTE_MAX_ETHPORTS * size), RTE_CACHE_LINE_SIZE);
2784         if (port_config == NULL)
2785                 rte_panic("port_config is NULL: Memory Allocation failure\n");
2786         /* initialize all ports */
2787         for (portid = 0; portid < nb_ports; portid++) {
2788                 /* skip ports that are not enabled */
2789                 if ((enabled_port_mask & (1 << portid)) == 0) {
2790                         printf("\nSkipping disabled port %d\n", portid);
2791                         num_ports--;
2792                         continue;
2793                 }
2794
2795                 /* init port */
2796                 printf("Initializing port %d ... ", portid );
2797                 fflush(stdout);
2798
2799                 nb_rx_queue = get_port_n_rx_queues(portid);
2800                 n_tx_queue = nb_rx_queue;
2801                 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
2802                         n_tx_queue = MAX_TX_QUEUE_PER_PORT;
2803
2804                 port_config[portid].port_id = portid;
2805                 port_config[portid].nrx_queue = nb_rx_queue;
2806                 port_config[portid].ntx_queue = n_tx_queue;
2807                 port_config[portid].state = 1;
2808                 port_config[portid].promisc = promiscuous_on;
2809                 port_config[portid].mempool.pool_size = MEMPOOL_SIZE;
2810                 port_config[portid].mempool.buffer_size = BUFFER_SIZE + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
2811                 port_config[portid].mempool.cache_size = CACHE_SIZE;
2812                 port_config[portid].mempool.cpu_socket_id = rte_socket_id();
2813                 memcpy (&port_config[portid].port_conf, &port_conf, sizeof(struct rte_eth_conf));
2814                 memcpy (&port_config[portid].rx_conf, &rx_conf, sizeof(struct rte_eth_rxconf));
2815                 memcpy (&port_config[portid].tx_conf, &tx_conf, sizeof(struct rte_eth_txconf));
2816
2817                 /* Enable TCP and UDP HW Checksum , when required */
2818                 //port_config[portid].tx_conf.txq_flags &=
2819                 //    ~(ETH_TXQ_FLAGS_NOXSUMTCP|ETH_TXQ_FLAGS_NOXSUMUDP);
2820
2821                 if (ifm_port_setup (portid, &port_config[portid]))
2822                        rte_panic ("Port Setup Failed:  %"PRIu32"\n", portid);
2823         }
2824
2825         check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask);
2826
2827         l3fwd_init();
2828         create_arp_table();
2829         create_nd_table();
2830         populate_lpm_routes();
2831         convert_ipstr_to_numeric();
2832         /* launch per-lcore init on every lcore */
2833         rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
2834         cl = cmdline_stdin_new(main_ctx, "Replay>");
2835         if (cl == NULL)
2836                 rte_panic("Cannot create cmdline instance\n");
2837         cmdline_interact(cl);
2838         cmdline_stdin_exit(cl);
2839         exit_loop = 0;
2840         rte_exit(0, "Bye!\n");
2841         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2842                 if (rte_eal_wait_lcore(lcore_id) < 0)
2843                         return -1;
2844         }
2845
2846         return 0;
2847 }
2848 /**********************************************************/
2849
2850 struct cmd_obj_clear_result {
2851         cmdline_fixed_string_t clear;
2852         cmdline_fixed_string_t udp_replay;
2853         cmdline_fixed_string_t stats;
2854 };
2855
2856 static void cmd_clear_udp_replay_stats_parsed(
2857         __rte_unused void *parsed_result,
2858         __rte_unused struct cmdline *cl,
2859    __attribute__((unused)) void *data)
2860 {
2861
2862         clear_stats();
2863 }
2864
2865 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_udp_replay_string =
2866         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, udp_replay, "UDP_Replay");
2867 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_clear_string =
2868         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, clear, "clear");
2869 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_stats_string =
2870         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, stats, "stats");
2871
2872 cmdline_parse_inst_t cmd_clear_udp_replay_stats = {
2873         .f = cmd_clear_udp_replay_stats_parsed,  /* function to call */
2874         .data = NULL,      /* 2nd arg of func */
2875         .help_str = "clears UDP_Replay stats for rx/tx",
2876         .tokens = {        /* token list, NULL terminated */
2877                 (void *)&cmd_clear_udp_replay_stats_udp_replay_string,
2878                 (void *)&cmd_clear_udp_replay_stats_clear_string,
2879                 (void *)&cmd_clear_udp_replay_stats_stats_string,
2880                 NULL,
2881         },
2882 };
2883 /**********************************************************/
2884 struct cmd_obj_add_result {
2885         cmdline_fixed_string_t action;
2886         cmdline_fixed_string_t name;
2887 };
2888
2889 static void cmd_udp_replay_stats_parsed(
2890         __rte_unused void *parsed_result,
2891         __rte_unused struct cmdline *cl,
2892    __attribute__((unused)) void *data)
2893 {
2894         print_l4stats();
2895 }
2896
2897 cmdline_parse_token_string_t cmd_udp_replay_stats_udp_replay_string =
2898         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, action, "UDP_Replay");
2899 cmdline_parse_token_string_t cmd_udp_replay_stats_stats_string =
2900         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, name, "stats");
2901
2902 cmdline_parse_inst_t cmd_udp_replay_stats = {
2903         .f = cmd_udp_replay_stats_parsed,  /* function to call */
2904         .data = NULL,      /* 2nd arg of func */
2905         .help_str = "UDP_Replay stats for rx/tx",
2906         .tokens = {        /* token list, NULL terminated */
2907                 (void *)&cmd_udp_replay_stats_udp_replay_string,
2908                 (void *)&cmd_udp_replay_stats_stats_string,
2909                 NULL,
2910         },
2911 };
2912 /* quit*/
2913 struct cmd_quit_result {
2914         cmdline_fixed_string_t quit;
2915 };
2916
2917 static void
2918 cmd_quit_parsed(
2919         __rte_unused void *parsed_result,
2920         struct cmdline *cl,
2921         __rte_unused void *data)
2922 {
2923         cmdline_quit(cl);
2924 }
2925
2926 static cmdline_parse_token_string_t cmd_quit_quit =
2927         TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
2928
2929 static cmdline_parse_inst_t cmd_quit = {
2930         .f = cmd_quit_parsed,
2931         .data = NULL,
2932         .help_str = "Quit",
2933         .tokens = {
2934                   (void *) &cmd_quit_quit,
2935                   NULL,
2936         },
2937 };
2938
2939 /**********************************************************/
2940 /****** CONTEXT (list of instruction) */
2941 cmdline_parse_ctx_t main_ctx[] = {
2942         (cmdline_parse_inst_t *)&cmd_udp_replay_stats,
2943         (cmdline_parse_inst_t *)&cmd_clear_udp_replay_stats,
2944         (cmdline_parse_inst_t *)&cmd_quit,
2945         NULL,
2946 };