VNFs/UDP_Replay/main.c

   1 /*
   2 // Copyright (c) 2016-2017 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //      http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 /*
  18 Correlated traffic VNF :
  19 ------------------------
  20 1. Receive UDP packet
  21 2. Modify received packet
  22      a. exchange src mac and destination mac
  23      b. exchange src ip and destination IP for both IPv4 and IPv6 cases
  24      c. exchange UDP src port and UDP destination port
  25      d. change the len of the response according to the IMIX definition (
  26         option to make traffic more realistic to emulate some IoT payloads)
  27 3. send modified packet to the port where it was received.
  28
  29 Such VNF does not need LPM and routing table implementations.
  30 As the packet modification is very minimal  and there is no memory access as the packet is stored in L3 cache the
  31 performance of the solution should be sufficient for testing the UDP NAT performance.
  32 */
  33
  34 #include <stdlib.h>
  35 #include <stdint.h>
  36 #include <inttypes.h>
  37 #include <sys/types.h>
  38 #include <string.h>
  39 #include <sys/queue.h>
  40 #include <stdarg.h>
  41 #include <errno.h>
  42 #include <getopt.h>
  43
  44 #include <rte_common.h>
  45 #include <rte_vect.h>
  46 #include <rte_byteorder.h>
  47 #include <rte_log.h>
  48 #include <rte_memory.h>
  49 #include <rte_memcpy.h>
  50 #include <rte_memzone.h>
  51 #include <rte_eal.h>
  52 #include <rte_per_lcore.h>
  53 #include <rte_launch.h>
  54 #include <rte_atomic.h>
  55 #include <rte_cycles.h>
  56 #include <rte_prefetch.h>
  57 #include <rte_lcore.h>
  58 #include <rte_per_lcore.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_interrupts.h>
  61 #include <rte_pci.h>
  62 #include <rte_random.h>
  63 #include <rte_debug.h>
  64 #include <rte_ether.h>
  65 #include <rte_ethdev.h>
  66 #include <rte_ring.h>
  67 #include <rte_mempool.h>
  68 #include <rte_mbuf.h>
  69 #include <rte_ip.h>
  70 #include <rte_tcp.h>
  71 #include <rte_udp.h>
  72 #include <rte_string_fns.h>
  73 #include <rte_version.h>
  74
  75 #include <cmdline_parse.h>
  76 #include <cmdline_parse_etheraddr.h>
  77 #include <cmdline_rdline.h>
  78 #include <cmdline_socket.h>
  79 #include <cmdline.h>
  80 #include <cmdline_parse_num.h>
  81 #include <cmdline_parse_string.h>
  82 #include <cmdline_parse_ipaddr.h>
  83
  84 #include "parse_obj_list.h"
  85
  86 #define APP_LOOKUP_EXACT_MATCH          0
  87 #define APP_LOOKUP_LPM                  1
  88 #define DO_RFC_1812_CHECKS
  89 #if 1
  90 #ifndef APP_LOOKUP_METHOD
  91 #define APP_LOOKUP_METHOD             APP_LOOKUP_EXACT_MATCH
  92 #endif
  93 #endif
  94
  95 #include <stdio.h>
  96 #include <netinet/in.h>
  97 #include <termios.h>
  98
  99 /*
 100  *  When set to zero, simple forwaring path is eanbled.
 101  *  When set to one, optimized forwarding path is enabled.
 102  *  Note that LPM optimisation path uses SSE4.1 instructions.
 103  */
 104 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__))
 105 #define ENABLE_MULTI_BUFFER_OPTIMIZE    0
 106 #else
 107 #define ENABLE_MULTI_BUFFER_OPTIMIZE    1
 108 #endif
 109
 110 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 111 #include <rte_hash.h>
 112 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 113 #include <rte_lpm.h>
 114 #include <rte_lpm6.h>
 115 #else
 116 #error "APP_LOOKUP_METHOD set to incorrect value"
 117 #endif
 118
 119 #ifndef IPv6_BYTES
 120 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
 121                        "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
 122 #define IPv6_BYTES(addr) \
 123         addr[0],  addr[1], addr[2],  addr[3], \
 124         addr[4],  addr[5], addr[6],  addr[7], \
 125         addr[8],  addr[9], addr[10], addr[11],\
 126         addr[12], addr[13],addr[14], addr[15]
 127 #endif
 128
 129
 130 #define RTE_LOGTYPE_UDP_Replay RTE_LOGTYPE_USER1
 131
 132 #define MAX_JUMBO_PKT_LEN  9600
 133
 134 #define IPV6_ADDR_LEN 16
 135
 136 #define MEMPOOL_CACHE_SIZE 256
 137
 138 /*
 139  * This expression is used to calculate the number of mbufs needed depending on user input, taking
 140  *  into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore.
 141  *  RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192
 142  */
 143
 144 #define NB_MBUF RTE_MAX (                                                                                                                                       \
 145                                 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT +                                                        \
 146                                 nb_ports*nb_lcores*MAX_PKT_BURST +                                                                                      \
 147                                 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT +                                                          \
 148                                 nb_lcores*MEMPOOL_CACHE_SIZE),                                                                                          \
 149                                 (unsigned)8192)
 150
 151 #define MAX_PKT_BURST     32
 152 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
 153
 154 /*
 155  * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send.
 156  */
 157 #define MAX_TX_BURST    (MAX_PKT_BURST / 2)
 158
 159 #define NB_SOCKETS 8
 160
 161 /* Configure how many packets ahead to prefetch, when reading packets */
 162 #define PREFETCH_OFFSET 3
 163
 164 /* Used to mark destination port as 'invalid'. */
 165 #define BAD_PORT        ((uint16_t)-1)
 166
 167 #define FWDSTEP 4
 168
 169 /*
 170  * Configurable number of RX/TX ring descriptors
 171  */
 172 #define RTE_TEST_RX_DESC_DEFAULT 128
 173 #define RTE_TEST_TX_DESC_DEFAULT 512
 174 static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
 175 static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
 176 static uint64_t rcv_pkt_count[32] = {0};
 177 static uint64_t tx_pkt_count[32] = {0};
 178
 179 /* ethernet addresses of ports */
 180 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
 181
 182 static struct ether_addr ports_eth_addr[RTE_MAX_ETHPORTS];
 183
 184 static __m128i val_eth[RTE_MAX_ETHPORTS];
 185
 186 cmdline_parse_ctx_t main_ctx[];
 187
 188 /* replace first 12B of the ethernet header. */
 189 #define MASK_ETH        0x3f
 190
 191 /* mask of enabled ports */
 192 static uint32_t enabled_port_mask = 0;
 193 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */
 194 static int numa_on = 1; /**< NUMA is enabled by default. */
 195 static int csum_on = 1; /**< NUMA is enabled by default. */
 196
 197 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 198 static int ipv6 = 0; /**< ipv6 is false by default. */
 199 #endif
 200
 201
 202 static void
 203 print_ethaddr(const char *name, const struct ether_addr *eth_addr);
 204
 205 int print_stats(void);
 206 int clear_stats(void);
 207
 208 struct mbuf_table {
 209         uint16_t len;
 210         struct rte_mbuf *m_table[MAX_PKT_BURST];
 211 };
 212
 213 struct lcore_rx_queue {
 214         uint8_t port_id;
 215         uint8_t queue_id;
 216 } __rte_cache_aligned;
 217
 218 #define MAX_RX_QUEUE_PER_LCORE 16
 219 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
 220 #define MAX_RX_QUEUE_PER_PORT 128
 221
 222 #define MAX_LCORE_PARAMS 1024
 223 struct lcore_params {
 224         uint8_t port_id;
 225         uint8_t queue_id;
 226         uint8_t lcore_id;
 227 } __rte_cache_aligned;
 228
 229 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
 230 static struct lcore_params lcore_params_array_default[] = {
 231         {0, 0, 2},
 232         {0, 1, 2},
 233         {0, 2, 2},
 234         {1, 0, 2},
 235         {1, 1, 2},
 236         {1, 2, 2},
 237         {2, 0, 2},
 238         {3, 0, 3},
 239         {3, 1, 3},
 240 };
 241
 242 static struct lcore_params * lcore_params = lcore_params_array_default;
 243 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
 244                                 sizeof(lcore_params_array_default[0]);
 245
 246 static struct rte_eth_conf port_conf = {
 247         .rxmode = {
 248                 .mq_mode = ETH_MQ_RX_RSS,
 249                 .max_rx_pkt_len = ETHER_MAX_LEN,
 250                 .split_hdr_size = 0,
 251                 .header_split   = 0, /**< Header Split disabled */
 252                 .hw_ip_checksum = 1, /**< IP checksum offload enabled */
 253                 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
 254                 .jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
 255                 .hw_strip_crc   = 0, /**< CRC stripped by hardware */
 256         },
 257         .rx_adv_conf = {
 258                 .rss_conf = {
 259                         .rss_key = NULL,
 260                         .rss_hf = ETH_RSS_IP,
 261                 },
 262         },
 263         .txmode = {
 264                 .mq_mode = ETH_MQ_TX_NONE,
 265         },
 266 };
 267
 268 static struct rte_mempool * pktmbuf_pool[NB_SOCKETS];
 269
 270 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 271
 272 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 273 #include <rte_hash_crc.h>
 274 #define DEFAULT_HASH_FUNC       rte_hash_crc
 275 #else
 276 #include <rte_jhash.h>
 277 #define DEFAULT_HASH_FUNC       rte_jhash
 278 #endif
 279
 280 struct ipv4_5tuple {
 281         uint32_t ip_dst;
 282         uint32_t ip_src;
 283         uint16_t port_dst;
 284         uint16_t port_src;
 285         uint8_t  proto;
 286 } __attribute__((__packed__));
 287
 288 union ipv4_5tuple_host {
 289         struct {
 290                 uint8_t  pad0;
 291                 uint8_t  proto;
 292                 uint16_t pad1;
 293                 uint32_t ip_src;
 294                 uint32_t ip_dst;
 295                 uint16_t port_src;
 296                 uint16_t port_dst;
 297         };
 298         __m128i xmm;
 299 };
 300
 301 #define XMM_NUM_IN_IPV6_5TUPLE 3
 302
 303 struct ipv6_5tuple {
 304         uint8_t  ip_dst[IPV6_ADDR_LEN];
 305         uint8_t  ip_src[IPV6_ADDR_LEN];
 306         uint16_t port_dst;
 307         uint16_t port_src;
 308         uint8_t  proto;
 309 } __attribute__((__packed__));
 310
 311 union ipv6_5tuple_host {
 312         struct {
 313                 uint16_t pad0;
 314                 uint8_t  proto;
 315                 uint8_t  pad1;
 316                 uint8_t  ip_src[IPV6_ADDR_LEN];
 317                 uint8_t  ip_dst[IPV6_ADDR_LEN];
 318                 uint16_t port_src;
 319                 uint16_t port_dst;
 320                 uint64_t reserve;
 321         };
 322         __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE];
 323 };
 324
 325 struct ipv4_udp_replay_route {
 326         struct ipv4_5tuple key;
 327         uint8_t if_out;
 328 };
 329
 330 struct ipv6_udp_replay_route {
 331         struct ipv6_5tuple key;
 332         uint8_t if_out;
 333 };
 334
 335 static struct ipv4_udp_replay_route ipv4_udp_replay_route_array[] = {
 336         {{IPv4(101,0,0,0), IPv4(100,10,0,1),  101, 11, IPPROTO_TCP}, 0},
 337         {{IPv4(201,0,0,0), IPv4(200,20,0,1),  102, 12, IPPROTO_TCP}, 1},
 338         {{IPv4(111,0,0,0), IPv4(100,30,0,1),  101, 11, IPPROTO_TCP}, 2},
 339         {{IPv4(211,0,0,0), IPv4(200,40,0,1),  102, 12, IPPROTO_TCP}, 3},
 340 };
 341
 342 static struct ipv6_udp_replay_route ipv6_udp_replay_route_array[] = {
 343         {{
 344         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 345         {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 346         101, 11, IPPROTO_TCP}, 0},
 347
 348         {{
 349         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 350         {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 351         102, 12, IPPROTO_TCP}, 1},
 352
 353         {{
 354         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 355         {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 356         101, 11, IPPROTO_TCP}, 2},
 357
 358         {{
 359         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
 360         {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
 361         102, 12, IPPROTO_TCP}, 3},
 362 };
 363
 364 typedef struct rte_hash lookup_struct_t;
 365 static lookup_struct_t *ipv4_udp_replay_lookup_struct[NB_SOCKETS];
 366 static lookup_struct_t *ipv6_udp_replay_lookup_struct[NB_SOCKETS];
 367
 368 #ifdef RTE_ARCH_X86_64
 369 /* default to 4 million hash entries (approx) */
 370 #define UDP_Replay_HASH_ENTRIES         1024*1024*4
 371 #else
 372 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */
 373 #define UDP_Replay_HASH_ENTRIES         1024*1024*1
 374 #endif
 375 #define HASH_ENTRY_NUMBER_DEFAULT       4
 376
 377 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
 378
 379 static inline uint32_t
 380 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
 381         uint32_t init_val)
 382 {
 383         const union ipv4_5tuple_host *k;
 384         uint32_t t;
 385         const uint32_t *p;
 386
 387         k = data;
 388         t = k->proto;
 389         p = (const uint32_t *)&k->port_src;
 390
 391 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 392         init_val = rte_hash_crc_4byte(t, init_val);
 393         init_val = rte_hash_crc_4byte(k->ip_src, init_val);
 394         init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
 395         init_val = rte_hash_crc_4byte(*p, init_val);
 396 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 397         init_val = rte_jhash_1word(t, init_val);
 398         init_val = rte_jhash_1word(k->ip_src, init_val);
 399         init_val = rte_jhash_1word(k->ip_dst, init_val);
 400         init_val = rte_jhash_1word(*p, init_val);
 401 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 402         return (init_val);
 403 }
 404
 405 static inline uint32_t
 406 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val)
 407 {
 408         const union ipv6_5tuple_host *k;
 409         uint32_t t;
 410         const uint32_t *p;
 411 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 412         const uint32_t  *ip_src0, *ip_src1, *ip_src2, *ip_src3;
 413         const uint32_t  *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
 414 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 415
 416         k = data;
 417         t = k->proto;
 418         p = (const uint32_t *)&k->port_src;
 419
 420 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
 421         ip_src0 = (const uint32_t *) k->ip_src;
 422         ip_src1 = (const uint32_t *)(k->ip_src+4);
 423         ip_src2 = (const uint32_t *)(k->ip_src+8);
 424         ip_src3 = (const uint32_t *)(k->ip_src+12);
 425         ip_dst0 = (const uint32_t *) k->ip_dst;
 426         ip_dst1 = (const uint32_t *)(k->ip_dst+4);
 427         ip_dst2 = (const uint32_t *)(k->ip_dst+8);
 428         ip_dst3 = (const uint32_t *)(k->ip_dst+12);
 429         init_val = rte_hash_crc_4byte(t, init_val);
 430         init_val = rte_hash_crc_4byte(*ip_src0, init_val);
 431         init_val = rte_hash_crc_4byte(*ip_src1, init_val);
 432         init_val = rte_hash_crc_4byte(*ip_src2, init_val);
 433         init_val = rte_hash_crc_4byte(*ip_src3, init_val);
 434         init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
 435         init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
 436         init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
 437         init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
 438         init_val = rte_hash_crc_4byte(*p, init_val);
 439 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 440         init_val = rte_jhash_1word(t, init_val);
 441         init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 442         init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
 443         init_val = rte_jhash_1word(*p, init_val);
 444 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
 445         return (init_val);
 446 }
 447
 448 #define IPV4_UDP_Replay_NUM_ROUTES \
 449         (sizeof(ipv4_udp_replay_route_array) / sizeof(ipv4_udp_replay_route_array[0]))
 450
 451 #define IPV6_UDP_Replay_NUM_ROUTES \
 452         (sizeof(ipv6_udp_replay_route_array) / sizeof(ipv6_udp_replay_route_array[0]))
 453
 454 static uint8_t ipv4_udp_replay_out_if[UDP_Replay_HASH_ENTRIES] __rte_cache_aligned;
 455 static uint8_t ipv6_udp_replay_out_if[UDP_Replay_HASH_ENTRIES] __rte_cache_aligned;
 456
 457 #endif
 458
 459 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 460 struct ipv4_udp_replay_route {
 461         uint32_t ip;
 462         uint8_t  depth;
 463         uint8_t  if_out;
 464 };
 465
 466 struct ipv6_udp_replay_route {
 467         uint8_t ip[16];
 468         uint8_t  depth;
 469         uint8_t  if_out;
 470 };
 471
 472 static struct ipv4_udp_replay_route ipv4_udp_replay_route_array[] = {
 473         {IPv4(1,1,1,0), 24, 0},
 474         {IPv4(2,1,1,0), 24, 1},
 475         {IPv4(3,1,1,0), 24, 2},
 476         {IPv4(4,1,1,0), 24, 3},
 477         {IPv4(5,1,1,0), 24, 4},
 478         {IPv4(6,1,1,0), 24, 5},
 479         {IPv4(7,1,1,0), 24, 6},
 480         {IPv4(8,1,1,0), 24, 7},
 481 };
 482
 483 static struct ipv6_udp_replay_route ipv6_udp_replay_route_array[] = {
 484         {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0},
 485         {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1},
 486         {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2},
 487         {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3},
 488         {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4},
 489         {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5},
 490         {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6},
 491         {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7},
 492 };
 493
 494 #define IPV4_UDP_Replay_NUM_ROUTES \
 495         (sizeof(ipv4_udp_replay_route_array) / sizeof(ipv4_udp_replay_route_array[0]))
 496 #define IPV6_UDP_Replay_NUM_ROUTES \
 497         (sizeof(ipv6_udp_replay_route_array) / sizeof(ipv6_udp_replay_route_array[0]))
 498
 499 #define IPV4_UDP_Replay_LPM_MAX_RULES         1024
 500 #define IPV6_UDP_Replay_LPM_MAX_RULES         1024
 501 #define IPV6_UDP_Replay_LPM_NUMBER_TBL8S (1 << 16)
 502
 503 typedef struct rte_lpm lookup_struct_t;
 504 typedef struct rte_lpm6 lookup6_struct_t;
 505 static lookup_struct_t *ipv4_udp_replay_lookup_struct[NB_SOCKETS];
 506 static lookup6_struct_t *ipv6_udp_replay_lookup_struct[NB_SOCKETS];
 507 #endif
 508
 509 struct lcore_conf {
 510         uint16_t n_rx_queue;
 511         struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
 512         uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
 513         struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
 514         lookup_struct_t * ipv4_lookup_struct;
 515 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 516         lookup6_struct_t * ipv6_lookup_struct;
 517 #else
 518         lookup_struct_t * ipv6_lookup_struct;
 519 #endif
 520 } __rte_cache_aligned;
 521
 522 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
 523
 524 /* Send burst of packets on an output interface */
 525 static inline int
 526 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port)
 527 {
 528         struct rte_mbuf **m_table;
 529         int ret;
 530         uint16_t queueid;
 531
 532         queueid = qconf->tx_queue_id[port];
 533         m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
 534
 535         ret = rte_eth_tx_burst(port, queueid, m_table, n);
 536         if (unlikely(ret < n)) {
 537                 do {
 538                         rte_pktmbuf_free(m_table[ret]);
 539                 } while (++ret < n);
 540         }
 541         /*Tx Pkt count*/
 542         tx_pkt_count[port] += ret;
 543         return 0;
 544 }
 545
 546 /* Enqueue a single packet, and send burst if queue is filled */
 547 static inline int
 548 send_single_packet(struct rte_mbuf *m, uint8_t port)
 549 {
 550         uint32_t lcore_id;
 551         uint16_t len;
 552         struct lcore_conf *qconf;
 553
 554         lcore_id = rte_lcore_id();
 555
 556         qconf = &lcore_conf[lcore_id];
 557         len = qconf->tx_mbufs[port].len;
 558         qconf->tx_mbufs[port].m_table[len] = m;
 559         len++;
 560
 561         /* enough pkts to be sent */
 562         if (unlikely(len == MAX_PKT_BURST)) {
 563                 send_burst(qconf, MAX_PKT_BURST, port);
 564                 len = 0;
 565         }
 566
 567         qconf->tx_mbufs[port].len = len;
 568         return 0;
 569 }
 570
 571 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 572 static inline __attribute__((always_inline)) void
 573 send_packetsx4(struct lcore_conf *qconf, uint8_t port,
 574         struct rte_mbuf *m[], uint32_t num)
 575 {
 576         uint32_t len, j, n;
 577
 578         len = qconf->tx_mbufs[port].len;
 579
 580         /*
 581          * If TX buffer for that queue is empty, and we have enough packets,
 582          * then send them straightway.
 583          */
 584         if (num >= MAX_TX_BURST && len == 0) {
 585                 n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num);
 586                 if (unlikely(n < num)) {
 587                         do {
 588                                 rte_pktmbuf_free(m[n]);
 589                         } while (++n < num);
 590                 }
 591                 return;
 592         }
 593
 594         /*
 595          * Put packets into TX buffer for that queue.
 596          */
 597
 598         n = len + num;
 599         n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num;
 600
 601         j = 0;
 602         switch (n % FWDSTEP) {
 603         while (j < n) {
 604         case 0:
 605                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 606                 j++;
 607         case 3:
 608                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 609                 j++;
 610         case 2:
 611                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 612                 j++;
 613         case 1:
 614                 qconf->tx_mbufs[port].m_table[len + j] = m[j];
 615                 j++;
 616         }
 617         }
 618
 619         len += n;
 620
 621         /* enough pkts to be sent */
 622         if (unlikely(len == MAX_PKT_BURST)) {
 623
 624                 send_burst(qconf, MAX_PKT_BURST, port);
 625
 626                 /* copy rest of the packets into the TX buffer. */
 627                 len = num - n;
 628                 j = 0;
 629                 switch (len % FWDSTEP) {
 630                 while (j < len) {
 631                 case 0:
 632                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 633                         j++;
 634                 case 3:
 635                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 636                         j++;
 637                 case 2:
 638                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 639                         j++;
 640                 case 1:
 641                         qconf->tx_mbufs[port].m_table[j] = m[n + j];
 642                         j++;
 643                 }
 644                 }
 645         }
 646
 647         qconf->tx_mbufs[port].len = len;
 648 }
 649 #endif /* APP_LOOKUP_LPM */
 650
 651 #ifdef DO_RFC_1812_CHECKS
 652 static inline int
 653 is_valid_ipv4_pkt(struct ipv4_hdr *pkt, uint32_t link_len)
 654 {
 655         /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
 656         /*
 657          * 1. The packet length reported by the Link Layer must be large
 658          * enough to hold the minimum length legal IP datagram (20 bytes).
 659          */
 660         if (link_len < sizeof(struct ipv4_hdr))
 661                 return -1;
 662
 663         /* 2. The IP checksum must be correct. */
 664         /* this is checked in H/W */
 665
 666         /*
 667          * 3. The IP version number must be 4. If the version number is not 4
 668          * then the packet may be another version of IP, such as IPng or
 669          * ST-II.
 670          */
 671         if (((pkt->version_ihl) >> 4) != 4)
 672                 return -3;
 673         /*
 674          * 4. The IP header length field must be large enough to hold the
 675          * minimum length legal IP datagram (20 bytes = 5 words).
 676          */
 677         if ((pkt->version_ihl & 0xf) < 5)
 678                 return -4;
 679
 680         /*
 681          * 5. The IP total length field must be large enough to hold the IP
 682          * datagram header, whose length is specified in the IP header length
 683          * field.
 684          */
 685         if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr))
 686                 return -5;
 687
 688         return 0;
 689 }
 690 #endif
 691
 692 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
 693
 694 static __m128i mask0;
 695 static __m128i mask1;
 696 static __m128i mask2;
 697 static inline uint8_t
 698 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_udp_replay_lookup_struct)
 699 {
 700         int ret = 0;
 701         union ipv4_5tuple_host key;
 702
 703         ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live);
 704         __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr));
 705         /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */
 706         key.xmm = _mm_and_si128(data, mask0);
 707         /* Find destination port */
 708         ret = rte_hash_lookup(ipv4_udp_replay_lookup_struct, (const void *)&key);
 709         return (uint8_t)((ret < 0)? portid : ipv4_udp_replay_out_if[ret]);
 710 }
 711
 712 static inline uint8_t
 713 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup_struct_t * ipv6_udp_replay_lookup_struct)
 714 {
 715         int ret = 0;
 716         union ipv6_5tuple_host key;
 717
 718         ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len);
 719         __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr));
 720         __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)));
 721         __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i)));
 722         /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
 723         key.xmm[0] = _mm_and_si128(data0, mask1);
 724         /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */
 725         key.xmm[1] = data1;
 726         /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */
 727         key.xmm[2] = _mm_and_si128(data2, mask2);
 728
 729         /* Find destination port */
 730         ret = rte_hash_lookup(ipv6_udp_replay_lookup_struct, (const void *)&key);
 731         return (uint8_t)((ret < 0)? portid : ipv6_udp_replay_out_if[ret]);
 732 }
 733 #endif
 734
 735 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
 736
 737 static inline uint8_t
 738 get_ipv4_dst_port(void *ipv4_hdr,  uint8_t portid, lookup_struct_t * ipv4_udp_replay_lookup_struct)
 739 {
 740         uint8_t next_hop;
 741
 742         return (uint8_t) ((rte_lpm_lookup(ipv4_udp_replay_lookup_struct,
 743                 rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr),
 744                 &next_hop) == 0) ? next_hop : portid);
 745 }
 746
 747 static inline uint8_t
 748 get_ipv6_dst_port(void *ipv6_hdr,  uint8_t portid, lookup6_struct_t * ipv6_udp_replay_lookup_struct)
 749 {
 750         uint8_t next_hop;
 751         return (uint8_t) ((rte_lpm6_lookup(ipv6_udp_replay_lookup_struct,
 752                         ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)?
 753                         next_hop : portid);
 754 }
 755 #endif
 756
 757 static inline void udp_replay_simple_replay(struct rte_mbuf *m, uint8_t portid,
 758         struct lcore_conf *qconf)  __attribute__((unused));
 759
 760 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \
 761         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
 762
 763 #define MASK_ALL_PKTS    0xff
 764 #define EXCLUDE_1ST_PKT 0xfe
 765 #define EXCLUDE_2ND_PKT 0xfd
 766 #define EXCLUDE_3RD_PKT 0xfb
 767 #define EXCLUDE_4TH_PKT 0xf7
 768 #define EXCLUDE_5TH_PKT 0xef
 769 #define EXCLUDE_6TH_PKT 0xdf
 770 #define EXCLUDE_7TH_PKT 0xbf
 771 #define EXCLUDE_8TH_PKT 0x7f
 772
 773 static inline void
 774 simple_ipv4_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
 775 {
 776         struct ether_hdr *eth_hdr[8];
 777         struct ether_hdr tmp;
 778         struct ipv4_hdr *ipv4_hdr[8];
 779         struct udp_hdr *udp_hdr[8];
 780
 781         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
 782         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
 783         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
 784         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
 785         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
 786         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
 787         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
 788         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
 789
 790
 791         memset(&tmp,0,sizeof (struct ether_hdr));
 792
 793         int i;
 794         for(i=0;i<8;i++)
 795         {
 796
 797         ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
 798         ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
 799         ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
 800         }
 801
 802         /* Handle IPv4 headers.*/
 803         ipv4_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv4_hdr *,
 804                                               sizeof(struct ether_hdr));
 805         ipv4_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv4_hdr *,
 806                                               sizeof(struct ether_hdr));
 807         ipv4_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv4_hdr *,
 808                                               sizeof(struct ether_hdr));
 809         ipv4_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv4_hdr *,
 810                                               sizeof(struct ether_hdr));
 811         ipv4_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv4_hdr *,
 812                                               sizeof(struct ether_hdr));
 813         ipv4_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv4_hdr *,
 814                                               sizeof(struct ether_hdr));
 815         ipv4_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv4_hdr *,
 816                                               sizeof(struct ether_hdr));
 817         ipv4_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv4_hdr *,
 818                                               sizeof(struct ether_hdr));
 819         struct ipv4_hdr temp_ipv4;
 820         for(i=0;i<8;i++)
 821         {
 822         temp_ipv4.dst_addr = ipv4_hdr[i]->dst_addr;
 823         ipv4_hdr[i]->dst_addr = ipv4_hdr[i]->src_addr;
 824         ipv4_hdr[i]->src_addr = temp_ipv4.dst_addr;
 825         }
 826
 827         /* Handle UDP headers.*/
 828         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
 829                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 830
 831         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
 832                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 833         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
 834                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 835         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
 836                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 837         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
 838                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 839         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
 840                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 841         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
 842                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 843         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
 844                          sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
 845        /*1) memcpy or assignment.*/
 846
 847         struct udp_hdr temp_udp;
 848         for(i=0;i<8;i++)
 849         {
 850         temp_udp.dst_port = udp_hdr[i]->dst_port;
 851         udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
 852         udp_hdr[i]->src_port = temp_udp.dst_port;
 853         }
 854 #ifdef DO_RFC_1812_CHECKS
 855         /* Check to make sure the packet is valid (RFC1812) */
 856         uint8_t valid_mask = MASK_ALL_PKTS;
 857         if (is_valid_ipv4_pkt(ipv4_hdr[0], m[0]->pkt_len) < 0) {
 858                 rte_pktmbuf_free(m[0]);
 859                 valid_mask &= EXCLUDE_1ST_PKT;
 860         }
 861         if (is_valid_ipv4_pkt(ipv4_hdr[1], m[1]->pkt_len) < 0) {
 862                 rte_pktmbuf_free(m[1]);
 863                 valid_mask &= EXCLUDE_2ND_PKT;
 864         }
 865         if (is_valid_ipv4_pkt(ipv4_hdr[2], m[2]->pkt_len) < 0) {
 866                 rte_pktmbuf_free(m[2]);
 867                 valid_mask &= EXCLUDE_3RD_PKT;
 868         }
 869         if (is_valid_ipv4_pkt(ipv4_hdr[3], m[3]->pkt_len) < 0) {
 870                 rte_pktmbuf_free(m[3]);
 871                 valid_mask &= EXCLUDE_4TH_PKT;
 872         }
 873         if (is_valid_ipv4_pkt(ipv4_hdr[4], m[4]->pkt_len) < 0) {
 874                 rte_pktmbuf_free(m[4]);
 875                 valid_mask &= EXCLUDE_5TH_PKT;
 876         }
 877         if (is_valid_ipv4_pkt(ipv4_hdr[5], m[5]->pkt_len) < 0) {
 878                 rte_pktmbuf_free(m[5]);
 879                 valid_mask &= EXCLUDE_6TH_PKT;
 880         }
 881         if (is_valid_ipv4_pkt(ipv4_hdr[6], m[6]->pkt_len) < 0) {
 882                 rte_pktmbuf_free(m[6]);
 883                 valid_mask &= EXCLUDE_7TH_PKT;
 884         }
 885         if (is_valid_ipv4_pkt(ipv4_hdr[7], m[7]->pkt_len) < 0) {
 886                 rte_pktmbuf_free(m[7]);
 887                 valid_mask &= EXCLUDE_8TH_PKT;
 888         }
 889         if (unlikely(valid_mask != MASK_ALL_PKTS)) {
 890                 if (valid_mask == 0){
 891                         return;
 892                 } else {
 893                         uint8_t i = 0;
 894                         for (i = 0; i < 8; i++) {
 895                                 if ((0x1 << i) & valid_mask) {
 896                                         udp_replay_simple_replay(m[i], portid, qconf);
 897                                 }
 898                         }
 899                         return;
 900                 }
 901         }
 902 #endif // End of #ifdef DO_RFC_1812_CHECKS
 903
 904 #ifdef DO_RFC_1812_CHECKS
 905         /* Update time to live and header checksum */
 906         --(ipv4_hdr[0]->time_to_live);
 907         --(ipv4_hdr[1]->time_to_live);
 908         --(ipv4_hdr[2]->time_to_live);
 909         --(ipv4_hdr[3]->time_to_live);
 910         ++(ipv4_hdr[0]->hdr_checksum);
 911         ++(ipv4_hdr[1]->hdr_checksum);
 912         ++(ipv4_hdr[2]->hdr_checksum);
 913         ++(ipv4_hdr[3]->hdr_checksum);
 914         --(ipv4_hdr[4]->time_to_live);
 915         --(ipv4_hdr[5]->time_to_live);
 916         --(ipv4_hdr[6]->time_to_live);
 917         --(ipv4_hdr[7]->time_to_live);
 918         ++(ipv4_hdr[4]->hdr_checksum);
 919         ++(ipv4_hdr[5]->hdr_checksum);
 920         ++(ipv4_hdr[6]->hdr_checksum);
 921         ++(ipv4_hdr[7]->hdr_checksum);
 922 #endif
 923
 924         send_single_packet(m[0],portid );
 925         send_single_packet(m[1],portid );
 926         send_single_packet(m[2],portid );
 927         send_single_packet(m[3],portid);
 928         send_single_packet(m[4],portid);
 929         send_single_packet(m[5],portid);
 930         send_single_packet(m[6],portid);
 931         send_single_packet(m[7],portid);
 932
 933 }
 934
 935 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1,
 936                                  union ipv6_5tuple_host * key)
 937 {
 938         __m128i tmpdata0 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
 939         __m128i tmpdata1 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i)));
 940         __m128i tmpdata2 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i) + sizeof(__m128i)));
 941         key->xmm[0] = _mm_and_si128(tmpdata0, mask0);
 942         key->xmm[1] = tmpdata1;
 943         key->xmm[2] = _mm_and_si128(tmpdata2, mask1);
 944         return;
 945 }
 946
 947 static inline void
 948 simple_ipv6_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
 949 {
 950         struct ether_hdr *eth_hdr[8],tmp;
 951         __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[8], temp_ipv6;
 952         int32_t ret[8];
 953         union ipv6_5tuple_host key[8];
 954         struct udp_hdr *udp_hdr[8];
 955
 956         eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
 957         eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
 958         eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
 959         eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
 960         eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
 961         eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
 962         eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
 963         eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
 964
 965         memset(&tmp,0,sizeof (struct ether_hdr));
 966
 967         int i;
 968         for(i=0;i<8;i++)
 969         {
 970             ether_addr_copy(&eth_hdr[i]->s_addr, &tmp.s_addr);
 971             ether_addr_copy(&eth_hdr[i]->d_addr, &eth_hdr[i]->s_addr);
 972             ether_addr_copy(&tmp.s_addr, &eth_hdr[i]->d_addr);
 973         }
 974         /* Handle IPv6 headers.*/
 975         ipv6_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv6_hdr *,
 976                                               sizeof(struct ether_hdr));
 977         ipv6_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv6_hdr *,
 978                                               sizeof(struct ether_hdr));
 979         ipv6_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv6_hdr *,
 980                                               sizeof(struct ether_hdr));
 981         ipv6_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv6_hdr *,
 982                                               sizeof(struct ether_hdr));
 983         ipv6_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv6_hdr *,
 984                                               sizeof(struct ether_hdr));
 985         ipv6_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv6_hdr *,
 986                                               sizeof(struct ether_hdr));
 987         ipv6_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv6_hdr *,
 988                                               sizeof(struct ether_hdr));
 989         ipv6_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv6_hdr *,
 990                                               sizeof(struct ether_hdr));
 991         for(i=0;i<8;i++)
 992         {
 993            memcpy(temp_ipv6.dst_addr,ipv6_hdr[i]->dst_addr,16);
 994            memcpy(ipv6_hdr[i]->dst_addr,ipv6_hdr[i]->src_addr,16);
 995            memcpy(ipv6_hdr[i]->src_addr,temp_ipv6.dst_addr,16);
 996         }
 997
 998         /* Handle UDP headers.*/
 999         udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1000                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1001
1002         udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1003                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1004         udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1005                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1006         udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1007                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1008         udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1009                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1010         udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1011                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1012         udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1013                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1014         udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1015                          sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1016        /*1) memcpy or assignment.*/
1017
1018         struct udp_hdr temp_udp;
1019         for(i=0;i<8;i++)
1020         {
1021            temp_udp.dst_port = udp_hdr[i]->dst_port;
1022            udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1023            udp_hdr[i]->src_port = temp_udp.dst_port;
1024         }
1025         const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3],
1026                                 &key[4], &key[5], &key[6], &key[7]};
1027 #if RTE_VERSION < 0x100b0000
1028         rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1029 #else
1030         rte_hash_lookup_bulk(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1031 #endif
1032         send_single_packet(m[0],portid);
1033         send_single_packet(m[1],portid);
1034         send_single_packet(m[2],portid);
1035         send_single_packet(m[3],portid);
1036         send_single_packet(m[4],portid);
1037         send_single_packet(m[5],portid);
1038         send_single_packet(m[6],portid);
1039         send_single_packet(m[7],portid);
1040
1041 }
1042 #endif /* APP_LOOKUP_METHOD */
1043
1044 static inline __attribute__((always_inline)) void
1045 udp_replay_simple_replay(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf)
1046 {
1047         struct ether_hdr *eth_hdr,tmp;
1048         struct ipv4_hdr *ipv4_hdr,temp_ipv4;
1049         uint8_t dst_port;
1050         struct udp_hdr *udp_hdr,temp_udp;
1051
1052         eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1053         ether_addr_copy(&eth_hdr->s_addr, &tmp.s_addr);
1054         ether_addr_copy(&eth_hdr->d_addr, &eth_hdr->s_addr);
1055         ether_addr_copy(&tmp.s_addr, &eth_hdr->d_addr);
1056         struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1057
1058         if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1059                 /* Handle IPv4 headers.*/
1060                 ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
1061                                                    sizeof(struct ether_hdr));
1062         temp_ipv4.dst_addr = ipv4_hdr->dst_addr;
1063         ipv4_hdr->dst_addr = ipv4_hdr->src_addr;
1064         ipv4_hdr->src_addr = temp_ipv4.dst_addr;
1065 #ifdef DO_RFC_1812_CHECKS
1066                 /* Check to make sure the packet is valid (RFC1812) */
1067                 if (is_valid_ipv4_pkt(ipv4_hdr, m->pkt_len) < 0) {
1068                         rte_pktmbuf_free(m);
1069                         return;
1070                 }
1071 #endif
1072
1073                  dst_port = get_ipv4_dst_port(ipv4_hdr, portid,
1074                         qconf->ipv4_lookup_struct);
1075                         dst_port = portid;
1076
1077 #ifdef DO_RFC_1812_CHECKS
1078                 /* Update time to live and header checksum */
1079                 --(ipv4_hdr->time_to_live);
1080                 ++(ipv4_hdr->hdr_checksum);
1081 #endif
1082         /* Handle UDP headers.*/
1083         udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1084                          (sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr)));
1085
1086         /*Swapping Src and Dst Port*/
1087         temp_udp.dst_port = udp_hdr->dst_port;
1088         udp_hdr->dst_port = udp_hdr->src_port;
1089         udp_hdr->src_port = temp_udp.dst_port;
1090
1091                 send_single_packet(m, dst_port);
1092         } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1093                 /* Handle IPv6 headers.*/
1094                 struct ipv6_hdr *ipv6_hdr,temp_ipv6;
1095
1096                 ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
1097                                                    sizeof(struct ether_hdr));
1098
1099         /*Swapping of Src and Dst IP address*/
1100         memcpy(temp_ipv6.dst_addr,ipv6_hdr->dst_addr,16);
1101         memcpy(ipv6_hdr->dst_addr,ipv6_hdr->src_addr,16);
1102         memcpy(ipv6_hdr->src_addr,temp_ipv6.dst_addr,16);
1103
1104
1105                 dst_port = get_ipv6_dst_port(ipv6_hdr, portid, qconf->ipv6_lookup_struct);
1106         /* Handle UDP headers.*/
1107         udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1108                          (sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr)));
1109         /*Swapping Src and Dst Port*/
1110         temp_udp.dst_port = udp_hdr->dst_port;
1111         udp_hdr->dst_port = udp_hdr->src_port;
1112         udp_hdr->src_port = temp_udp.dst_port;
1113                 send_single_packet(m, portid);
1114         } else
1115                 /* Free the mbuf that contains non-IPV4/IPV6 packet */
1116                 rte_pktmbuf_free(m);
1117 }
1118
1119 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1120         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1121 #ifdef DO_RFC_1812_CHECKS
1122
1123 #define IPV4_MIN_VER_IHL        0x45
1124 #define IPV4_MAX_VER_IHL        0x4f
1125 #define IPV4_MAX_VER_IHL_DIFF   (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL)
1126
1127 /* Minimum value of IPV4 total length (20B) in network byte order. */
1128 #define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8)
1129
1130 /*
1131  * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2:
1132  * - The IP version number must be 4.
1133  * - The IP header length field must be large enough to hold the
1134  *    minimum length legal IP datagram (20 bytes = 5 words).
1135  * - The IP total length field must be large enough to hold the IP
1136  *   datagram header, whose length is specified in the IP header length
1137  *   field.
1138  * If we encounter invalid IPV4 packet, then set destination port for it
1139  * to BAD_PORT value.
1140  */
1141 static inline __attribute__((always_inline)) void
1142 rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype)
1143 {
1144         uint8_t ihl;
1145
1146         if (RTE_ETH_IS_IPV4_HDR(ptype)) {
1147                 ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL;
1148
1149                 ipv4_hdr->time_to_live--;
1150                 ipv4_hdr->hdr_checksum++;
1151
1152                 if (ihl > IPV4_MAX_VER_IHL_DIFF ||
1153                                 ((uint8_t)ipv4_hdr->total_length == 0 &&
1154                                 ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) {
1155                         dp[0] = BAD_PORT;
1156                 }
1157         }
1158 }
1159
1160 #else
1161 #define rfc1812_process(mb, dp) do { } while (0)
1162 #endif /* DO_RFC_1812_CHECKS */
1163 #endif /* APP_LOOKUP_LPM && ENABLE_MULTI_BUFFER_OPTIMIZE */
1164
1165
1166 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1167         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1168
1169 static inline __attribute__((always_inline)) uint16_t
1170 get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt,
1171         uint32_t dst_ipv4, uint8_t portid)
1172 {
1173         uint8_t next_hop;
1174         struct ipv6_hdr *ipv6_hdr;
1175         struct ether_hdr *eth_hdr;
1176         struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1177
1178         if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1179                 if (rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4,
1180                                 &next_hop) != 0)
1181                         next_hop = portid;
1182         } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1183                 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1184                 ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1);
1185                 if (rte_lpm6_lookup(qconf->ipv6_lookup_struct,
1186                                 ipv6_hdr->dst_addr, &next_hop) != 0)
1187                         next_hop = portid;
1188         } else {
1189                 next_hop = portid;
1190         }
1191
1192         return next_hop;
1193 }
1194
1195 static inline void
1196 process_packet(struct lcore_conf *qconf, struct rte_mbuf *pkt,
1197         uint16_t *dst_port, uint8_t portid)
1198 {
1199         struct ether_hdr *eth_hdr;
1200         struct ipv4_hdr *ipv4_hdr;
1201         uint32_t dst_ipv4;
1202         uint16_t dp;
1203         __m128i te, ve;
1204
1205         eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1206         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1207
1208         /*Add swap*/
1209         dst_ipv4 = ipv4_hdr->dst_addr;
1210         dst_ipv4 = rte_be_to_cpu_32(dst_ipv4);
1211
1212         /*Changing the dp to incoming port*/
1213         dp = get_dst_port(qconf, pkt, dst_ipv4, portid);
1214         dp = portid;
1215
1216         te = _mm_loadu_si128((__m128i *)eth_hdr);
1217         ve = val_eth[dp];
1218
1219         dst_port[0] = dp;
1220         rfc1812_process(ipv4_hdr, dst_port, pkt->packet_type);
1221
1222         te =  _mm_blend_epi16(te, ve, MASK_ETH);
1223         _mm_storeu_si128((__m128i *)eth_hdr, te);
1224 }
1225 /* Wont be using the following fucntion*/
1226
1227 /*
1228  * Read packet_type and destination IPV4 addresses from 4 mbufs.
1229  */
1230 static inline void
1231 processx4_step1(struct rte_mbuf *pkt[FWDSTEP],
1232                 __m128i *dip,
1233                 uint32_t *ipv4_flag)
1234 {
1235         struct ipv4_hdr *ipv4_hdr;
1236         struct ether_hdr *eth_hdr;
1237         uint32_t x0, x1, x2, x3;
1238
1239         eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *);
1240         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1241         x0 = ipv4_hdr->dst_addr;
1242         ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4;
1243
1244         eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *);
1245         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1246         x1 = ipv4_hdr->dst_addr;
1247         ipv4_flag[0] &= pkt[1]->packet_type;
1248
1249         eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *);
1250         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1251         x2 = ipv4_hdr->dst_addr;
1252         ipv4_flag[0] &= pkt[2]->packet_type;
1253
1254         eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *);
1255         ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1256         x3 = ipv4_hdr->dst_addr;
1257         ipv4_flag[0] &= pkt[3]->packet_type;
1258
1259         dip[0] = _mm_set_epi32(x3, x2, x1, x0);
1260 }
1261
1262 /*
1263  * Lookup into LPM for destination port.
1264  * If lookup fails, use incoming port (portid) as destination port.
1265  */
1266 static inline void
1267 processx4_step2(const struct lcore_conf *qconf,
1268                 __m128i dip,
1269                 uint32_t ipv4_flag,
1270                 uint8_t portid,
1271                 struct rte_mbuf *pkt[FWDSTEP],
1272                 uint16_t dprt[FWDSTEP])
1273 {
1274         rte_xmm_t dst;
1275         const  __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11,
1276                                                 4, 5, 6, 7, 0, 1, 2, 3);
1277
1278         /* Byte swap 4 IPV4 addresses. */
1279         dip = _mm_shuffle_epi8(dip, bswap_mask);
1280
1281         /* if all 4 packets are IPV4. */
1282         if (likely(ipv4_flag)) {
1283                 rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid);
1284         } else {
1285                 dst.x = dip;
1286                 dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid);
1287                 dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid);
1288                 dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid);
1289                 dprt[3] = get_dst_port(qconf, pkt[3], dst.u32[3], portid);
1290         }
1291 }
1292
1293 /*
1294  * Update source and destination MAC addresses in the ethernet header.
1295  * Perform RFC1812 checks and updates for IPV4 packets.
1296  */
1297 static inline void
1298 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP])
1299 {
1300         __m128i te[FWDSTEP];
1301         __m128i ve[FWDSTEP];
1302         __m128i *p[FWDSTEP];
1303
1304         p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *);
1305         p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *);
1306         p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *);
1307         p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *);
1308
1309         ve[0] = val_eth[dst_port[0]];
1310         te[0] = _mm_loadu_si128(p[0]);
1311
1312         ve[1] = val_eth[dst_port[1]];
1313         te[1] = _mm_loadu_si128(p[1]);
1314
1315         ve[2] = val_eth[dst_port[2]];
1316         te[2] = _mm_loadu_si128(p[2]);
1317
1318         ve[3] = val_eth[dst_port[3]];
1319         te[3] = _mm_loadu_si128(p[3]);
1320
1321         /* Update first 12 bytes, keep rest bytes intact. */
1322         te[0] =  _mm_blend_epi16(te[0], ve[0], MASK_ETH);
1323         te[1] =  _mm_blend_epi16(te[1], ve[1], MASK_ETH);
1324         te[2] =  _mm_blend_epi16(te[2], ve[2], MASK_ETH);
1325         te[3] =  _mm_blend_epi16(te[3], ve[3], MASK_ETH);
1326
1327         _mm_storeu_si128(p[0], te[0]);
1328         _mm_storeu_si128(p[1], te[1]);
1329         _mm_storeu_si128(p[2], te[2]);
1330         _mm_storeu_si128(p[3], te[3]);
1331
1332         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1),
1333                 &dst_port[0], pkt[0]->packet_type);
1334         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1),
1335                 &dst_port[1], pkt[1]->packet_type);
1336         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1),
1337                 &dst_port[2], pkt[2]->packet_type);
1338         rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1),
1339                 &dst_port[3], pkt[3]->packet_type);
1340 }
1341
1342 /*
1343  * We group consecutive packets with the same destionation port into one burst.
1344  * To avoid extra latency this is done together with some other packet
1345  * processing, but after we made a final decision about packet's destination.
1346  * To do this we maintain:
1347  * pnum - array of number of consecutive packets with the same dest port for
1348  * each packet in the input burst.
1349  * lp - pointer to the last updated element in the pnum.
1350  * dlp - dest port value lp corresponds to.
1351  */
1352
1353 #define GRPSZ   (1 << FWDSTEP)
1354 #define GRPMSK  (GRPSZ - 1)
1355
1356 #define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx)  do { \
1357         if (likely((dlp) == (dcp)[(idx)])) {         \
1358                 (lp)[0]++;                           \
1359         } else {                                     \
1360                 (dlp) = (dcp)[idx];                  \
1361                 (lp) = (pn) + (idx);                 \
1362                 (lp)[0] = 1;                         \
1363         }                                            \
1364 } while (0)
1365
1366 /*
1367  * Group consecutive packets with the same destination port in bursts of 4.
1368  * Suppose we have array of destionation ports:
1369  * dst_port[] = {a, b, c, d,, e, ... }
1370  * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
1371  * We doing 4 comparisions at once and the result is 4 bit mask.
1372  * This mask is used as an index into prebuild array of pnum values.
1373  */
1374 static inline uint16_t *
1375 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2)
1376 {
1377         static const struct {
1378                 uint64_t pnum; /* prebuild 4 values for pnum[]. */
1379                 int32_t  idx;  /* index for new last updated elemnet. */
1380                 uint16_t lpv;  /* add value to the last updated element. */
1381         } gptbl[GRPSZ] = {
1382         {
1383                 /* 0: a != b, b != c, c != d, d != e */
1384                 .pnum = UINT64_C(0x0001000100010001),
1385                 .idx = 4,
1386                 .lpv = 0,
1387         },
1388         {
1389                 /* 1: a == b, b != c, c != d, d != e */
1390                 .pnum = UINT64_C(0x0001000100010002),
1391                 .idx = 4,
1392                 .lpv = 1,
1393         },
1394         {
1395                 /* 2: a != b, b == c, c != d, d != e */
1396                 .pnum = UINT64_C(0x0001000100020001),
1397                 .idx = 4,
1398                 .lpv = 0,
1399         },
1400         {
1401                 /* 3: a == b, b == c, c != d, d != e */
1402                 .pnum = UINT64_C(0x0001000100020003),
1403                 .idx = 4,
1404                 .lpv = 2,
1405         },
1406         {
1407                 /* 4: a != b, b != c, c == d, d != e */
1408                 .pnum = UINT64_C(0x0001000200010001),
1409                 .idx = 4,
1410                 .lpv = 0,
1411         },
1412         {
1413                 /* 5: a == b, b != c, c == d, d != e */
1414                 .pnum = UINT64_C(0x0001000200010002),
1415                 .idx = 4,
1416                 .lpv = 1,
1417         },
1418         {
1419                 /* 6: a != b, b == c, c == d, d != e */
1420                 .pnum = UINT64_C(0x0001000200030001),
1421                 .idx = 4,
1422                 .lpv = 0,
1423         },
1424         {
1425                 /* 7: a == b, b == c, c == d, d != e */
1426                 .pnum = UINT64_C(0x0001000200030004),
1427                 .idx = 4,
1428                 .lpv = 3,
1429         },
1430         {
1431                 /* 8: a != b, b != c, c != d, d == e */
1432                 .pnum = UINT64_C(0x0002000100010001),
1433                 .idx = 3,
1434                 .lpv = 0,
1435         },
1436         {
1437                 /* 9: a == b, b != c, c != d, d == e */
1438                 .pnum = UINT64_C(0x0002000100010002),
1439                 .idx = 3,
1440                 .lpv = 1,
1441         },
1442         {
1443                 /* 0xa: a != b, b == c, c != d, d == e */
1444                 .pnum = UINT64_C(0x0002000100020001),
1445                 .idx = 3,
1446                 .lpv = 0,
1447         },
1448         {
1449                 /* 0xb: a == b, b == c, c != d, d == e */
1450                 .pnum = UINT64_C(0x0002000100020003),
1451                 .idx = 3,
1452                 .lpv = 2,
1453         },
1454         {
1455                 /* 0xc: a != b, b != c, c == d, d == e */
1456                 .pnum = UINT64_C(0x0002000300010001),
1457                 .idx = 2,
1458                 .lpv = 0,
1459         },
1460         {
1461                 /* 0xd: a == b, b != c, c == d, d == e */
1462                 .pnum = UINT64_C(0x0002000300010002),
1463                 .idx = 2,
1464                 .lpv = 1,
1465         },
1466         {
1467                 /* 0xe: a != b, b == c, c == d, d == e */
1468                 .pnum = UINT64_C(0x0002000300040001),
1469                 .idx = 1,
1470                 .lpv = 0,
1471         },
1472         {
1473                 /* 0xf: a == b, b == c, c == d, d == e */
1474                 .pnum = UINT64_C(0x0002000300040005),
1475                 .idx = 0,
1476                 .lpv = 4,
1477         },
1478         };
1479
1480         union {
1481                 uint16_t u16[FWDSTEP + 1];
1482                 uint64_t u64;
1483         } *pnum = (void *)pn;
1484
1485         int32_t v;
1486
1487         dp1 = _mm_cmpeq_epi16(dp1, dp2);
1488         dp1 = _mm_unpacklo_epi16(dp1, dp1);
1489         v = _mm_movemask_ps((__m128)dp1);
1490
1491         /* update last port counter. */
1492         lp[0] += gptbl[v].lpv;
1493
1494         /* if dest port value has changed. */
1495         if (v != GRPMSK) {
1496                 lp = pnum->u16 + gptbl[v].idx;
1497                 lp[0] = 1;
1498                 pnum->u64 = gptbl[v].pnum;
1499         }
1500
1501         return lp;
1502 }
1503
1504 #endif /* APP_LOOKUP_METHOD */
1505
1506 /* main processing loop */
1507 static int
1508 main_loop(__attribute__((unused)) void *dummy)
1509 {
1510         struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
1511         unsigned lcore_id;
1512         uint64_t prev_tsc, diff_tsc, cur_tsc;
1513         int i, j, nb_rx;
1514         uint8_t portid, queueid;
1515         struct lcore_conf *qconf;
1516         const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
1517                 US_PER_S * BURST_TX_DRAIN_US;
1518
1519 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1520         (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1521         int32_t k;
1522         uint16_t dlp;
1523         uint16_t *lp;
1524         uint16_t dst_port[MAX_PKT_BURST];
1525         __m128i dip[MAX_PKT_BURST / FWDSTEP];
1526         uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP];
1527         uint16_t pnum[MAX_PKT_BURST + 1];
1528 #endif
1529
1530         prev_tsc = 0;
1531
1532         lcore_id = rte_lcore_id();
1533         qconf = &lcore_conf[lcore_id];
1534
1535         if (qconf->n_rx_queue == 0) {
1536                 RTE_LOG(INFO, UDP_Replay, "lcore %u has nothing to do\n", lcore_id);
1537                 return 0;
1538         }
1539
1540         RTE_LOG(INFO, UDP_Replay, "entering main loop on lcore %u\n", lcore_id);
1541
1542         for (i = 0; i < qconf->n_rx_queue; i++) {
1543
1544                 portid = qconf->rx_queue_list[i].port_id;
1545                 queueid = qconf->rx_queue_list[i].queue_id;
1546                 RTE_LOG(INFO, UDP_Replay, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id,
1547                         portid, queueid);
1548         }
1549
1550         while (1) {
1551
1552                 cur_tsc = rte_rdtsc();
1553
1554                 /*
1555                  * TX burst queue drain
1556                  */
1557                 diff_tsc = cur_tsc - prev_tsc;
1558                 if (unlikely(diff_tsc > drain_tsc)) {
1559
1560                         /*
1561                          * This could be optimized (use queueid instead of
1562                          * portid), but it is not called so often
1563                          */
1564                         for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
1565                                 if (qconf->tx_mbufs[portid].len == 0)
1566                                         continue;
1567                                 send_burst(qconf,
1568                                         qconf->tx_mbufs[portid].len,
1569                                         portid);
1570                                 qconf->tx_mbufs[portid].len = 0;
1571                         }
1572
1573                         prev_tsc = cur_tsc;
1574                 }
1575
1576                 /*
1577                  * Read packet from RX queues
1578                  */
1579                 for (i = 0; i < qconf->n_rx_queue; ++i) {
1580                         portid = qconf->rx_queue_list[i].port_id;
1581                         queueid = qconf->rx_queue_list[i].queue_id;
1582                         nb_rx = rte_eth_rx_burst(portid, queueid, pkts_burst,
1583                                 MAX_PKT_BURST);
1584                         if(nb_rx)
1585                             rcv_pkt_count[portid] += nb_rx;
1586                         if (nb_rx == 0)
1587                                 continue;
1588
1589 #if (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)
1590 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1591                         {
1592                                 /*
1593                                  * Send nb_rx - nb_rx%8 packets
1594                                  * in groups of 8.
1595                                  */
1596                                 int32_t n = RTE_ALIGN_FLOOR(nb_rx, 8);
1597                                 for (j = 0; j < n; j += 8) {
1598                                         uint32_t pkt_type =
1599                                                 pkts_burst[j]->packet_type &
1600                                                 pkts_burst[j+1]->packet_type &
1601                                                 pkts_burst[j+2]->packet_type &
1602                                                 pkts_burst[j+3]->packet_type &
1603                                                 pkts_burst[j+4]->packet_type &
1604                                                 pkts_burst[j+5]->packet_type &
1605                                                 pkts_burst[j+6]->packet_type &
1606                                                 pkts_burst[j+7]->packet_type;
1607                                         if (pkt_type & RTE_PTYPE_L3_IPV4) {
1608                                                 simple_ipv4_replay_8pkts(
1609                                                 &pkts_burst[j], portid, qconf);
1610                                         } else if (pkt_type &
1611                                                 RTE_PTYPE_L3_IPV6) {
1612                                                 simple_ipv6_replay_8pkts(&pkts_burst[j],
1613                                                                         portid, qconf);
1614                                         } else {
1615                                                 udp_replay_simple_replay(pkts_burst[j],
1616                                                                         portid, qconf);
1617                                                 udp_replay_simple_replay(pkts_burst[j+1],
1618                                                                         portid, qconf);
1619                                                 udp_replay_simple_replay(pkts_burst[j+2],
1620                                                                         portid, qconf);
1621                                                 udp_replay_simple_replay(pkts_burst[j+3],
1622                                                                         portid, qconf);
1623                                                 udp_replay_simple_replay(pkts_burst[j+4],
1624                                                                         portid, qconf);
1625                                                 udp_replay_simple_replay(pkts_burst[j+5],
1626                                                                         portid, qconf);
1627                                                 udp_replay_simple_replay(pkts_burst[j+6],
1628                                                                         portid, qconf);
1629                                                 udp_replay_simple_replay(pkts_burst[j+7],
1630                                                                         portid, qconf);
1631                                         }
1632                                 }
1633                                 for (; j < nb_rx ; j++) {
1634                                         udp_replay_simple_replay(pkts_burst[j],
1635                                                                 portid, qconf);
1636                                 }
1637                         }
1638 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1639
1640                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1641                         for (j = 0; j != k; j += FWDSTEP) {
1642                                 processx4_step1(&pkts_burst[j],
1643                                         &dip[j / FWDSTEP],
1644                                         &ipv4_flag[j / FWDSTEP]);
1645                         }
1646
1647                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1648                         for (j = 0; j != k; j += FWDSTEP) {
1649                                 processx4_step2(qconf, dip[j / FWDSTEP],
1650                                         ipv4_flag[j / FWDSTEP], portid,
1651                                         &pkts_burst[j], &dst_port[j]);
1652                         }
1653
1654                         /*
1655                          * Finish packet processing and group consecutive
1656                          * packets with the same destination port.
1657                          */
1658                         k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1659                         if (k != 0) {
1660                                 __m128i dp1, dp2;
1661
1662                                 lp = pnum;
1663                                 lp[0] = 1;
1664
1665                                 processx4_step3(pkts_burst, dst_port);
1666
1667                                 /* dp1: <d[0], d[1], d[2], d[3], ... > */
1668                                 dp1 = _mm_loadu_si128((__m128i *)dst_port);
1669
1670                                 for (j = FWDSTEP; j != k; j += FWDSTEP) {
1671                                         processx4_step3(&pkts_burst[j],
1672                                                 &dst_port[j]);
1673
1674                                         /*
1675                                          * dp2:
1676                                          * <d[j-3], d[j-2], d[j-1], d[j], ... >
1677                                          */
1678                                         dp2 = _mm_loadu_si128((__m128i *)
1679                                                 &dst_port[j - FWDSTEP + 1]);
1680                                         lp  = port_groupx4(&pnum[j - FWDSTEP],
1681                                                 lp, dp1, dp2);
1682
1683                                         /*
1684                                          * dp1:
1685                                          * <d[j], d[j+1], d[j+2], d[j+3], ... >
1686                                          */
1687                                         dp1 = _mm_srli_si128(dp2,
1688                                                 (FWDSTEP - 1) *
1689                                                 sizeof(dst_port[0]));
1690                                 }
1691
1692                                 /*
1693                                  * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
1694                                  */
1695                                 dp2 = _mm_shufflelo_epi16(dp1, 0xf9);
1696                                 lp  = port_groupx4(&pnum[j - FWDSTEP], lp,
1697                                         dp1, dp2);
1698
1699                                 /*
1700                                  * remove values added by the last repeated
1701                                  * dst port.
1702                                  */
1703                                 lp[0]--;
1704                                 dlp = dst_port[j - 1];
1705                         } else {
1706                                 /* set dlp and lp to the never used values. */
1707                                 dlp = BAD_PORT - 1;
1708                                 lp = pnum + MAX_PKT_BURST;
1709                         }
1710
1711                         /* Process up to last 3 packets one by one. */
1712                         switch (nb_rx % FWDSTEP) {
1713                         case 3:
1714                                 process_packet(qconf, pkts_burst[j],
1715                                         dst_port + j, portid);
1716                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1717                                 j++;
1718                         case 2:
1719                                 process_packet(qconf, pkts_burst[j],
1720                                         dst_port + j, portid);
1721                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1722                                 j++;
1723                         case 1:
1724                                 process_packet(qconf, pkts_burst[j],
1725                                         dst_port + j, portid);
1726                                 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
1727                                 j++;
1728                         }
1729
1730                         /*
1731                          * Send packets out, through destination port.
1732                          * Consecuteve pacekts with the same destination port
1733                          * are already grouped together.
1734                          * If destination port for the packet equals BAD_PORT,
1735                          * then free the packet without sending it out.
1736                          */
1737                         for (j = 0; j < nb_rx; j += k) {
1738
1739                                 int32_t m;
1740                                 uint16_t pn;
1741
1742                                 pn = dst_port[j];
1743                                 k = pnum[j];
1744
1745                                 if (likely(pn != BAD_PORT)) {
1746                                         send_packetsx4(qconf, pn,
1747                                                 pkts_burst + j, k);
1748                                 } else {
1749                                         for (m = j; m != j + k; m++)
1750                                                 rte_pktmbuf_free(pkts_burst[m]);
1751                                 }
1752                         }
1753
1754 #endif /* APP_LOOKUP_METHOD */
1755 #else /* ENABLE_MULTI_BUFFER_OPTIMIZE == 0 */
1756
1757                         /* Prefetch first packets */
1758                         for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
1759                                 rte_prefetch0(rte_pktmbuf_mtod(
1760                                                 pkts_burst[j], void *));
1761                         }
1762
1763                         /* Prefetch and forward already prefetched packets */
1764                         for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
1765                                 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
1766                                                 j + PREFETCH_OFFSET], void *));
1767                                 udp_replay_simple_replay(pkts_burst[j], portid,
1768                                         qconf);
1769                         }
1770
1771                         /* Forward remaining prefetched packets */
1772                         for (; j < nb_rx; j++) {
1773                                 udp_replay_simple_replay(pkts_burst[j], portid,
1774                                         qconf);
1775                         }
1776 #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */
1777                 }
1778         }
1779 }
1780
1781 /* display usage */
1782 int
1783 print_stats(void)
1784 {
1785         unsigned portid;
1786         uint16_t i;
1787         printf ("\n");
1788         printf ("UDP_Replay stats:\n");
1789         printf ("--------------\n");
1790         printf (" Port          Rx Packet               Tx Packet               Rx Pkt Drop             Tx Pkt Drop \n");
1791         for (i = 0; i < nb_lcore_params; ++i) {
1792                 portid = lcore_params[i].port_id;
1793                 printf (" %u            %lu             %lu                     0                       0", portid, rcv_pkt_count[(uint64_t)portid], tx_pkt_count[(uint64_t)portid]);
1794                 printf ("\n");
1795         }
1796         printf ("\n");
1797         return 0;
1798
1799 }
1800
1801 int
1802 clear_stats(void)
1803 {
1804         uint64_t i;
1805         for (i = 0; i < 32; i++) {
1806             rcv_pkt_count[i] = 0;
1807             tx_pkt_count[i] = 0;
1808         }
1809         print_stats();
1810         return 0;
1811 }
1812
1813 static int
1814 check_lcore_params(void)
1815 {
1816         uint8_t queue, lcore;
1817         uint16_t i;
1818         int socketid;
1819
1820         for (i = 0; i < nb_lcore_params; ++i) {
1821                 queue = lcore_params[i].queue_id;
1822                 if (queue >= MAX_RX_QUEUE_PER_PORT) {
1823                         printf("invalid queue number: %hhu\n", queue);
1824                         return -1;
1825                 }
1826                 lcore = lcore_params[i].lcore_id;
1827                 if (!rte_lcore_is_enabled(lcore)) {
1828                         printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
1829                         return -1;
1830                 }
1831                 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
1832                         (numa_on == 0)) {
1833                         printf("warning: lcore %hhu is on socket %d with numa off \n",
1834                                 lcore, socketid);
1835                 }
1836         }
1837         return 0;
1838 }
1839
1840 static int
1841 check_port_config(const unsigned nb_ports)
1842 {
1843         unsigned portid;
1844         uint16_t i;
1845
1846         for (i = 0; i < nb_lcore_params; ++i) {
1847                 portid = lcore_params[i].port_id;
1848                 if ((enabled_port_mask & (1 << portid)) == 0) {
1849                         printf("port %u is not enabled in port mask\n", portid);
1850                         return -1;
1851                 }
1852                 if (portid >= nb_ports) {
1853                         printf("port %u is not present on the board\n", portid);
1854                         return -1;
1855                 }
1856         }
1857         return 0;
1858 }
1859
1860 static uint8_t
1861 get_port_n_rx_queues(const uint8_t port)
1862 {
1863         int queue = -1;
1864         uint16_t i;
1865
1866         for (i = 0; i < nb_lcore_params; ++i) {
1867                 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue)
1868                         queue = lcore_params[i].queue_id;
1869         }
1870         return (uint8_t)(++queue);
1871 }
1872
1873 static int
1874 init_lcore_rx_queues(void)
1875 {
1876         uint16_t i, nb_rx_queue;
1877         uint8_t lcore;
1878
1879         for (i = 0; i < nb_lcore_params; ++i) {
1880                 lcore = lcore_params[i].lcore_id;
1881                 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
1882                 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
1883                         printf("error: too many queues (%u) for lcore: %u\n",
1884                                 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
1885                         return -1;
1886                 } else {
1887                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
1888                                 lcore_params[i].port_id;
1889                         lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
1890                                 lcore_params[i].queue_id;
1891                         lcore_conf[lcore].n_rx_queue++;
1892                 }
1893         }
1894         return 0;
1895 }
1896
1897 /* display usage */
1898 static void
1899 print_usage(const char *prgname)
1900 {
1901         printf ("%s [EAL options] -- -p PORTMASK -P"
1902                 "  [--config (port,queue,lcore)[,(port,queue,lcore]]"
1903                 "  [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
1904                 "  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
1905                 "  -P : enable promiscuous mode\n"
1906                 "  --config (port,queue,lcore): rx queues configuration\n"
1907                 "  --eth-dest=X,MM:MM:MM:MM:MM:MM: optional, ethernet destination for port X\n"
1908                 "  --no-numa: optional, disable numa awareness\n"
1909                 "  --no-hw-csum: optional, disable hw ip checksum\n"
1910                 "  --ipv6: optional, specify it if running ipv6 packets\n"
1911                 "  --enable-jumbo: enable jumbo frame"
1912                 " which max packet len is PKTLEN in decimal (64-9600)\n"
1913                 "  --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n",
1914                 prgname);
1915 }
1916
1917 static int parse_max_pkt_len(const char *pktlen)
1918 {
1919         char *end = NULL;
1920         unsigned long len;
1921
1922         /* parse decimal string */
1923         len = strtoul(pktlen, &end, 10);
1924         if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
1925                 return -1;
1926
1927         if (len == 0)
1928                 return -1;
1929
1930         return len;
1931 }
1932
1933 static int
1934 parse_portmask(const char *portmask)
1935 {
1936         char *end = NULL;
1937         unsigned long pm;
1938
1939         /* parse hexadecimal string */
1940         pm = strtoul(portmask, &end, 16);
1941         if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
1942                 return -1;
1943
1944         if (pm == 0)
1945                 return -1;
1946
1947         return pm;
1948 }
1949
1950 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1951 static int
1952 parse_hash_entry_number(const char *hash_entry_num)
1953 {
1954         char *end = NULL;
1955         unsigned long hash_en;
1956         /* parse hexadecimal string */
1957         hash_en = strtoul(hash_entry_num, &end, 16);
1958         if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
1959                 return -1;
1960
1961         if (hash_en == 0)
1962                 return -1;
1963
1964         return hash_en;
1965 }
1966 #endif
1967
1968 static int
1969 parse_config(const char *q_arg)
1970 {
1971         char s[256];
1972         const char *p, *p0 = q_arg;
1973         char *end;
1974         enum fieldnames {
1975                 FLD_PORT = 0,
1976                 FLD_QUEUE,
1977                 FLD_LCORE,
1978                 _NUM_FLD
1979         };
1980         unsigned long int_fld[_NUM_FLD];
1981         char *str_fld[_NUM_FLD];
1982         int i;
1983         unsigned size;
1984
1985         nb_lcore_params = 0;
1986
1987         while ((p = strchr(p0,'(')) != NULL) {
1988                 ++p;
1989                 if((p0 = strchr(p,')')) == NULL)
1990                         return -1;
1991
1992                 size = p0 - p;
1993                 if(size >= sizeof(s))
1994                         return -1;
1995
1996                 snprintf(s, sizeof(s), "%.*s", size, p);
1997                 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
1998                         return -1;
1999                 for (i = 0; i < _NUM_FLD; i++){
2000                         errno = 0;
2001                         int_fld[i] = strtoul(str_fld[i], &end, 0);
2002                         if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
2003                                 return -1;
2004                 }
2005                 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
2006                         printf("exceeded max number of lcore params: %hu\n",
2007                                 nb_lcore_params);
2008                         return -1;
2009                 }
2010                 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT];
2011                 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE];
2012                 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE];
2013                 ++nb_lcore_params;
2014         }
2015         lcore_params = lcore_params_array;
2016         return 0;
2017 }
2018
2019 static void
2020 parse_eth_dest(const char *optarg)
2021 {
2022         uint8_t portid;
2023         char *port_end;
2024         uint8_t c, *dest, peer_addr[6];
2025
2026         errno = 0;
2027         portid = strtoul(optarg, &port_end, 10);
2028         if (errno != 0 || port_end == optarg || *port_end++ != ',')
2029                 rte_exit(EXIT_FAILURE,
2030                 "Invalid eth-dest: %s", optarg);
2031         if (portid >= RTE_MAX_ETHPORTS)
2032                 rte_exit(EXIT_FAILURE,
2033                 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
2034                 portid, RTE_MAX_ETHPORTS);
2035
2036         if (cmdline_parse_etheraddr(NULL, port_end,
2037                 &peer_addr, sizeof(peer_addr)) < 0)
2038                 rte_exit(EXIT_FAILURE,
2039                 "Invalid ethernet address: %s\n",
2040                 port_end);
2041         dest = (uint8_t *)&dest_eth_addr[portid];
2042         for (c = 0; c < 6; c++)
2043                 dest[c] = peer_addr[c];
2044         *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
2045 }
2046
2047 #define CMD_LINE_OPT_CONFIG "config"
2048 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
2049 #define CMD_LINE_OPT_NO_NUMA "no-numa"
2050 #define CMD_LINE_OPT_NO_HW_CSUM "no-hw-csum"
2051 #define CMD_LINE_OPT_IPV6 "ipv6"
2052 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
2053 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
2054
2055 /* Parse the argument given in the command line of the application */
2056 static int
2057 parse_args(int argc, char **argv)
2058 {
2059         int opt, ret;
2060         char **argvopt;
2061         int option_index;
2062         char *prgname = argv[0];
2063         static struct option lgopts[] = {
2064                 {CMD_LINE_OPT_CONFIG, 1, 0, 0},
2065                 {CMD_LINE_OPT_ETH_DEST, 1, 0, 0},
2066                 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0},
2067                 {CMD_LINE_OPT_NO_HW_CSUM, 0, 0, 0},
2068                 {CMD_LINE_OPT_IPV6, 0, 0, 0},
2069                 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0},
2070                 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0},
2071                 {NULL, 0, 0, 0}
2072         };
2073
2074         argvopt = argv;
2075
2076         while ((opt = getopt_long(argc, argvopt, "p:P",
2077                                 lgopts, &option_index)) != EOF) {
2078
2079                 switch (opt) {
2080                 /* portmask */
2081                 case 'p':
2082                         enabled_port_mask = parse_portmask(optarg);
2083                         if (enabled_port_mask == 0) {
2084                                 printf("invalid portmask\n");
2085                                 print_usage(prgname);
2086                                 return -1;
2087                         }
2088                         break;
2089                 case 'P':
2090                         printf("Promiscuous mode selected\n");
2091                         promiscuous_on = 1;
2092                         break;
2093
2094                 /* long options */
2095                 case 0:
2096                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG,
2097                                 sizeof (CMD_LINE_OPT_CONFIG))) {
2098                                 ret = parse_config(optarg);
2099                                 if (ret) {
2100                                         printf("invalid config\n");
2101                                         print_usage(prgname);
2102                                         return -1;
2103                                 }
2104                         }
2105
2106                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ETH_DEST,
2107                                 sizeof(CMD_LINE_OPT_ETH_DEST))) {
2108                                         parse_eth_dest(optarg);
2109                         }
2110
2111                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA,
2112                                 sizeof(CMD_LINE_OPT_NO_NUMA))) {
2113                                 printf("numa is disabled \n");
2114                                 numa_on = 0;
2115                         }
2116
2117                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_HW_CSUM,
2118                                 sizeof(CMD_LINE_OPT_NO_HW_CSUM))) {
2119                                 printf("numa is hw ip checksum \n");
2120                                 port_conf.rxmode.hw_ip_checksum = 0;
2121                                 csum_on = 0;
2122                         }
2123
2124 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2125                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6,
2126                                 sizeof(CMD_LINE_OPT_IPV6))) {
2127                                 printf("ipv6 is specified \n");
2128                                 ipv6 = 1;
2129                         }
2130 #endif
2131
2132                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO,
2133                                 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) {
2134                                 struct option lenopts = {"max-pkt-len", required_argument, 0, 0};
2135
2136                                 printf("jumbo frame is enabled - disabling simple TX path\n");
2137                                 port_conf.rxmode.jumbo_frame = 1;
2138
2139                                 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */
2140                                 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) {
2141                                         ret = parse_max_pkt_len(optarg);
2142                                         if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){
2143                                                 printf("invalid packet length\n");
2144                                                 print_usage(prgname);
2145                                                 return -1;
2146                                         }
2147                                         port_conf.rxmode.max_rx_pkt_len = ret;
2148                                 }
2149                                 printf("set jumbo frame max packet length to %u\n",
2150                                                 (unsigned int)port_conf.rxmode.max_rx_pkt_len);
2151                         }
2152 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2153                         if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM,
2154                                 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) {
2155                                 ret = parse_hash_entry_number(optarg);
2156                                 if ((ret > 0) && (ret <= UDP_Replay_HASH_ENTRIES)) {
2157                                         hash_entry_number = ret;
2158                                 } else {
2159                                         printf("invalid hash entry number\n");
2160                                         print_usage(prgname);
2161                                         return -1;
2162                                 }
2163                         }
2164 #endif
2165                         break;
2166
2167                 default:
2168                         print_usage(prgname);
2169                         return -1;
2170                 }
2171         }
2172
2173         if (optind >= 0)
2174                 argv[optind-1] = prgname;
2175
2176         ret = optind-1;
2177         optind = 0; /* reset getopt lib */
2178         return ret;
2179 }
2180
2181 static void
2182 print_ethaddr(const char *name, const struct ether_addr *eth_addr)
2183 {
2184         char buf[ETHER_ADDR_FMT_SIZE];
2185         ether_format_addr(buf, ETHER_ADDR_FMT_SIZE, eth_addr);
2186         printf("%s%s", name, buf);
2187 }
2188
2189 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2190
2191 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1,
2192                 union ipv4_5tuple_host* key2)
2193 {
2194         key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
2195         key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
2196         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2197         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2198         key2->proto = key1->proto;
2199         key2->pad0 = 0;
2200         key2->pad1 = 0;
2201         return;
2202 }
2203
2204 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1,
2205                 union ipv6_5tuple_host* key2)
2206 {
2207         uint32_t i;
2208         for (i = 0; i < 16; i++)
2209         {
2210                 key2->ip_dst[i] = key1->ip_dst[i];
2211                 key2->ip_src[i] = key1->ip_src[i];
2212         }
2213         key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2214         key2->port_src = rte_cpu_to_be_16(key1->port_src);
2215         key2->proto = key1->proto;
2216         key2->pad0 = 0;
2217         key2->pad1 = 0;
2218         key2->reserve = 0;
2219         return;
2220 }
2221
2222 #define BYTE_VALUE_MAX 256
2223 #define ALL_32_BITS 0xffffffff
2224 #define BIT_8_TO_15 0x0000ff00
2225 static inline void
2226 populate_ipv4_few_flow_into_table(const struct rte_hash* h)
2227 {
2228         uint32_t i;
2229         int32_t ret;
2230         uint32_t array_len = sizeof(ipv4_udp_replay_route_array)/sizeof(ipv4_udp_replay_route_array[0]);
2231
2232         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2233         for (i = 0; i < array_len; i++) {
2234                 struct ipv4_udp_replay_route  entry;
2235                 union ipv4_5tuple_host newkey;
2236                 entry = ipv4_udp_replay_route_array[i];
2237                 convert_ipv4_5tuple(&entry.key, &newkey);
2238                 ret = rte_hash_add_key (h,(void *) &newkey);
2239                 if (ret < 0) {
2240                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2241                                 " to the udp_replay hash.\n", i);
2242                 }
2243                 ipv4_udp_replay_out_if[ret] = entry.if_out;
2244         }
2245         printf("Hash: Adding 0x%" PRIx32 " keys\n", array_len);
2246 }
2247
2248 #define BIT_16_TO_23 0x00ff0000
2249 static inline void
2250 populate_ipv6_few_flow_into_table(const struct rte_hash* h)
2251 {
2252         uint32_t i;
2253         int32_t ret;
2254         uint32_t array_len = sizeof(ipv6_udp_replay_route_array)/sizeof(ipv6_udp_replay_route_array[0]);
2255
2256         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2257         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2258         for (i = 0; i < array_len; i++) {
2259                 struct ipv6_udp_replay_route entry;
2260                 union ipv6_5tuple_host newkey;
2261                 entry = ipv6_udp_replay_route_array[i];
2262                 convert_ipv6_5tuple(&entry.key, &newkey);
2263                 ret = rte_hash_add_key (h, (void *) &newkey);
2264                 if (ret < 0) {
2265                         rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2266                                 " to the udp_replay hash.\n", i);
2267                 }
2268                 ipv6_udp_replay_out_if[ret] = entry.if_out;
2269         }
2270         printf("Hash: Adding 0x%" PRIx32 "keys\n", array_len);
2271 }
2272
2273 #define NUMBER_PORT_USED 4
2274 static inline void
2275 populate_ipv4_many_flow_into_table(const struct rte_hash* h,
2276                 unsigned int nr_flow)
2277 {
2278         unsigned i;
2279         mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2280         for (i = 0; i < nr_flow; i++) {
2281                 struct ipv4_udp_replay_route entry;
2282                 union ipv4_5tuple_host newkey;
2283                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2284                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2285                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2286                 /* Create the ipv4 exact match flow */
2287                 memset(&entry, 0, sizeof(entry));
2288                 switch (i & (NUMBER_PORT_USED -1)) {
2289                 case 0:
2290                         entry = ipv4_udp_replay_route_array[0];
2291                         entry.key.ip_dst = IPv4(101,c,b,a);
2292                         break;
2293                 case 1:
2294                         entry = ipv4_udp_replay_route_array[1];
2295                         entry.key.ip_dst = IPv4(201,c,b,a);
2296                         break;
2297                 case 2:
2298                         entry = ipv4_udp_replay_route_array[2];
2299                         entry.key.ip_dst = IPv4(111,c,b,a);
2300                         break;
2301                 case 3:
2302                         entry = ipv4_udp_replay_route_array[3];
2303                         entry.key.ip_dst = IPv4(211,c,b,a);
2304                         break;
2305                 };
2306                 convert_ipv4_5tuple(&entry.key, &newkey);
2307                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2308                 if (ret < 0) {
2309                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2310                 }
2311                 ipv4_udp_replay_out_if[ret] = (uint8_t) entry.if_out;
2312
2313         }
2314         printf("Hash: Adding 0x%x keys\n", nr_flow);
2315 }
2316
2317 static inline void
2318 populate_ipv6_many_flow_into_table(const struct rte_hash* h,
2319                 unsigned int nr_flow)
2320 {
2321         unsigned i;
2322         mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2323         mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2324         for (i = 0; i < nr_flow; i++) {
2325                 struct ipv6_udp_replay_route entry;
2326                 union ipv6_5tuple_host newkey;
2327                 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2328                 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2329                 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2330                 /* Create the ipv6 exact match flow */
2331                 memset(&entry, 0, sizeof(entry));
2332                 switch (i & (NUMBER_PORT_USED - 1)) {
2333                 case 0: entry = ipv6_udp_replay_route_array[0]; break;
2334                 case 1: entry = ipv6_udp_replay_route_array[1]; break;
2335                 case 2: entry = ipv6_udp_replay_route_array[2]; break;
2336                 case 3: entry = ipv6_udp_replay_route_array[3]; break;
2337                 };
2338                 entry.key.ip_dst[13] = c;
2339                 entry.key.ip_dst[14] = b;
2340                 entry.key.ip_dst[15] = a;
2341                 convert_ipv6_5tuple(&entry.key, &newkey);
2342                 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2343                 if (ret < 0) {
2344                         rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2345                 }
2346                 ipv6_udp_replay_out_if[ret] = (uint8_t) entry.if_out;
2347
2348         }
2349         printf("Hash: Adding 0x%x keys\n", nr_flow);
2350 }
2351
2352 static void
2353 setup_hash(int socketid)
2354 {
2355     struct rte_hash_parameters ipv4_udp_replay_hash_params = {
2356         .name = NULL,
2357         .entries = UDP_Replay_HASH_ENTRIES,
2358         .key_len = sizeof(union ipv4_5tuple_host),
2359         .hash_func = ipv4_hash_crc,
2360         .hash_func_init_val = 0,
2361     };
2362
2363     struct rte_hash_parameters ipv6_udp_replay_hash_params = {
2364         .name = NULL,
2365         .entries = UDP_Replay_HASH_ENTRIES,
2366         .key_len = sizeof(union ipv6_5tuple_host),
2367         .hash_func = ipv6_hash_crc,
2368         .hash_func_init_val = 0,
2369     };
2370
2371     char s[64];
2372
2373         /* create ipv4 hash */
2374         snprintf(s, sizeof(s), "ipv4_udp_replay_hash_%d", socketid);
2375         ipv4_udp_replay_hash_params.name = s;
2376         ipv4_udp_replay_hash_params.socket_id = socketid;
2377         ipv4_udp_replay_lookup_struct[socketid] = rte_hash_create(&ipv4_udp_replay_hash_params);
2378         if (ipv4_udp_replay_lookup_struct[socketid] == NULL)
2379                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay hash on "
2380                                 "socket %d\n", socketid);
2381
2382         /* create ipv6 hash */
2383         snprintf(s, sizeof(s), "ipv6_udp_replay_hash_%d", socketid);
2384         ipv6_udp_replay_hash_params.name = s;
2385         ipv6_udp_replay_hash_params.socket_id = socketid;
2386         ipv6_udp_replay_lookup_struct[socketid] = rte_hash_create(&ipv6_udp_replay_hash_params);
2387         if (ipv6_udp_replay_lookup_struct[socketid] == NULL)
2388                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay hash on "
2389                                 "socket %d\n", socketid);
2390
2391         if (hash_entry_number != HASH_ENTRY_NUMBER_DEFAULT) {
2392                 /* For testing hash matching with a large number of flows we
2393                  * generate millions of IP 5-tuples with an incremented dst
2394                  * address to initialize the hash table. */
2395                 if (ipv6 == 0) {
2396                         /* populate the ipv4 hash */
2397                         populate_ipv4_many_flow_into_table(
2398                                 ipv4_udp_replay_lookup_struct[socketid], hash_entry_number);
2399                 } else {
2400                         /* populate the ipv6 hash */
2401                         populate_ipv6_many_flow_into_table(
2402                                 ipv6_udp_replay_lookup_struct[socketid], hash_entry_number);
2403                 }
2404         } else {
2405                 /* Use data in ipv4/ipv6 udp_replay lookup table directly to initialize the hash table */
2406                 if (ipv6 == 0) {
2407                         /* populate the ipv4 hash */
2408                         populate_ipv4_few_flow_into_table(ipv4_udp_replay_lookup_struct[socketid]);
2409                 } else {
2410                         /* populate the ipv6 hash */
2411                         populate_ipv6_few_flow_into_table(ipv6_udp_replay_lookup_struct[socketid]);
2412                 }
2413         }
2414 }
2415 #endif
2416
2417 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2418 static void
2419 setup_lpm(int socketid)
2420 {
2421         struct rte_lpm6_config config;
2422         unsigned i;
2423         int ret;
2424         char s[64];
2425
2426         /* create the LPM table */
2427         snprintf(s, sizeof(s), "IPV4_UDP_Replay_LPM_%d", socketid);
2428         ipv4_udp_replay_lookup_struct[socketid] = rte_lpm_create(s, socketid,
2429                                 IPV4_UDP_Replay_LPM_MAX_RULES, 0);
2430         if (ipv4_udp_replay_lookup_struct[socketid] == NULL)
2431                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay LPM table"
2432                                 " on socket %d\n", socketid);
2433
2434         /* populate the LPM table */
2435         for (i = 0; i < IPV4_UDP_Replay_NUM_ROUTES; i++) {
2436
2437                 /* skip unused ports */
2438                 if ((1 << ipv4_udp_replay_route_array[i].if_out &
2439                                 enabled_port_mask) == 0)
2440                         continue;
2441
2442                 ret = rte_lpm_add(ipv4_udp_replay_lookup_struct[socketid],
2443                         ipv4_udp_replay_route_array[i].ip,
2444                         ipv4_udp_replay_route_array[i].depth,
2445                         ipv4_udp_replay_route_array[i].if_out);
2446
2447                 if (ret < 0) {
2448                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2449                                 "udp_replay LPM table on socket %d\n",
2450                                 i, socketid);
2451                 }
2452
2453                 printf("LPM: Adding route 0x%08x / %d (%d)\n",
2454                         (unsigned)ipv4_udp_replay_route_array[i].ip,
2455                         ipv4_udp_replay_route_array[i].depth,
2456                         ipv4_udp_replay_route_array[i].if_out);
2457         }
2458
2459         /* create the LPM6 table */
2460         snprintf(s, sizeof(s), "IPV6_UDP_Replay_LPM_%d", socketid);
2461
2462         config.max_rules = IPV6_UDP_Replay_LPM_MAX_RULES;
2463         config.number_tbl8s = IPV6_UDP_Replay_LPM_NUMBER_TBL8S;
2464         config.flags = 0;
2465         ipv6_udp_replay_lookup_struct[socketid] = rte_lpm6_create(s, socketid,
2466                                 &config);
2467         if (ipv6_udp_replay_lookup_struct[socketid] == NULL)
2468                 rte_exit(EXIT_FAILURE, "Unable to create the udp_replay LPM table"
2469                                 " on socket %d\n", socketid);
2470
2471         /* populate the LPM table */
2472         for (i = 0; i < IPV6_UDP_Replay_NUM_ROUTES; i++) {
2473
2474                 /* skip unused ports */
2475                 if ((1 << ipv6_udp_replay_route_array[i].if_out &
2476                                 enabled_port_mask) == 0)
2477                         continue;
2478
2479                 ret = rte_lpm6_add(ipv6_udp_replay_lookup_struct[socketid],
2480                         ipv6_udp_replay_route_array[i].ip,
2481                         ipv6_udp_replay_route_array[i].depth,
2482                         ipv6_udp_replay_route_array[i].if_out);
2483
2484                 if (ret < 0) {
2485                         rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2486                                 "udp_replay LPM table on socket %d\n",
2487                                 i, socketid);
2488                 }
2489
2490                 printf("LPM: Adding route %s / %d (%d)\n",
2491                         "IPV6",
2492                         ipv6_udp_replay_route_array[i].depth,
2493                         ipv6_udp_replay_route_array[i].if_out);
2494         }
2495 }
2496 #endif
2497
2498 static int
2499 init_mem(unsigned nb_mbuf)
2500 {
2501         struct lcore_conf *qconf;
2502         int socketid;
2503         unsigned lcore_id;
2504         char s[64];
2505
2506         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2507                 if (rte_lcore_is_enabled(lcore_id) == 0)
2508                         continue;
2509
2510                 if (numa_on)
2511                         socketid = rte_lcore_to_socket_id(lcore_id);
2512                 else
2513                         socketid = 0;
2514
2515                 if (socketid >= NB_SOCKETS) {
2516                         rte_exit(EXIT_FAILURE, "Socket %d of lcore %u is out of range %d\n",
2517                                 socketid, lcore_id, NB_SOCKETS);
2518                 }
2519                 if (pktmbuf_pool[socketid] == NULL) {
2520                         snprintf(s, sizeof(s), "mbuf_pool_%d", socketid);
2521                         pktmbuf_pool[socketid] =
2522                                 rte_pktmbuf_pool_create(s, nb_mbuf,
2523                                         MEMPOOL_CACHE_SIZE, 0,
2524                                         RTE_MBUF_DEFAULT_BUF_SIZE, socketid);
2525                         if (pktmbuf_pool[socketid] == NULL)
2526                                 rte_exit(EXIT_FAILURE,
2527                                                 "Cannot init mbuf pool on socket %d\n", socketid);
2528                         else
2529                                 printf("Allocated mbuf pool on socket %d\n", socketid);
2530
2531 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2532                         setup_lpm(socketid);
2533 #else
2534                         setup_hash(socketid);
2535 #endif
2536                 }
2537                 qconf = &lcore_conf[lcore_id];
2538                 qconf->ipv4_lookup_struct = ipv4_udp_replay_lookup_struct[socketid];
2539                 qconf->ipv6_lookup_struct = ipv6_udp_replay_lookup_struct[socketid];
2540         }
2541         return 0;
2542 }
2543
2544 /* Check the link status of all ports in up to 9s, and print them finally */
2545 static void
2546 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
2547 {
2548 #define CHECK_INTERVAL 100 /* 100ms */
2549 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2550         uint8_t portid, count, all_ports_up, print_flag = 0;
2551         struct rte_eth_link link;
2552
2553         printf("\nChecking link status");
2554         fflush(stdout);
2555         for (count = 0; count <= MAX_CHECK_TIME; count++) {
2556                 all_ports_up = 1;
2557                 for (portid = 0; portid < port_num; portid++) {
2558                         if ((port_mask & (1 << portid)) == 0)
2559                                 continue;
2560                         memset(&link, 0, sizeof(link));
2561                         rte_eth_link_get_nowait(portid, &link);
2562                         /* print link status if flag set */
2563                         if (print_flag == 1) {
2564                                 if (link.link_status)
2565                                         printf("Port %d Link Up - speed %u "
2566                                                 "Mbps - %s\n", (uint8_t)portid,
2567                                                 (unsigned)link.link_speed,
2568                                 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2569                                         ("full-duplex") : ("half-duplex\n"));
2570                                 else
2571                                         printf("Port %d Link Down\n",
2572                                                 (uint8_t)portid);
2573                                 continue;
2574                         }
2575                         /* clear all_ports_up flag if any link down */
2576                         if (link.link_status == 0) {
2577                                 all_ports_up = 0;
2578                                 break;
2579                         }
2580                 }
2581                 /* after finally printing all link status, get out */
2582                 if (print_flag == 1)
2583                         break;
2584
2585                 if (all_ports_up == 0) {
2586                         printf(".");
2587                         fflush(stdout);
2588                         rte_delay_ms(CHECK_INTERVAL);
2589                 }
2590
2591                 /* set the print_flag if all ports up or timeout */
2592                 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2593                         print_flag = 1;
2594                         printf("done\n");
2595                 }
2596         }
2597 }
2598
2599 int
2600 main(int argc, char **argv)
2601 {
2602         struct lcore_conf *qconf;
2603         struct rte_eth_dev_info dev_info;
2604         struct rte_eth_txconf *txconf;
2605         int ret;
2606         unsigned nb_ports;
2607         uint16_t queueid;
2608         unsigned lcore_id;
2609         uint32_t n_tx_queue, nb_lcores;
2610         uint8_t portid, nb_rx_queue, queue, socketid;
2611         struct cmdline *cl;
2612
2613         /* init EAL */
2614         ret = rte_eal_init(argc, argv);
2615         if (ret < 0)
2616                 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2617         argc -= ret;
2618         argv += ret;
2619         /* parse application arguments (after the EAL ones) */
2620         ret = parse_args(argc, argv);
2621         if (ret < 0)
2622                 rte_exit(EXIT_FAILURE, "Invalid UDP_Replay parameters\n");
2623
2624         if (check_lcore_params() < 0)
2625                 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
2626
2627         ret = init_lcore_rx_queues();
2628         if (ret < 0)
2629                 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2630
2631         nb_ports = rte_eth_dev_count();
2632         if (nb_ports > RTE_MAX_ETHPORTS)
2633                 nb_ports = RTE_MAX_ETHPORTS;
2634
2635         if (check_port_config(nb_ports) < 0)
2636                 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
2637
2638         nb_lcores = rte_lcore_count();
2639
2640         /* initialize all ports */
2641         for (portid = 0; portid < nb_ports; portid++) {
2642                 /* skip ports that are not enabled */
2643                 if ((enabled_port_mask & (1 << portid)) == 0) {
2644                         printf("\nSkipping disabled port %d\n", portid);
2645                         continue;
2646                 }
2647
2648                 /* init port */
2649                 printf("Initializing port %d ... ", portid );
2650                 fflush(stdout);
2651
2652                 nb_rx_queue = get_port_n_rx_queues(portid);
2653                 n_tx_queue = nb_rx_queue;
2654                 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
2655                         n_tx_queue = MAX_TX_QUEUE_PER_PORT;
2656                 printf("Creating queues: nb_rxq=%d nb_txq=%u... ",
2657                         nb_rx_queue, (unsigned)n_tx_queue );
2658                 ret = rte_eth_dev_configure(portid, nb_rx_queue,
2659                                         (uint16_t)n_tx_queue, &port_conf);
2660                 if (ret < 0) {
2661                         printf("Port configuration failed : port: %d... Try with hw-ip-checksum disabled\n", portid);
2662                         port_conf.rxmode.hw_ip_checksum = 0;
2663                         ret = rte_eth_dev_configure(portid, nb_rx_queue,
2664                                         (uint16_t)n_tx_queue, &port_conf);
2665                         if (ret < 0)
2666                              rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%d\n",
2667                                 ret, portid);
2668                 }
2669                 /*Since its just swapping of MAC we dont have to fill our own src mac*/
2670                 rte_eth_macaddr_get(portid, &ports_eth_addr[portid]);
2671                 print_ethaddr(" Address:", &ports_eth_addr[portid]);
2672
2673                 /* init memory */
2674                 ret = init_mem(NB_MBUF);
2675                 if (ret < 0)
2676                         rte_exit(EXIT_FAILURE, "init_mem failed\n");
2677
2678                 /* init one TX queue per couple (lcore,port) */
2679                 queueid = 0;
2680                 for (lcore_id = 0; lcore_id < n_tx_queue; lcore_id++) {
2681                         if (rte_lcore_is_enabled(lcore_id) == 0)
2682                                 continue;
2683
2684                         if (numa_on)
2685                                 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id);
2686                         else
2687                                 socketid = 0;
2688
2689                         printf("txq=%u,%d,%d ", lcore_id, queueid, socketid);
2690                         fflush(stdout);
2691
2692                         rte_eth_dev_info_get(portid, &dev_info);
2693                         txconf = &dev_info.default_txconf;
2694                         if (port_conf.rxmode.jumbo_frame)
2695                                 txconf->txq_flags = 0;
2696                         ret = rte_eth_tx_queue_setup(portid, queueid, nb_txd,
2697                                                      socketid, txconf);
2698                         if (ret < 0)
2699                                 rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup: err=%d, "
2700                                         "port=%d\n", ret, portid);
2701
2702                         qconf = &lcore_conf[lcore_id];
2703                         qconf->tx_queue_id[portid] = queueid;
2704                         queueid++;
2705                 }
2706                 printf("\n");
2707         }
2708
2709         for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
2710                 if (rte_lcore_is_enabled(lcore_id) == 0)
2711                         continue;
2712                 qconf = &lcore_conf[lcore_id];
2713                 printf("\nInitializing rx queues on lcore %u ... ", lcore_id );
2714                 fflush(stdout);
2715                 /* init RX queues */
2716                 for(queue = 0; queue < qconf->n_rx_queue; ++queue) {
2717                         portid = qconf->rx_queue_list[queue].port_id;
2718                         queueid = qconf->rx_queue_list[queue].queue_id;
2719
2720                         if (numa_on)
2721                                 socketid = (uint8_t)rte_lcore_to_socket_id(lcore_id);
2722                         else
2723                                 socketid = 0;
2724
2725                         printf("rxq=%d,%d,%d ", portid, queueid, socketid);
2726                         fflush(stdout);
2727
2728                         ret = rte_eth_rx_queue_setup(portid, queueid, nb_rxd,
2729                                         socketid,
2730                                         NULL,
2731                                         pktmbuf_pool[socketid]);
2732                         if (ret < 0)
2733                                 rte_exit(EXIT_FAILURE, "rte_eth_rx_queue_setup: err=%d,"
2734                                                 "port=%d\n", ret, portid);
2735                 }
2736         }
2737
2738         printf("\n");
2739
2740         /* start ports */
2741         for (portid = 0; portid < nb_ports; portid++) {
2742                 if ((enabled_port_mask & (1 << portid)) == 0) {
2743                         continue;
2744                 }
2745                 /* Start device */
2746                 ret = rte_eth_dev_start(portid);
2747                 if (ret < 0)
2748                         rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, port=%d\n",
2749                                 ret, portid);
2750
2751                 /*
2752                  * If enabled, put device in promiscuous mode.
2753                  * This allows IO forwarding mode to forward packets
2754                  * to itself through 2 cross-connected  ports of the
2755                  * target machine.
2756                  */
2757                 if (promiscuous_on)
2758                         rte_eth_promiscuous_enable(portid);
2759         }
2760
2761         check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask);
2762
2763         /* launch per-lcore init on every lcore */
2764         rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
2765         cl = cmdline_stdin_new(main_ctx, "Replay>");
2766         if (cl == NULL)
2767                 rte_panic("Cannot create cmdline instance\n");
2768         cmdline_interact(cl);
2769         cmdline_stdin_exit(cl);
2770         rte_exit(0, "Bye!\n");
2771         RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2772                 if (rte_eal_wait_lcore(lcore_id) < 0)
2773                         return -1;
2774         }
2775
2776         return 0;
2777 }
2778 /**********************************************************/
2779
2780 struct cmd_obj_clear_result {
2781         cmdline_fixed_string_t clear;
2782         cmdline_fixed_string_t udp_replay;
2783         cmdline_fixed_string_t stats;
2784 };
2785
2786 static void cmd_clear_udp_replay_stats_parsed(
2787         __rte_unused void *parsed_result,
2788         __rte_unused struct cmdline *cl,
2789    __attribute__((unused)) void *data)
2790 {
2791
2792         clear_stats();
2793 }
2794
2795 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_udp_replay_string =
2796         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, udp_replay, "UDP_Replay");
2797 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_clear_string =
2798         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, clear, "clear");
2799 cmdline_parse_token_string_t cmd_clear_udp_replay_stats_stats_string =
2800         TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, stats, "stats");
2801
2802 cmdline_parse_inst_t cmd_clear_udp_replay_stats = {
2803         .f = cmd_clear_udp_replay_stats_parsed,  /* function to call */
2804         .data = NULL,      /* 2nd arg of func */
2805         .help_str = "clears UDP_Replay stats for rx/tx",
2806         .tokens = {        /* token list, NULL terminated */
2807                 (void *)&cmd_clear_udp_replay_stats_udp_replay_string,
2808                 (void *)&cmd_clear_udp_replay_stats_clear_string,
2809                 (void *)&cmd_clear_udp_replay_stats_stats_string,
2810                 NULL,
2811         },
2812 };
2813 /**********************************************************/
2814 struct cmd_obj_add_result {
2815         cmdline_fixed_string_t action;
2816         cmdline_fixed_string_t name;
2817 };
2818
2819 static void cmd_udp_replay_stats_parsed(
2820         __rte_unused void *parsed_result,
2821         __rte_unused struct cmdline *cl,
2822    __attribute__((unused)) void *data)
2823 {
2824         print_stats();
2825 }
2826
2827 cmdline_parse_token_string_t cmd_udp_replay_stats_udp_replay_string =
2828         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, action, "UDP_Replay");
2829 cmdline_parse_token_string_t cmd_udp_replay_stats_stats_string =
2830         TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, name, "stats");
2831
2832 cmdline_parse_inst_t cmd_udp_replay_stats = {
2833         .f = cmd_udp_replay_stats_parsed,  /* function to call */
2834         .data = NULL,      /* 2nd arg of func */
2835         .help_str = "UDP_Replay stats for rx/tx",
2836         .tokens = {        /* token list, NULL terminated */
2837                 (void *)&cmd_udp_replay_stats_udp_replay_string,
2838                 (void *)&cmd_udp_replay_stats_stats_string,
2839                 NULL,
2840         },
2841 };
2842 /* quit*/
2843 struct cmd_quit_result {
2844         cmdline_fixed_string_t quit;
2845 };
2846
2847 static void
2848 cmd_quit_parsed(
2849         __rte_unused void *parsed_result,
2850         struct cmdline *cl,
2851         __rte_unused void *data)
2852 {
2853         cmdline_quit(cl);
2854 }
2855
2856 static cmdline_parse_token_string_t cmd_quit_quit =
2857         TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
2858
2859 static cmdline_parse_inst_t cmd_quit = {
2860         .f = cmd_quit_parsed,
2861         .data = NULL,
2862         .help_str = "Quit",
2863         .tokens = {
2864                   (void *) &cmd_quit_quit,
2865                   NULL,
2866         },
2867 };
2868
2869 /**********************************************************/
2870 /****** CONTEXT (list of instruction) */
2871 cmdline_parse_ctx_t main_ctx[] = {
2872         (cmdline_parse_inst_t *)&cmd_udp_replay_stats,
2873         (cmdline_parse_inst_t *)&cmd_clear_udp_replay_stats,
2874         (cmdline_parse_inst_t *)&cmd_quit,
2875         NULL,
2876 };