2 // Copyright (c) 2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
17 /* Changes for Correlated traffic VNF
19 2. Modify received packet
20 a.exchange src mac and destination mac
21 b.exchange src ip and destination IP for both IPv4 and IPv6 cases
22 c.exchange UDP src port and UDP destination port
23 d.change the len of the response according to the IMIX definition (
24 option to make traffic more realistic to emulate some IoT payloads)
25 3. send modified packet to the port where it was received.
27 Such VNF does not need LPM and routing table implementations.
28 As the packet modification is very minimal and there is no memory access as the packet is stored in L3 cache the
29 performance of the solution should be sufficient for testing the UDP NAT performance.
34 #include <sys/types.h>
36 #include <sys/queue.h>
41 #include <rte_common.h>
43 #include <rte_byteorder.h>
45 #include <rte_memory.h>
46 #include <rte_memcpy.h>
47 #include <rte_memzone.h>
49 #include <rte_per_lcore.h>
50 #include <rte_launch.h>
51 #include <rte_atomic.h>
52 #include <rte_cycles.h>
53 #include <rte_prefetch.h>
54 #include <rte_lcore.h>
55 #include <rte_per_lcore.h>
56 #include <rte_branch_prediction.h>
57 #include <rte_interrupts.h>
59 #include <rte_random.h>
60 #include <rte_debug.h>
61 #include <rte_ether.h>
62 #include <rte_ethdev.h>
64 #include <rte_mempool.h>
69 #include <rte_string_fns.h>
71 #include <cmdline_parse.h>
72 #include <cmdline_parse_etheraddr.h>
73 #include <cmdline_rdline.h>
74 #include <cmdline_socket.h>
76 #include <cmdline_parse_num.h>
77 #include <cmdline_parse_string.h>
78 #include <cmdline_parse_ipaddr.h>
79 #include <rte_errno.h>
80 #include <rte_cfgfile.h>
82 #include "parse_obj_list.h"
86 #include "interface.h"
87 #include "l3fwd_common.h"
88 #include "l3fwd_lpm4.h"
89 #include "l3fwd_lpm6.h"
90 #include "lib_icmpv6.h"
92 #include "vnf_common.h"
95 #define APP_LOOKUP_EXACT_MATCH 0
96 #define APP_LOOKUP_LPM 1
97 #define DO_RFC_1812_CHECKS
99 #ifndef APP_LOOKUP_METHOD
100 #define APP_LOOKUP_METHOD APP_LOOKUP_EXACT_MATCH
106 #include <netinet/in.h>
109 * When set to zero, simple forwaring path is eanbled.
110 * When set to one, optimized forwarding path is enabled.
111 * Note that LPM optimisation path uses SSE4.1 instructions.
113 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && !defined(__SSE4_1__))
114 #define ENABLE_MULTI_BUFFER_OPTIMIZE 0
116 #define ENABLE_MULTI_BUFFER_OPTIMIZE 1
119 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
120 #include <rte_hash.h>
121 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
123 #include <rte_lpm6.h>
125 #error "APP_LOOKUP_METHOD set to incorrect value"
129 #define IPv6_BYTES_FMT "%02x%02x:%02x%02x:%02x%02x:%02x%02x:"\
130 "%02x%02x:%02x%02x:%02x%02x:%02x%02x"
131 #define IPv6_BYTES(addr) \
132 addr[0], addr[1], addr[2], addr[3], \
133 addr[4], addr[5], addr[6], addr[7], \
134 addr[8], addr[9], addr[10], addr[11],\
135 addr[12], addr[13],addr[14], addr[15]
139 #define RTE_LOGTYPE_L3FWD RTE_LOGTYPE_USER1
141 #define MAX_JUMBO_PKT_LEN 9600
143 #define IPV6_ADDR_LEN 16
145 #define MEMPOOL_CACHE_SIZE 256
148 * This expression is used to calculate the number of mbufs needed depending on user input, taking
149 * into account memory for rx and tx hardware rings, cache per lcore and mtable per port per lcore.
150 * RTE_MAX is used to ensure that NB_MBUF never goes below a minimum value of 8192
153 #define NB_MBUF RTE_MAX ( \
154 (nb_ports*nb_rx_queue*RTE_TEST_RX_DESC_DEFAULT + \
155 nb_ports*nb_lcores*MAX_PKT_BURST + \
156 nb_ports*n_tx_queue*RTE_TEST_TX_DESC_DEFAULT + \
157 nb_lcores*MEMPOOL_CACHE_SIZE), \
160 #define MAX_PKT_BURST 32
161 #define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
164 * Try to avoid TX buffering if we have at least MAX_TX_BURST packets to send.
166 #define MAX_TX_BURST (MAX_PKT_BURST / 2)
170 /* Configure how many packets ahead to prefetch, when reading packets */
171 #define PREFETCH_OFFSET 3
173 /* Used to mark destination port as 'invalid'. */
174 #define BAD_PORT ((uint16_t)-1)
179 * Configurable number of RX/TX ring descriptors
181 #define RTE_TEST_RX_DESC_DEFAULT 128
182 #define RTE_TEST_TX_DESC_DEFAULT 512
183 static uint64_t rcv_pkt_count[32] = {0};
184 static uint64_t tx_pkt_count[32] = {0};
186 struct sockaddr_in ipaddr1, ipaddr2;
188 /* ethernet addresses of ports */
189 static uint64_t dest_eth_addr[RTE_MAX_ETHPORTS];
190 static __m128i val_eth[RTE_MAX_ETHPORTS];
192 cmdline_parse_ctx_t main_ctx[];
194 uint32_t timer_lcore;
195 uint32_t exit_loop = 1;
197 port_config_t *port_config;
198 #define MEMPOOL_SIZE 32 * 1024
199 #define BUFFER_SIZE 2048
200 #define CACHE_SIZE 256
203 /* replace first 12B of the ethernet header. */
204 #define MASK_ETH 0x3f
206 #define IP_TYPE_IPV4 0
207 #define IP_TYPE_IPV6 1
210 const char* ipv4[MAX_IP];
211 uint8_t link_ipv6[MAX_IP][16];
212 uint32_t type, numports;
214 /* mask of enabled ports */
215 static uint32_t enabled_port_mask = 0;
216 static int promiscuous_on = 0; /**< Ports set in promiscuous mode off by default. */
217 static int numa_on = 1; /**< NUMA is enabled by default. */
218 static int csum_on = 1; /**< NUMA is enabled by default. */
219 struct pipeline_params def_pipeline_params = {
228 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
229 static int ipv6 = 0; /**< ipv6 is false by default. */
232 void convert_ipstr_to_numeric(void);
234 int print_l4stats(void);
235 int clear_stats(void);
239 struct rte_mbuf *m_table[MAX_PKT_BURST];
242 struct lcore_rx_queue {
245 } __rte_cache_aligned;
247 #define MAX_RX_QUEUE_PER_LCORE 16
248 #define MAX_TX_QUEUE_PER_PORT RTE_MAX_ETHPORTS
249 #define MAX_RX_QUEUE_PER_PORT 128
251 #define MAX_LCORE_PARAMS 1024
252 struct lcore_params {
256 } __rte_cache_aligned;
258 static struct lcore_params lcore_params_array[MAX_LCORE_PARAMS];
259 static struct lcore_params lcore_params_array_default[] = {
271 static struct lcore_params * lcore_params = lcore_params_array_default;
272 static uint16_t nb_lcore_params = sizeof(lcore_params_array_default) /
273 sizeof(lcore_params_array_default[0]);
275 static struct rte_eth_conf port_conf = {
277 .mq_mode = ETH_MQ_RX_RSS,
278 .max_rx_pkt_len = ETHER_MAX_LEN,
280 .header_split = 0, /**< Header Split disabled */
281 .hw_ip_checksum = 1, /**< IP checksum offload enabled */
282 .hw_vlan_filter = 0, /**< VLAN filtering disabled */
283 .jumbo_frame = 0, /**< Jumbo Frame Support disabled */
284 .hw_strip_crc = 0, /**< CRC stripped by hardware */
289 .rss_hf = ETH_RSS_IP,
293 .mq_mode = ETH_MQ_TX_NONE,
297 /* empty vmdq configuration structure. Filled in programatically */
298 static struct rte_eth_rxconf rx_conf = {
304 .rx_free_thresh = 64,
306 .rx_deferred_start = 0,
309 static struct rte_eth_txconf tx_conf = {
317 .txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
318 ETH_TXQ_FLAGS_NOOFFLOADS,
319 .tx_deferred_start = 0,
322 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
324 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
325 #include <rte_hash_crc.h>
326 #define DEFAULT_HASH_FUNC rte_hash_crc
328 #include <rte_jhash.h>
329 #define DEFAULT_HASH_FUNC rte_jhash
338 } __attribute__((__packed__));
340 union ipv4_5tuple_host {
353 #define XMM_NUM_IN_IPV6_5TUPLE 3
356 uint8_t ip_dst[IPV6_ADDR_LEN];
357 uint8_t ip_src[IPV6_ADDR_LEN];
361 } __attribute__((__packed__));
363 union ipv6_5tuple_host {
368 uint8_t ip_src[IPV6_ADDR_LEN];
369 uint8_t ip_dst[IPV6_ADDR_LEN];
374 __m128i xmm[XMM_NUM_IN_IPV6_5TUPLE];
377 struct ipv4_l3fwd_route {
378 struct ipv4_5tuple key;
382 struct ipv6_l3fwd_route {
383 struct ipv6_5tuple key;
387 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
388 {{IPv4(101,0,0,0), IPv4(100,10,0,1), 101, 11, IPPROTO_TCP}, 0},
389 {{IPv4(201,0,0,0), IPv4(200,20,0,1), 102, 12, IPPROTO_TCP}, 1},
390 {{IPv4(111,0,0,0), IPv4(100,30,0,1), 101, 11, IPPROTO_TCP}, 2},
391 {{IPv4(211,0,0,0), IPv4(200,40,0,1), 102, 12, IPPROTO_TCP}, 3},
394 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
396 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
397 {0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
398 101, 11, IPPROTO_TCP}, 0},
401 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
402 {0xfe, 0x90, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
403 102, 12, IPPROTO_TCP}, 1},
406 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
407 {0xfe, 0xa0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
408 101, 11, IPPROTO_TCP}, 2},
411 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1e, 0x67, 0xff, 0xfe, 0, 0, 0},
412 {0xfe, 0xb0, 0, 0, 0, 0, 0, 0, 0x02, 0x1b, 0x21, 0xff, 0xfe, 0x91, 0x38, 0x05},
413 102, 12, IPPROTO_TCP}, 3},
416 typedef struct rte_hash lookup_struct_t;
418 #ifdef RTE_ARCH_X86_64
419 /* default to 4 million hash entries (approx) */
420 #define L3FWD_HASH_ENTRIES 1024*1024*4
422 /* 32-bit has less address-space for hugepage memory, limit to 1M entries */
423 #define L3FWD_HASH_ENTRIES 1024*1024*1
425 #define HASH_ENTRY_NUMBER_DEFAULT 4
427 static uint32_t hash_entry_number = HASH_ENTRY_NUMBER_DEFAULT;
430 app_link_up_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
436 app_link_down_internal(__rte_unused struct app_params *app, struct app_link_params *cp)
442 * inet_pton_ipv4(src, dst)
443 * like inet_aton() but without all the hexadecimal and shorthand.
445 * 1 if `src' is a valid dotted quad, else 0.
447 * does not touch `dst' unless it's returning 1.
451 static int inet_pton_ipv4(const char *src, unsigned char *dst)
453 static const char digits[] = "0123456789";
454 int saw_digit, octets, ch;
455 unsigned char tmp[INADDRSZ], *tp;
460 while ((ch = *src++) != '\0') {
463 if ((pch = strchr(digits, ch)) != NULL) {
464 unsigned int new = *tp * 10 + (pch - digits);
473 *tp = (unsigned char)new;
474 } else if (ch == '.' && saw_digit) {
485 memcpy(dst, tmp, INADDRSZ);
490 * inet_pton_ipv6(src, dst)
491 * convert presentation level address to network order binary form.
493 * 1 if `src' is a valid [RFC1884 2.2] address, else 0.
495 * (1) does not touch `dst' unless it's returning 1.
496 * (2) :: in a full address is silently ignored.
498 * inspired by Mark Andrews.
502 static int inet_pton_ipv6(const char *src, unsigned char *dst)
504 static const char xdigits_l[] = "0123456789abcdef",
505 xdigits_u[] = "0123456789ABCDEF";
506 unsigned char tmp[IN6ADDRSZ], *tp = 0, *endp = 0, *colonp = 0;
507 const char *xdigits = 0, *curtok = 0;
508 int ch = 0, saw_xdigit = 0, count_xdigit = 0;
509 unsigned int val = 0;
510 unsigned dbloct_count = 0;
512 memset((tp = tmp), '\0', IN6ADDRSZ);
513 endp = tp + IN6ADDRSZ;
515 /* Leading :: requires some special handling. */
520 saw_xdigit = count_xdigit = 0;
523 while ((ch = *src++) != '\0') {
526 if ((pch = strchr((xdigits = xdigits_l), ch)) == NULL)
527 pch = strchr((xdigits = xdigits_u), ch);
529 if (count_xdigit >= 4)
532 val |= (pch - xdigits);
546 } else if (*src == '\0') {
549 if (tp + sizeof(int16_t) > endp)
551 *tp++ = (unsigned char)((val >> 8) & 0xff);
552 *tp++ = (unsigned char)(val & 0xff);
559 if (ch == '.' && ((tp + INADDRSZ) <= endp) &&
560 inet_pton_ipv4(curtok, tp) > 0) {
564 break; /* '\0' was seen by inet_pton4(). */
569 if (tp + sizeof(int16_t) > endp)
571 *tp++ = (unsigned char)((val >> 8) & 0xff);
572 *tp++ = (unsigned char)(val & 0xff);
575 if (colonp != NULL) {
576 /* if we already have 8 double octets, having a colon means error */
577 if (dbloct_count == 8)
581 * Since some memmove()'s erroneously fail to handle
582 * overlapping regions, we'll do the shift by hand.
584 const int n = tp - colonp;
587 for (i = 1; i <= n; i++) {
588 endp[-i] = colonp[n - i];
595 memcpy(dst, tmp, IN6ADDRSZ);
599 static int my_inet_pton_ipv6(int af, const char *src, void *dst)
603 return inet_pton_ipv4(src, dst);
605 return inet_pton_ipv6(src, dst);
607 errno = EAFNOSUPPORT;
612 void convert_ipstr_to_numeric(void)
616 for (i = 0; i < numports; i++)
618 if (type == IP_TYPE_IPV4) {
619 memset(&ipaddr1, '\0', sizeof(struct sockaddr_in));
620 ipaddr1.sin_addr.s_addr = inet_addr(ipv4[i]);
621 ifm_add_ipv4_port(i, ipaddr1.sin_addr.s_addr, 24);
622 } else if (type == IP_TYPE_IPV6) {
623 ifm_add_ipv6_port(i, &link_ipv6[i][0], 128);
628 static inline uint32_t
629 ipv4_hash_crc(const void *data, __rte_unused uint32_t data_len,
632 const union ipv4_5tuple_host *k;
638 p = (const uint32_t *)&k->port_src;
640 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
641 init_val = rte_hash_crc_4byte(t, init_val);
642 init_val = rte_hash_crc_4byte(k->ip_src, init_val);
643 init_val = rte_hash_crc_4byte(k->ip_dst, init_val);
644 init_val = rte_hash_crc_4byte(*p, init_val);
645 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
646 init_val = rte_jhash_1word(t, init_val);
647 init_val = rte_jhash_1word(k->ip_src, init_val);
648 init_val = rte_jhash_1word(k->ip_dst, init_val);
649 init_val = rte_jhash_1word(*p, init_val);
650 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
653 static uint64_t arp_pkts[32] = {0};
655 static inline int check_arpicmp(struct rte_mbuf *pkt)
658 uint8_t in_port_id = pkt->port;
659 uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
661 uint16_t *eth_proto =
662 RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
665 uint32_t prot_offset =
666 MBUF_HDR_ROOM + ETH_HDR_SIZE + IP_HDR_PROTOCOL_OFST;
668 protocol = RTE_MBUF_METADATA_UINT8_PTR(pkt, prot_offset);
670 if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_ARP) ||
671 ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV4)
672 && (*protocol == IP_PROTOCOL_ICMP))) {
673 process_arpicmp_pkt(pkt, ifm_get_port(in_port_id));
674 arp_pkts[in_port_id]++;
682 static inline int check_arpicmpv6(struct rte_mbuf *pkt)
685 struct ether_hdr *eth_h;
686 struct ipv6_hdr *ipv6_h;
688 uint8_t in_port_id = pkt->port;
689 uint32_t eth_proto_offset = MBUF_HDR_ROOM + 12;
691 uint16_t *eth_proto =
692 RTE_MBUF_METADATA_UINT16_PTR(pkt, eth_proto_offset);
694 eth_h = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
695 ipv6_h = (struct ipv6_hdr *)((char *)eth_h + sizeof(struct ether_hdr));
697 if ((rte_be_to_cpu_16(*eth_proto) == ETH_TYPE_IPV6)
698 && (ipv6_h->proto == ICMPV6_PROTOCOL_ID)) {
699 process_icmpv6_pkt(pkt, ifm_get_port(in_port_id));
707 static inline uint32_t
708 ipv6_hash_crc(const void *data, __rte_unused uint32_t data_len, uint32_t init_val)
710 const union ipv6_5tuple_host *k;
713 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
714 const uint32_t *ip_src0, *ip_src1, *ip_src2, *ip_src3;
715 const uint32_t *ip_dst0, *ip_dst1, *ip_dst2, *ip_dst3;
716 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
720 p = (const uint32_t *)&k->port_src;
722 #ifdef RTE_MACHINE_CPUFLAG_SSE4_2
723 ip_src0 = (const uint32_t *) k->ip_src;
724 ip_src1 = (const uint32_t *)(k->ip_src+4);
725 ip_src2 = (const uint32_t *)(k->ip_src+8);
726 ip_src3 = (const uint32_t *)(k->ip_src+12);
727 ip_dst0 = (const uint32_t *) k->ip_dst;
728 ip_dst1 = (const uint32_t *)(k->ip_dst+4);
729 ip_dst2 = (const uint32_t *)(k->ip_dst+8);
730 ip_dst3 = (const uint32_t *)(k->ip_dst+12);
731 init_val = rte_hash_crc_4byte(t, init_val);
732 init_val = rte_hash_crc_4byte(*ip_src0, init_val);
733 init_val = rte_hash_crc_4byte(*ip_src1, init_val);
734 init_val = rte_hash_crc_4byte(*ip_src2, init_val);
735 init_val = rte_hash_crc_4byte(*ip_src3, init_val);
736 init_val = rte_hash_crc_4byte(*ip_dst0, init_val);
737 init_val = rte_hash_crc_4byte(*ip_dst1, init_val);
738 init_val = rte_hash_crc_4byte(*ip_dst2, init_val);
739 init_val = rte_hash_crc_4byte(*ip_dst3, init_val);
740 init_val = rte_hash_crc_4byte(*p, init_val);
741 #else /* RTE_MACHINE_CPUFLAG_SSE4_2 */
742 init_val = rte_jhash_1word(t, init_val);
743 init_val = rte_jhash(k->ip_src, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
744 init_val = rte_jhash(k->ip_dst, sizeof(uint8_t) * IPV6_ADDR_LEN, init_val);
745 init_val = rte_jhash_1word(*p, init_val);
746 #endif /* RTE_MACHINE_CPUFLAG_SSE4_2 */
750 #define IPV4_L3FWD_NUM_ROUTES \
751 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0]))
753 #define IPV6_L3FWD_NUM_ROUTES \
754 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0]))
756 static uint8_t ipv4_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
757 static uint8_t ipv6_l3fwd_out_if[L3FWD_HASH_ENTRIES] __rte_cache_aligned;
761 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
762 struct ipv4_l3fwd_route {
768 struct ipv6_l3fwd_route {
774 static struct ipv4_l3fwd_route ipv4_l3fwd_route_array[] = {
775 {IPv4(1,1,1,0), 24, 0},
776 {IPv4(2,1,1,0), 24, 1},
777 {IPv4(3,1,1,0), 24, 2},
778 {IPv4(4,1,1,0), 24, 3},
779 {IPv4(5,1,1,0), 24, 4},
780 {IPv4(6,1,1,0), 24, 5},
781 {IPv4(7,1,1,0), 24, 6},
782 {IPv4(8,1,1,0), 24, 7},
785 static struct ipv6_l3fwd_route ipv6_l3fwd_route_array[] = {
786 {{1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 0},
787 {{2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 1},
788 {{3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 2},
789 {{4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 3},
790 {{5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 4},
791 {{6,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 5},
792 {{7,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 6},
793 {{8,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, 48, 7},
796 #define IPV4_L3FWD_NUM_ROUTES \
797 (sizeof(ipv4_l3fwd_route_array) / sizeof(ipv4_l3fwd_route_array[0]))
798 #define IPV6_L3FWD_NUM_ROUTES \
799 (sizeof(ipv6_l3fwd_route_array) / sizeof(ipv6_l3fwd_route_array[0]))
801 #define IPV4_L3FWD_LPM_MAX_RULES 1024
802 #define IPV6_L3FWD_LPM_MAX_RULES 1024
803 #define IPV6_L3FWD_LPM_NUMBER_TBL8S (1 << 16)
805 typedef struct rte_lpm lookup_struct_t;
806 typedef struct rte_lpm6 lookup6_struct_t;
807 static lookup_struct_t *ipv4_l3fwd_lookup_struct[NB_SOCKETS];
808 static lookup6_struct_t *ipv6_l3fwd_lookup_struct[NB_SOCKETS];
813 struct lcore_rx_queue rx_queue_list[MAX_RX_QUEUE_PER_LCORE];
814 uint16_t tx_queue_id[RTE_MAX_ETHPORTS];
815 struct mbuf_table tx_mbufs[RTE_MAX_ETHPORTS];
816 lookup_struct_t * ipv4_lookup_struct;
817 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
818 lookup6_struct_t * ipv6_lookup_struct;
820 lookup_struct_t * ipv6_lookup_struct;
822 } __rte_cache_aligned;
824 static struct lcore_conf lcore_conf[RTE_MAX_LCORE];
826 /* Send burst of packets on an output interface */
828 send_burst(struct lcore_conf *qconf, uint16_t n, uint8_t port)
830 struct rte_mbuf **m_table;
834 queueid = qconf->tx_queue_id[port];
835 m_table = (struct rte_mbuf **)qconf->tx_mbufs[port].m_table;
837 ret = rte_eth_tx_burst(port, queueid, m_table, n);
838 if (unlikely(ret < n)) {
840 rte_pktmbuf_free(m_table[ret]);
844 tx_pkt_count[port] += ret;
848 /* Enqueue a single packet, and send burst if queue is filled */
850 send_single_packet(struct rte_mbuf *m, uint8_t port)
854 struct lcore_conf *qconf;
856 lcore_id = rte_lcore_id();
858 qconf = &lcore_conf[lcore_id];
859 len = qconf->tx_mbufs[port].len;
860 qconf->tx_mbufs[port].m_table[len] = m;
863 /* enough pkts to be sent */
864 if (unlikely(len == MAX_PKT_BURST)) {
865 send_burst(qconf, MAX_PKT_BURST, port);
869 qconf->tx_mbufs[port].len = len;
873 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
874 static inline __attribute__((always_inline)) void
875 send_packetsx4(struct lcore_conf *qconf, uint8_t port,
876 struct rte_mbuf *m[], uint32_t num)
880 len = qconf->tx_mbufs[port].len;
883 * If TX buffer for that queue is empty, and we have enough packets,
884 * then send them straightway.
886 if (num >= MAX_TX_BURST && len == 0) {
887 n = rte_eth_tx_burst(port, qconf->tx_queue_id[port], m, num);
888 if (unlikely(n < num)) {
890 rte_pktmbuf_free(m[n]);
897 * Put packets into TX buffer for that queue.
901 n = (n > MAX_PKT_BURST) ? MAX_PKT_BURST - len : num;
904 switch (n % FWDSTEP) {
907 qconf->tx_mbufs[port].m_table[len + j] = m[j];
910 qconf->tx_mbufs[port].m_table[len + j] = m[j];
913 qconf->tx_mbufs[port].m_table[len + j] = m[j];
916 qconf->tx_mbufs[port].m_table[len + j] = m[j];
923 /* enough pkts to be sent */
924 if (unlikely(len == MAX_PKT_BURST)) {
926 send_burst(qconf, MAX_PKT_BURST, port);
928 /* copy rest of the packets into the TX buffer. */
931 switch (len % FWDSTEP) {
934 qconf->tx_mbufs[port].m_table[j] = m[n + j];
937 qconf->tx_mbufs[port].m_table[j] = m[n + j];
940 qconf->tx_mbufs[port].m_table[j] = m[n + j];
943 qconf->tx_mbufs[port].m_table[j] = m[n + j];
949 qconf->tx_mbufs[port].len = len;
951 #endif /* APP_LOOKUP_LPM */
953 #ifdef DO_RFC_1812_CHECKS
955 is_valid_pkt_ipv4(struct ipv4_hdr *pkt, uint32_t link_len)
957 /* From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2 */
959 * 1. The packet length reported by the Link Layer must be large
960 * enough to hold the minimum length legal IP datagram (20 bytes).
962 if (link_len < sizeof(struct ipv4_hdr))
965 /* 2. The IP checksum must be correct. */
966 /* this is checked in H/W */
969 * 3. The IP version number must be 4. If the version number is not 4
970 * then the packet may be another version of IP, such as IPng or
973 if (((pkt->version_ihl) >> 4) != 4)
976 * 4. The IP header length field must be large enough to hold the
977 * minimum length legal IP datagram (20 bytes = 5 words).
979 if ((pkt->version_ihl & 0xf) < 5)
983 * 5. The IP total length field must be large enough to hold the IP
984 * datagram header, whose length is specified in the IP header length
987 if (rte_cpu_to_be_16(pkt->total_length) < sizeof(struct ipv4_hdr))
994 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
996 static __m128i mask0;
997 static __m128i mask1;
998 static __m128i mask2;
999 static inline uint8_t
1000 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct)
1003 union ipv4_5tuple_host key;
1005 ipv4_hdr = (uint8_t *)ipv4_hdr + offsetof(struct ipv4_hdr, time_to_live);
1006 __m128i data = _mm_loadu_si128((__m128i*)(ipv4_hdr));
1007 /* Get 5 tuple: dst port, src port, dst IP address, src IP address and protocol */
1008 key.xmm = _mm_and_si128(data, mask0);
1009 /* Find destination port */
1010 ret = rte_hash_lookup(ipv4_l3fwd_lookup_struct, (const void *)&key);
1011 return (uint8_t)((ret < 0)? portid : ipv4_l3fwd_out_if[ret]);
1014 static inline uint8_t
1015 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup_struct_t * ipv6_l3fwd_lookup_struct)
1018 union ipv6_5tuple_host key;
1020 ipv6_hdr = (uint8_t *)ipv6_hdr + offsetof(struct ipv6_hdr, payload_len);
1021 __m128i data0 = _mm_loadu_si128((__m128i*)(ipv6_hdr));
1022 __m128i data1 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)));
1023 __m128i data2 = _mm_loadu_si128((__m128i*)(((uint8_t*)ipv6_hdr)+sizeof(__m128i)+sizeof(__m128i)));
1024 /* Get part of 5 tuple: src IP address lower 96 bits and protocol */
1025 key.xmm[0] = _mm_and_si128(data0, mask1);
1026 /* Get part of 5 tuple: dst IP address lower 96 bits and src IP address higher 32 bits */
1028 /* Get part of 5 tuple: dst port and src port and dst IP address higher 32 bits */
1029 key.xmm[2] = _mm_and_si128(data2, mask2);
1031 /* Find destination port */
1032 ret = rte_hash_lookup(ipv6_l3fwd_lookup_struct, (const void *)&key);
1033 return (uint8_t)((ret < 0)? portid : ipv6_l3fwd_out_if[ret]);
1037 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1039 static inline uint8_t
1040 get_ipv4_dst_port(void *ipv4_hdr, uint8_t portid, lookup_struct_t * ipv4_l3fwd_lookup_struct)
1044 return (uint8_t) ((rte_lpm_lookup(ipv4_l3fwd_lookup_struct,
1045 rte_be_to_cpu_32(((struct ipv4_hdr *)ipv4_hdr)->dst_addr),
1046 &next_hop) == 0) ? next_hop : portid);
1049 static inline uint8_t
1050 get_ipv6_dst_port(void *ipv6_hdr, uint8_t portid, lookup6_struct_t * ipv6_l3fwd_lookup_struct)
1053 return (uint8_t) ((rte_lpm6_lookup(ipv6_l3fwd_lookup_struct,
1054 ((struct ipv6_hdr*)ipv6_hdr)->dst_addr, &next_hop) == 0)?
1059 static inline void l3fwd_simple_replay(struct rte_mbuf *m, uint8_t portid,
1060 struct lcore_conf *qconf) __attribute__((unused));
1062 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH) && \
1063 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1065 #define MASK_ALL_PKTS 0xff
1066 #define EXCLUDE_1ST_PKT 0xfe
1067 #define EXCLUDE_2ND_PKT 0xfd
1068 #define EXCLUDE_3RD_PKT 0xfb
1069 #define EXCLUDE_4TH_PKT 0xf7
1070 #define EXCLUDE_5TH_PKT 0xef
1071 #define EXCLUDE_6TH_PKT 0xdf
1072 #define EXCLUDE_7TH_PKT 0xbf
1073 #define EXCLUDE_8TH_PKT 0x7f
1076 simple_ipv4_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
1078 struct ether_hdr *eth_hdr[8];
1079 struct ether_hdr tmp;
1080 struct ipv4_hdr *ipv4_hdr[8];
1081 struct udp_hdr *udp_hdr[8];
1083 l2_phy_interface_t *port = ifm_get_port(portid);
1086 printf("port may be un initialized\n");
1090 for ( i = 0; i < 8; i++)
1093 a[i] = check_arpicmp(m[i]);
1095 printf("null packet received\n");
1100 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
1101 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
1102 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
1103 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
1104 eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
1105 eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
1106 eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
1107 eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
1110 memset(&tmp,0,sizeof (struct ether_hdr));
1114 ether_addr_copy(ð_hdr[i]->s_addr, &tmp.s_addr);
1115 ether_addr_copy(ð_hdr[i]->d_addr, ð_hdr[i]->s_addr);
1116 ether_addr_copy(&tmp.s_addr, ð_hdr[i]->d_addr);
1119 /* Handle IPv4 headers.*/
1120 ipv4_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv4_hdr *,
1121 sizeof(struct ether_hdr));
1122 ipv4_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv4_hdr *,
1123 sizeof(struct ether_hdr));
1124 ipv4_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv4_hdr *,
1125 sizeof(struct ether_hdr));
1126 ipv4_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv4_hdr *,
1127 sizeof(struct ether_hdr));
1128 ipv4_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv4_hdr *,
1129 sizeof(struct ether_hdr));
1130 ipv4_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv4_hdr *,
1131 sizeof(struct ether_hdr));
1132 ipv4_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv4_hdr *,
1133 sizeof(struct ether_hdr));
1134 ipv4_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv4_hdr *,
1135 sizeof(struct ether_hdr));
1136 struct ipv4_hdr temp_ipv4;
1139 temp_ipv4.dst_addr = ipv4_hdr[i]->dst_addr;
1140 ipv4_hdr[i]->dst_addr = ipv4_hdr[i]->src_addr;
1141 ipv4_hdr[i]->src_addr = temp_ipv4.dst_addr;
1144 /* Handle UDP headers.*/
1145 udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1146 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1148 udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1149 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1150 udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1151 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1152 udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1153 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1154 udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1155 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1156 udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1157 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1158 udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1159 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1160 udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1161 sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr));
1162 /*1) memcpy or assignment.*/
1164 struct udp_hdr temp_udp;
1167 temp_udp.dst_port = udp_hdr[i]->dst_port;
1168 udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1169 udp_hdr[i]->src_port = temp_udp.dst_port;
1171 #ifdef DO_RFC_1812_CHECKS
1172 /* Check to make sure the packet is valid (RFC1812) */
1173 uint8_t valid_mask = MASK_ALL_PKTS;
1174 if (is_valid_pkt_ipv4(ipv4_hdr[0], m[0]->pkt_len) < 0) {
1175 rte_pktmbuf_free(m[0]);
1176 valid_mask &= EXCLUDE_1ST_PKT;
1178 if (is_valid_pkt_ipv4(ipv4_hdr[1], m[1]->pkt_len) < 0) {
1179 rte_pktmbuf_free(m[1]);
1180 valid_mask &= EXCLUDE_2ND_PKT;
1182 if (is_valid_pkt_ipv4(ipv4_hdr[2], m[2]->pkt_len) < 0) {
1183 rte_pktmbuf_free(m[2]);
1184 valid_mask &= EXCLUDE_3RD_PKT;
1186 if (is_valid_pkt_ipv4(ipv4_hdr[3], m[3]->pkt_len) < 0) {
1187 rte_pktmbuf_free(m[3]);
1188 valid_mask &= EXCLUDE_4TH_PKT;
1190 if (is_valid_pkt_ipv4(ipv4_hdr[4], m[4]->pkt_len) < 0) {
1191 rte_pktmbuf_free(m[4]);
1192 valid_mask &= EXCLUDE_5TH_PKT;
1194 if (is_valid_pkt_ipv4(ipv4_hdr[5], m[5]->pkt_len) < 0) {
1195 rte_pktmbuf_free(m[5]);
1196 valid_mask &= EXCLUDE_6TH_PKT;
1198 if (is_valid_pkt_ipv4(ipv4_hdr[6], m[6]->pkt_len) < 0) {
1199 rte_pktmbuf_free(m[6]);
1200 valid_mask &= EXCLUDE_7TH_PKT;
1202 if (is_valid_pkt_ipv4(ipv4_hdr[7], m[7]->pkt_len) < 0) {
1203 rte_pktmbuf_free(m[7]);
1204 valid_mask &= EXCLUDE_8TH_PKT;
1206 if (unlikely(valid_mask != MASK_ALL_PKTS)) {
1207 if (valid_mask == 0){
1211 for (i = 0; i < 8; i++) {
1212 if ((0x1 << i) & valid_mask) {
1213 l3fwd_simple_replay(m[i], portid, qconf);
1219 #endif // End of #ifdef DO_RFC_1812_CHECKS
1221 #ifdef DO_RFC_1812_CHECKS
1222 /* Update time to live and header checksum */
1223 --(ipv4_hdr[0]->time_to_live);
1224 --(ipv4_hdr[1]->time_to_live);
1225 --(ipv4_hdr[2]->time_to_live);
1226 --(ipv4_hdr[3]->time_to_live);
1227 ++(ipv4_hdr[0]->hdr_checksum);
1228 ++(ipv4_hdr[1]->hdr_checksum);
1229 ++(ipv4_hdr[2]->hdr_checksum);
1230 ++(ipv4_hdr[3]->hdr_checksum);
1231 --(ipv4_hdr[4]->time_to_live);
1232 --(ipv4_hdr[5]->time_to_live);
1233 --(ipv4_hdr[6]->time_to_live);
1234 --(ipv4_hdr[7]->time_to_live);
1235 ++(ipv4_hdr[4]->hdr_checksum);
1236 ++(ipv4_hdr[5]->hdr_checksum);
1237 ++(ipv4_hdr[6]->hdr_checksum);
1238 ++(ipv4_hdr[7]->hdr_checksum);
1241 for ( i = 0; i < 8; i++)
1243 /* if not already processed as a arp/icmp pkt */
1245 port->transmit_single_pkt(port, m[i]);
1246 tx_pkt_count[(uint64_t)port]++;
1252 static inline void get_ipv6_5tuple(struct rte_mbuf* m0, __m128i mask0, __m128i mask1,
1253 union ipv6_5tuple_host * key)
1255 __m128i tmpdata0 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len)));
1256 __m128i tmpdata1 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i)));
1257 __m128i tmpdata2 = _mm_loadu_si128(rte_pktmbuf_mtod_offset(m0, __m128i *, sizeof(struct ether_hdr) + offsetof(struct ipv6_hdr, payload_len) + sizeof(__m128i) + sizeof(__m128i)));
1258 key->xmm[0] = _mm_and_si128(tmpdata0, mask0);
1259 key->xmm[1] = tmpdata1;
1260 key->xmm[2] = _mm_and_si128(tmpdata2, mask1);
1265 simple_ipv6_replay_8pkts(struct rte_mbuf *m[8], uint8_t portid, struct lcore_conf *qconf)
1267 struct ether_hdr *eth_hdr[8],tmp;
1270 __attribute__((unused)) struct ipv6_hdr *ipv6_hdr[8], temp_ipv6;
1272 union ipv6_5tuple_host key[8];
1273 struct udp_hdr *udp_hdr[8];
1274 l2_phy_interface_t *port = ifm_get_port(portid);
1277 printf("port may be un initialized\n");
1281 for ( i = 0; i < 8; i++)
1283 a[i] = check_arpicmpv6(m[i]);
1286 eth_hdr[0] = rte_pktmbuf_mtod(m[0], struct ether_hdr *);
1287 eth_hdr[1] = rte_pktmbuf_mtod(m[1], struct ether_hdr *);
1288 eth_hdr[2] = rte_pktmbuf_mtod(m[2], struct ether_hdr *);
1289 eth_hdr[3] = rte_pktmbuf_mtod(m[3], struct ether_hdr *);
1290 eth_hdr[4] = rte_pktmbuf_mtod(m[4], struct ether_hdr *);
1291 eth_hdr[5] = rte_pktmbuf_mtod(m[5], struct ether_hdr *);
1292 eth_hdr[6] = rte_pktmbuf_mtod(m[6], struct ether_hdr *);
1293 eth_hdr[7] = rte_pktmbuf_mtod(m[7], struct ether_hdr *);
1295 memset(&tmp,0,sizeof (struct ether_hdr));
1300 ether_addr_copy(ð_hdr[i]->s_addr, &tmp.s_addr);
1301 ether_addr_copy(ð_hdr[i]->d_addr, ð_hdr[i]->s_addr);
1302 ether_addr_copy(&tmp.s_addr, ð_hdr[i]->d_addr);
1304 /* Handle IPv6 headers.*/
1305 ipv6_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct ipv6_hdr *,
1306 sizeof(struct ether_hdr));
1307 ipv6_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct ipv6_hdr *,
1308 sizeof(struct ether_hdr));
1309 ipv6_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct ipv6_hdr *,
1310 sizeof(struct ether_hdr));
1311 ipv6_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct ipv6_hdr *,
1312 sizeof(struct ether_hdr));
1313 ipv6_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct ipv6_hdr *,
1314 sizeof(struct ether_hdr));
1315 ipv6_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct ipv6_hdr *,
1316 sizeof(struct ether_hdr));
1317 ipv6_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct ipv6_hdr *,
1318 sizeof(struct ether_hdr));
1319 ipv6_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct ipv6_hdr *,
1320 sizeof(struct ether_hdr));
1323 memcpy(temp_ipv6.dst_addr,ipv6_hdr[i]->dst_addr,16);
1324 memcpy(ipv6_hdr[i]->dst_addr,ipv6_hdr[i]->src_addr,16);
1325 memcpy(ipv6_hdr[i]->src_addr,temp_ipv6.dst_addr,16);
1328 /* Handle UDP headers.*/
1329 udp_hdr[0] = rte_pktmbuf_mtod_offset(m[0], struct udp_hdr *,
1330 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1332 udp_hdr[1] = rte_pktmbuf_mtod_offset(m[1], struct udp_hdr *,
1333 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1334 udp_hdr[2] = rte_pktmbuf_mtod_offset(m[2], struct udp_hdr *,
1335 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1336 udp_hdr[3] = rte_pktmbuf_mtod_offset(m[3], struct udp_hdr *,
1337 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1338 udp_hdr[4] = rte_pktmbuf_mtod_offset(m[4], struct udp_hdr *,
1339 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1340 udp_hdr[5] = rte_pktmbuf_mtod_offset(m[5], struct udp_hdr *,
1341 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1342 udp_hdr[6] = rte_pktmbuf_mtod_offset(m[6], struct udp_hdr *,
1343 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1344 udp_hdr[7] = rte_pktmbuf_mtod_offset(m[7], struct udp_hdr *,
1345 sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr));
1346 /*1) memcpy or assignment.*/
1348 struct udp_hdr temp_udp;
1351 temp_udp.dst_port = udp_hdr[i]->dst_port;
1352 udp_hdr[i]->dst_port = udp_hdr[i]->src_port;
1353 udp_hdr[i]->src_port = temp_udp.dst_port;
1355 const void *key_array[8] = {&key[0], &key[1], &key[2], &key[3],
1356 &key[4], &key[5], &key[6], &key[7]};
1357 rte_hash_lookup_multi(qconf->ipv6_lookup_struct, &key_array[0], 8, ret);
1359 for ( i = 0; i < 8; i++)
1361 /* if not already processed as a arp/icmp pkt */
1363 port->transmit_single_pkt(port, m[i]);
1364 tx_pkt_count[(uint64_t)portid]++;
1369 #endif /* APP_LOOKUP_METHOD */
1371 static inline __attribute__((always_inline)) void
1372 l3fwd_simple_replay(struct rte_mbuf *m, uint8_t portid, struct lcore_conf *qconf)
1374 struct ether_hdr *eth_hdr,tmp;
1375 struct ipv4_hdr *ipv4_hdr,temp_ipv4;
1376 struct udp_hdr *udp_hdr,temp_udp;
1377 l2_phy_interface_t *port = ifm_get_port(portid);
1380 printf("port may be un initialized\n");
1385 printf("Null packet received\n");
1389 /* arp packet already processed return back */
1390 if (!check_arpicmp(m))
1394 printf("qconf configuration is NULL\n");
1396 eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
1397 ether_addr_copy(ð_hdr->s_addr, &tmp.s_addr);
1398 ether_addr_copy(ð_hdr->d_addr, ð_hdr->s_addr);
1399 ether_addr_copy(&tmp.s_addr, ð_hdr->d_addr);
1400 struct ether_hdr *eth_h = rte_pktmbuf_mtod(m, struct ether_hdr *);
1402 if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv4) {
1403 /* Handle IPv4 headers.*/
1404 ipv4_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
1405 sizeof(struct ether_hdr));
1406 temp_ipv4.dst_addr = ipv4_hdr->dst_addr;
1407 ipv4_hdr->dst_addr = ipv4_hdr->src_addr;
1408 ipv4_hdr->src_addr = temp_ipv4.dst_addr;
1410 #ifdef DO_RFC_1812_CHECKS
1411 /* Check to make sure the packet is valid (RFC1812) */
1412 if (is_valid_pkt_ipv4(ipv4_hdr, m->pkt_len) < 0) {
1413 printf("not of type 1812\n");
1414 rte_pktmbuf_free(m);
1419 #ifdef DO_RFC_1812_CHECKS
1420 /* Update time to live and header checksum */
1421 --(ipv4_hdr->time_to_live);
1422 ++(ipv4_hdr->hdr_checksum);
1424 /* Handle UDP headers.*/
1425 udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1426 (sizeof(struct ether_hdr)+sizeof(struct ipv4_hdr)));
1427 /*Swapping Src and Dst Port*/
1428 temp_udp.dst_port = udp_hdr->dst_port;
1429 udp_hdr->dst_port = udp_hdr->src_port;
1430 udp_hdr->src_port = temp_udp.dst_port;
1433 port->transmit_single_pkt(port, m);
1434 tx_pkt_count[portid]++;
1436 } else if ((rte_cpu_to_be_16(eth_h->ether_type)) == ETHER_TYPE_IPv6) {
1437 /* Handle IPv6 headers.*/
1438 struct ipv6_hdr *ipv6_hdr,temp_ipv6;
1439 /* Handle IPv4 headers.*/
1441 ipv6_hdr = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
1442 sizeof(struct ether_hdr));
1444 /*Swapping of Src and Dst IP address*/
1445 memcpy(temp_ipv6.dst_addr,ipv6_hdr->dst_addr,16);
1446 memcpy(ipv6_hdr->dst_addr,ipv6_hdr->src_addr,16);
1447 memcpy(ipv6_hdr->src_addr,temp_ipv6.dst_addr,16);
1449 /* Handle UDP headers.*/
1450 udp_hdr = rte_pktmbuf_mtod_offset(m, struct udp_hdr *,
1451 (sizeof(struct ether_hdr)+sizeof(struct ipv6_hdr)));
1452 /*Swapping Src and Dst Port*/
1453 temp_udp.dst_port = udp_hdr->dst_port;
1454 udp_hdr->dst_port = udp_hdr->src_port;
1455 udp_hdr->src_port = temp_udp.dst_port;
1456 send_single_packet(m, portid);
1458 /* Free the mbuf that contains non-IPV4/IPV6 packet */
1459 rte_pktmbuf_free(m);
1462 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1463 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1464 #ifdef DO_RFC_1812_CHECKS
1466 #define IPV4_MIN_VER_IHL 0x45
1467 #define IPV4_MAX_VER_IHL 0x4f
1468 #define IPV4_MAX_VER_IHL_DIFF (IPV4_MAX_VER_IHL - IPV4_MIN_VER_IHL)
1470 /* Minimum value of IPV4 total length (20B) in network byte order. */
1471 #define IPV4_MIN_LEN_BE (sizeof(struct ipv4_hdr) << 8)
1474 * From http://www.rfc-editor.org/rfc/rfc1812.txt section 5.2.2:
1475 * - The IP version number must be 4.
1476 * - The IP header length field must be large enough to hold the
1477 * minimum length legal IP datagram (20 bytes = 5 words).
1478 * - The IP total length field must be large enough to hold the IP
1479 * datagram header, whose length is specified in the IP header length
1481 * If we encounter invalid IPV4 packet, then set destination port for it
1482 * to BAD_PORT value.
1484 static inline __attribute__((always_inline)) void
1485 rfc1812_process(struct ipv4_hdr *ipv4_hdr, uint16_t *dp, uint32_t ptype)
1489 if (RTE_ETH_IS_IPV4_HDR(ptype)) {
1490 ihl = ipv4_hdr->version_ihl - IPV4_MIN_VER_IHL;
1492 ipv4_hdr->time_to_live--;
1493 ipv4_hdr->hdr_checksum++;
1495 if (ihl > IPV4_MAX_VER_IHL_DIFF ||
1496 ((uint8_t)ipv4_hdr->total_length == 0 &&
1497 ipv4_hdr->total_length < IPV4_MIN_LEN_BE)) {
1504 #define rfc1812_process(mb, dp) do { } while (0)
1505 #endif /* DO_RFC_1812_CHECKS */
1506 #endif /* APP_LOOKUP_LPM && ENABLE_MULTI_BUFFER_OPTIMIZE */
1509 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1510 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1512 static inline __attribute__((always_inline)) uint16_t
1513 get_dst_port(const struct lcore_conf *qconf, struct rte_mbuf *pkt,
1514 uint32_t dst_ipv4, uint8_t portid)
1517 struct ipv6_hdr *ipv6_hdr;
1518 struct ether_hdr *eth_hdr;
1520 if (RTE_ETH_IS_IPV4_HDR(pkt->packet_type)) {
1521 if (rte_lpm_lookup(qconf->ipv4_lookup_struct, dst_ipv4,
1524 } else if (RTE_ETH_IS_IPV6_HDR(pkt->packet_type)) {
1525 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1526 ipv6_hdr = (struct ipv6_hdr *)(eth_hdr + 1);
1527 if (rte_lpm6_lookup(qconf->ipv6_lookup_struct,
1528 ipv6_hdr->dst_addr, &next_hop) != 0)
1538 process_packet(struct lcore_conf *qconf, struct rte_mbuf *pkt,
1539 uint16_t *dst_port, uint8_t portid)
1541 struct ether_hdr *eth_hdr;
1542 struct ipv4_hdr *ipv4_hdr;
1547 eth_hdr = rte_pktmbuf_mtod(pkt, struct ether_hdr *);
1548 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1551 dst_ipv4 = ipv4_hdr->dst_addr;
1552 dst_ipv4 = rte_be_to_cpu_32(dst_ipv4);
1554 /*Changing the dp to incoming port*/
1555 dp = get_dst_port(qconf, pkt, dst_ipv4, portid);
1558 te = _mm_loadu_si128((__m128i *)eth_hdr);
1562 rfc1812_process(ipv4_hdr, dst_port, pkt->packet_type);
1564 te = _mm_blend_epi16(te, ve, MASK_ETH);
1565 _mm_storeu_si128((__m128i *)eth_hdr, te);
1567 /* Wont be using the following fucntion*/
1570 * Read packet_type and destination IPV4 addresses from 4 mbufs.
1573 processx4_step1(struct rte_mbuf *pkt[FWDSTEP],
1575 uint32_t *ipv4_flag)
1577 struct ipv4_hdr *ipv4_hdr;
1578 struct ether_hdr *eth_hdr;
1579 uint32_t x0, x1, x2, x3;
1581 eth_hdr = rte_pktmbuf_mtod(pkt[0], struct ether_hdr *);
1582 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1583 x0 = ipv4_hdr->dst_addr;
1584 ipv4_flag[0] = pkt[0]->packet_type & RTE_PTYPE_L3_IPV4;
1586 eth_hdr = rte_pktmbuf_mtod(pkt[1], struct ether_hdr *);
1587 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1588 x1 = ipv4_hdr->dst_addr;
1589 ipv4_flag[0] &= pkt[1]->packet_type;
1591 eth_hdr = rte_pktmbuf_mtod(pkt[2], struct ether_hdr *);
1592 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1593 x2 = ipv4_hdr->dst_addr;
1594 ipv4_flag[0] &= pkt[2]->packet_type;
1596 eth_hdr = rte_pktmbuf_mtod(pkt[3], struct ether_hdr *);
1597 ipv4_hdr = (struct ipv4_hdr *)(eth_hdr + 1);
1598 x3 = ipv4_hdr->dst_addr;
1599 ipv4_flag[0] &= pkt[3]->packet_type;
1601 dip[0] = _mm_set_epi32(x3, x2, x1, x0);
1605 * Lookup into LPM for destination port.
1606 * If lookup fails, use incoming port (portid) as destination port.
1609 processx4_step2(const struct lcore_conf *qconf,
1613 struct rte_mbuf *pkt[FWDSTEP],
1614 uint16_t dprt[FWDSTEP])
1617 const __m128i bswap_mask = _mm_set_epi8(12, 13, 14, 15, 8, 9, 10, 11,
1618 4, 5, 6, 7, 0, 1, 2, 3);
1620 /* Byte swap 4 IPV4 addresses. */
1621 dip = _mm_shuffle_epi8(dip, bswap_mask);
1623 /* if all 4 packets are IPV4. */
1624 if (likely(ipv4_flag)) {
1625 rte_lpm_lookupx4(qconf->ipv4_lookup_struct, dip, dprt, portid);
1628 dprt[0] = get_dst_port(qconf, pkt[0], dst.u32[0], portid);
1629 dprt[1] = get_dst_port(qconf, pkt[1], dst.u32[1], portid);
1630 dprt[2] = get_dst_port(qconf, pkt[2], dst.u32[2], portid);
1631 dprt[3] = get_dst_port(qconf, pkt[3], dst.u32[3], portid);
1636 * Update source and destination MAC addresses in the ethernet header.
1637 * Perform RFC1812 checks and updates for IPV4 packets.
1640 processx4_step3(struct rte_mbuf *pkt[FWDSTEP], uint16_t dst_port[FWDSTEP])
1642 __m128i te[FWDSTEP];
1643 __m128i ve[FWDSTEP];
1644 __m128i *p[FWDSTEP];
1646 p[0] = rte_pktmbuf_mtod(pkt[0], __m128i *);
1647 p[1] = rte_pktmbuf_mtod(pkt[1], __m128i *);
1648 p[2] = rte_pktmbuf_mtod(pkt[2], __m128i *);
1649 p[3] = rte_pktmbuf_mtod(pkt[3], __m128i *);
1651 ve[0] = val_eth[dst_port[0]];
1652 te[0] = _mm_loadu_si128(p[0]);
1654 ve[1] = val_eth[dst_port[1]];
1655 te[1] = _mm_loadu_si128(p[1]);
1657 ve[2] = val_eth[dst_port[2]];
1658 te[2] = _mm_loadu_si128(p[2]);
1660 ve[3] = val_eth[dst_port[3]];
1661 te[3] = _mm_loadu_si128(p[3]);
1663 /* Update first 12 bytes, keep rest bytes intact. */
1664 te[0] = _mm_blend_epi16(te[0], ve[0], MASK_ETH);
1665 te[1] = _mm_blend_epi16(te[1], ve[1], MASK_ETH);
1666 te[2] = _mm_blend_epi16(te[2], ve[2], MASK_ETH);
1667 te[3] = _mm_blend_epi16(te[3], ve[3], MASK_ETH);
1669 _mm_storeu_si128(p[0], te[0]);
1670 _mm_storeu_si128(p[1], te[1]);
1671 _mm_storeu_si128(p[2], te[2]);
1672 _mm_storeu_si128(p[3], te[3]);
1674 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[0] + 1),
1675 &dst_port[0], pkt[0]->packet_type);
1676 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[1] + 1),
1677 &dst_port[1], pkt[1]->packet_type);
1678 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[2] + 1),
1679 &dst_port[2], pkt[2]->packet_type);
1680 rfc1812_process((struct ipv4_hdr *)((struct ether_hdr *)p[3] + 1),
1681 &dst_port[3], pkt[3]->packet_type);
1685 * We group consecutive packets with the same destionation port into one burst.
1686 * To avoid extra latency this is done together with some other packet
1687 * processing, but after we made a final decision about packet's destination.
1688 * To do this we maintain:
1689 * pnum - array of number of consecutive packets with the same dest port for
1690 * each packet in the input burst.
1691 * lp - pointer to the last updated element in the pnum.
1692 * dlp - dest port value lp corresponds to.
1695 #define GRPSZ (1 << FWDSTEP)
1696 #define GRPMSK (GRPSZ - 1)
1698 #define GROUP_PORT_STEP(dlp, dcp, lp, pn, idx) do { \
1699 if (likely((dlp) == (dcp)[(idx)])) { \
1702 (dlp) = (dcp)[idx]; \
1703 (lp) = (pn) + (idx); \
1709 * Group consecutive packets with the same destination port in bursts of 4.
1710 * Suppose we have array of destionation ports:
1711 * dst_port[] = {a, b, c, d,, e, ... }
1712 * dp1 should contain: <a, b, c, d>, dp2: <b, c, d, e>.
1713 * We doing 4 comparisions at once and the result is 4 bit mask.
1714 * This mask is used as an index into prebuild array of pnum values.
1716 static inline uint16_t *
1717 port_groupx4(uint16_t pn[FWDSTEP + 1], uint16_t *lp, __m128i dp1, __m128i dp2)
1719 static const struct {
1720 uint64_t pnum; /* prebuild 4 values for pnum[]. */
1721 int32_t idx; /* index for new last updated elemnet. */
1722 uint16_t lpv; /* add value to the last updated element. */
1725 /* 0: a != b, b != c, c != d, d != e */
1726 .pnum = UINT64_C(0x0001000100010001),
1731 /* 1: a == b, b != c, c != d, d != e */
1732 .pnum = UINT64_C(0x0001000100010002),
1737 /* 2: a != b, b == c, c != d, d != e */
1738 .pnum = UINT64_C(0x0001000100020001),
1743 /* 3: a == b, b == c, c != d, d != e */
1744 .pnum = UINT64_C(0x0001000100020003),
1749 /* 4: a != b, b != c, c == d, d != e */
1750 .pnum = UINT64_C(0x0001000200010001),
1755 /* 5: a == b, b != c, c == d, d != e */
1756 .pnum = UINT64_C(0x0001000200010002),
1761 /* 6: a != b, b == c, c == d, d != e */
1762 .pnum = UINT64_C(0x0001000200030001),
1767 /* 7: a == b, b == c, c == d, d != e */
1768 .pnum = UINT64_C(0x0001000200030004),
1773 /* 8: a != b, b != c, c != d, d == e */
1774 .pnum = UINT64_C(0x0002000100010001),
1779 /* 9: a == b, b != c, c != d, d == e */
1780 .pnum = UINT64_C(0x0002000100010002),
1785 /* 0xa: a != b, b == c, c != d, d == e */
1786 .pnum = UINT64_C(0x0002000100020001),
1791 /* 0xb: a == b, b == c, c != d, d == e */
1792 .pnum = UINT64_C(0x0002000100020003),
1797 /* 0xc: a != b, b != c, c == d, d == e */
1798 .pnum = UINT64_C(0x0002000300010001),
1803 /* 0xd: a == b, b != c, c == d, d == e */
1804 .pnum = UINT64_C(0x0002000300010002),
1809 /* 0xe: a != b, b == c, c == d, d == e */
1810 .pnum = UINT64_C(0x0002000300040001),
1815 /* 0xf: a == b, b == c, c == d, d == e */
1816 .pnum = UINT64_C(0x0002000300040005),
1823 uint16_t u16[FWDSTEP + 1];
1825 } *pnum = (void *)pn;
1829 dp1 = _mm_cmpeq_epi16(dp1, dp2);
1830 dp1 = _mm_unpacklo_epi16(dp1, dp1);
1831 v = _mm_movemask_ps((__m128)dp1);
1833 /* update last port counter. */
1834 lp[0] += gptbl[v].lpv;
1836 /* if dest port value has changed. */
1838 lp = pnum->u16 + gptbl[v].idx;
1840 pnum->u64 = gptbl[v].pnum;
1846 #endif /* APP_LOOKUP_METHOD */
1848 /* main processing loop */
1850 main_loop(__attribute__((unused)) void *dummy)
1853 uint8_t portid, queueid;
1854 struct lcore_conf *qconf;
1856 l2_phy_interface_t *port;
1857 struct rte_mbuf *pkts_burst[IFM_BURST_SIZE];
1858 uint32_t nb_tx = 0, nb_rx, j, i;
1859 const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /US_PER_S * BURST_TX_DRAIN_US;
1860 uint64_t prev_tsc = 0, cur_tsc, diff_tsc;
1862 #if ((APP_LOOKUP_METHOD == APP_LOOKUP_LPM) && \
1863 (ENABLE_MULTI_BUFFER_OPTIMIZE == 1))
1867 uint16_t dst_port[MAX_PKT_BURST];
1868 __m128i dip[MAX_PKT_BURST / FWDSTEP];
1869 uint32_t ipv4_flag[MAX_PKT_BURST / FWDSTEP];
1870 uint16_t pnum[MAX_PKT_BURST + 1];
1875 lcore_id = rte_lcore_id();
1876 qconf = &lcore_conf[lcore_id];
1878 if (qconf->n_rx_queue == 0) {
1879 RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", lcore_id);
1883 RTE_LOG(INFO, L3FWD, "entering main loop on lcore %u\n", lcore_id);
1884 for (i = 0; i < qconf->n_rx_queue; i++) {
1885 portid = qconf->rx_queue_list[i].port_id;
1886 queueid = qconf->rx_queue_list[i].queue_id;
1887 RTE_LOG(INFO, L3FWD, " -- lcoreid=%u portid=%hhu rxqueueid=%hhu\n", lcore_id,
1893 cur_tsc = rte_rdtsc();
1896 * TX burst queue drain
1898 diff_tsc = cur_tsc - prev_tsc;
1899 if (unlikely(diff_tsc > drain_tsc)) {
1902 * This could be optimized (use queueid instead of
1903 * portid), but it is not called so often
1905 for (portid = 0; portid < num_ports; portid++) {
1906 port = ifm_get_port(portid);
1910 if(port->tx_buf_len > 0){
1911 RTE_SET_USED(nb_tx);
1912 port->tx_buf_len = 0;
1922 * Read packet from RX queues
1924 for (i = 0; i < qconf->n_rx_queue; ++i) {
1925 portid = qconf->rx_queue_list[i].port_id;
1926 queueid = qconf->rx_queue_list[i].queue_id;
1927 port = ifm_get_port(portid);
1929 nb_rx = port->retrieve_bulk_pkts(portid,
1930 queueid, pkts_burst);
1931 port->n_rxpkts += nb_rx;
1933 printf("port may be un initialized\n");
1938 rcv_pkt_count[portid] += nb_rx;
1943 #if (ENABLE_MULTI_BUFFER_OPTIMIZE == 1)
1944 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
1947 * Send nb_rx - nb_rx%8 packets
1950 uint32_t n = RTE_ALIGN_FLOOR(nb_rx, 8);
1951 for (j = 0; j < n; j += 8) {
1953 pkts_burst[j]->packet_type &
1954 pkts_burst[j+1]->packet_type &
1955 pkts_burst[j+2]->packet_type &
1956 pkts_burst[j+3]->packet_type &
1957 pkts_burst[j+4]->packet_type &
1958 pkts_burst[j+5]->packet_type &
1959 pkts_burst[j+6]->packet_type &
1960 pkts_burst[j+7]->packet_type;
1961 if (pkt_type & RTE_PTYPE_L3_IPV4) {
1962 simple_ipv4_replay_8pkts(
1963 &pkts_burst[j], portid, qconf);
1964 } else if (pkt_type &
1965 RTE_PTYPE_L3_IPV6) {
1966 simple_ipv6_replay_8pkts(&pkts_burst[j],
1969 for (i = j; i < j + 8; i++) {
1970 l3fwd_simple_replay(pkts_burst[i],
1975 for (; j < nb_rx ; j++) {
1976 l3fwd_simple_replay(pkts_burst[j],
1980 #elif (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
1982 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1983 for (j = 0; j != k; j += FWDSTEP) {
1984 processx4_step1(&pkts_burst[j],
1986 &ipv4_flag[j / FWDSTEP]);
1989 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
1990 for (j = 0; j != k; j += FWDSTEP) {
1991 processx4_step2(qconf, dip[j / FWDSTEP],
1992 ipv4_flag[j / FWDSTEP], portid,
1993 &pkts_burst[j], &dst_port[j]);
1997 * Finish packet processing and group consecutive
1998 * packets with the same destination port.
2000 k = RTE_ALIGN_FLOOR(nb_rx, FWDSTEP);
2007 processx4_step3(pkts_burst, dst_port);
2009 /* dp1: <d[0], d[1], d[2], d[3], ... > */
2010 dp1 = _mm_loadu_si128((__m128i *)dst_port);
2012 for (j = FWDSTEP; j != k; j += FWDSTEP) {
2013 processx4_step3(&pkts_burst[j],
2018 * <d[j-3], d[j-2], d[j-1], d[j], ... >
2020 dp2 = _mm_loadu_si128((__m128i *)
2021 &dst_port[j - FWDSTEP + 1]);
2022 lp = port_groupx4(&pnum[j - FWDSTEP],
2027 * <d[j], d[j+1], d[j+2], d[j+3], ... >
2029 dp1 = _mm_srli_si128(dp2,
2031 sizeof(dst_port[0]));
2035 * dp2: <d[j-3], d[j-2], d[j-1], d[j-1], ... >
2037 dp2 = _mm_shufflelo_epi16(dp1, 0xf9);
2038 lp = port_groupx4(&pnum[j - FWDSTEP], lp,
2042 * remove values added by the last repeated
2046 dlp = dst_port[j - 1];
2048 /* set dlp and lp to the never used values. */
2050 lp = pnum + MAX_PKT_BURST;
2053 /* Process up to last 3 packets one by one. */
2054 switch (nb_rx % FWDSTEP) {
2056 process_packet(qconf, pkts_burst[j],
2057 dst_port + j, portid);
2058 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2061 process_packet(qconf, pkts_burst[j],
2062 dst_port + j, portid);
2063 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2066 process_packet(qconf, pkts_burst[j],
2067 dst_port + j, portid);
2068 GROUP_PORT_STEP(dlp, dst_port, lp, pnum, j);
2073 * Send packets out, through destination port.
2074 * Consecuteve pacekts with the same destination port
2075 * are already grouped together.
2076 * If destination port for the packet equals BAD_PORT,
2077 * then free the packet without sending it out.
2079 for (j = 0; j < nb_rx; j += k) {
2087 if (likely(pn != BAD_PORT)) {
2088 send_packetsx4(qconf, pn,
2091 for (m = j; m != j + k; m++)
2092 rte_pktmbuf_free(pkts_burst[m]);
2096 #endif /* APP_LOOKUP_METHOD */
2097 #else /* ENABLE_MULTI_BUFFER_OPTIMIZE == 0 */
2099 /* Prefetch first packets */
2100 for (j = 0; j < PREFETCH_OFFSET && j < nb_rx; j++) {
2101 rte_prefetch0(rte_pktmbuf_mtod(
2102 pkts_burst[j], void *));
2105 /* Prefetch and forward already prefetched packets */
2106 for (j = 0; j < (nb_rx - PREFETCH_OFFSET); j++) {
2107 rte_prefetch0(rte_pktmbuf_mtod(pkts_burst[
2108 j + PREFETCH_OFFSET], void *));
2109 l3fwd_simple_replay(pkts_burst[j], portid,
2113 /* Forward remaining prefetched packets */
2114 for (; j < nb_rx; j++) {
2115 l3fwd_simple_replay(pkts_burst[j], portid,
2118 #endif /* ENABLE_MULTI_BUFFER_OPTIMIZE */
2132 printf ("UDP_Replay stats:\n");
2133 printf ("--------------\n");
2134 printf (" Port Rx Packet Tx Packet Rx Pkt Drop Tx Pkt Drop arp_pkts\n");
2135 for (i = 0; i < nb_lcore_params; ++i) {
2136 portid = lcore_params[i].port_id;
2137 printf (" %u %"PRId64" %"PRId64" 0 0 %"PRId64"",portid, rcv_pkt_count[(uint64_t)portid], tx_pkt_count[(uint64_t)portid], arp_pkts[(uint64_t)portid]);
2149 for (i = 0; i < 32; i++) {
2150 rcv_pkt_count[i] = 0;
2151 tx_pkt_count[i] = 0;
2158 check_lcore_params(void)
2160 uint8_t queue, lcore;
2164 for (i = 0; i < nb_lcore_params; ++i) {
2165 queue = lcore_params[i].queue_id;
2166 if (queue >= MAX_RX_QUEUE_PER_PORT) {
2167 printf("invalid queue number: %hhu\n", queue);
2170 lcore = lcore_params[i].lcore_id;
2171 if (!rte_lcore_is_enabled(lcore)) {
2172 printf("error: lcore %hhu is not enabled in lcore mask\n", lcore);
2175 if ((socketid = rte_lcore_to_socket_id(lcore) != 0) &&
2177 printf("warning: lcore %hhu is on socket %d with numa off \n",
2185 check_port_config(const unsigned nb_ports)
2190 for (i = 0; i < nb_lcore_params; ++i) {
2191 portid = lcore_params[i].port_id;
2192 if ((enabled_port_mask & (1 << portid)) == 0) {
2193 printf("port %u is not enabled in port mask\n", portid);
2196 if (portid >= nb_ports) {
2197 printf("port %u is not present on the board\n", portid);
2205 get_port_n_rx_queues(const uint8_t port)
2210 for (i = 0; i < nb_lcore_params; ++i) {
2211 if (lcore_params[i].port_id == port && lcore_params[i].queue_id > queue)
2212 queue = lcore_params[i].queue_id;
2214 return (uint8_t)(++queue);
2218 init_lcore_rx_queues(void)
2220 uint16_t i, nb_rx_queue;
2223 for (i = 0; i < nb_lcore_params; ++i) {
2224 lcore = lcore_params[i].lcore_id;
2225 nb_rx_queue = lcore_conf[lcore].n_rx_queue;
2226 if (nb_rx_queue >= MAX_RX_QUEUE_PER_LCORE) {
2227 printf("error: too many queues (%u) for lcore: %u\n",
2228 (unsigned)nb_rx_queue + 1, (unsigned)lcore);
2231 lcore_conf[lcore].rx_queue_list[nb_rx_queue].port_id =
2232 lcore_params[i].port_id;
2233 lcore_conf[lcore].rx_queue_list[nb_rx_queue].queue_id =
2234 lcore_params[i].queue_id;
2235 lcore_conf[lcore].n_rx_queue++;
2243 print_usage(const char *prgname)
2245 printf ("%s [EAL options] -- -p PORTMASK -P"
2246 " [--config (port,queue,lcore)[,(port,queue,lcore]]"
2247 " [--enable-jumbo [--max-pkt-len PKTLEN]]\n"
2248 " -p PORTMASK: hexadecimal bitmask of ports to configure\n"
2249 " -P : enable promiscuous mode\n"
2250 " --config (port,queue,lcore): rx queues configuration\n"
2251 " --eth-dest=X,MM:MM:MM:MM:MM:MM: optional, ethernet destination for port X\n"
2252 " --no-numa: optional, disable numa awareness\n"
2253 " --no-hw-csum: optional, disable hw ip checksum\n"
2254 " --ipv6: optional, specify it if running ipv6 packets\n"
2255 " --enable-jumbo: enable jumbo frame"
2256 " which max packet len is PKTLEN in decimal (64-9600)\n"
2257 " --hash-entry-num: specify the hash entry number in hexadecimal to be setup\n",
2261 static int parse_max_pkt_len(const char *pktlen)
2266 /* parse decimal string */
2267 len = strtoul(pktlen, &end, 10);
2268 if ((pktlen[0] == '\0') || (end == NULL) || (*end != '\0'))
2278 parse_link_ip(const char *file_name)
2282 struct rte_cfgfile *file;
2286 file = rte_cfgfile_load(file_name, 0);
2288 entry = rte_cfgfile_get_entry(file, "linkip", "num_ports");
2289 numports = (uint32_t)atoi(entry);
2290 if (numports <= 0 || numports > 32)
2291 rte_panic("numports is not valid\n");
2293 entry = rte_cfgfile_get_entry(file, "linkip", "ip_type");
2294 type = (uint32_t)atoi(entry);
2296 for (i = 0;i < numports; i++) {
2297 sprintf(buf, "port%d", i);
2298 entry = rte_cfgfile_get_entry(file, "linkip", buf);
2304 ipv4[i] = strdup(entry);
2306 my_inet_pton_ipv6(AF_INET6, entry, &link_ipv6[i][0]);
2313 parse_portmask(const char *portmask)
2318 /* parse hexadecimal string */
2319 pm = strtoul(portmask, &end, 16);
2320 if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0'))
2329 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2331 parse_hash_entry_number(const char *hash_entry_num)
2334 unsigned long hash_en;
2335 /* parse hexadecimal string */
2336 hash_en = strtoul(hash_entry_num, &end, 16);
2337 if ((hash_entry_num[0] == '\0') || (end == NULL) || (*end != '\0'))
2348 parse_config(const char *q_arg)
2351 const char *p, *p0 = q_arg;
2359 unsigned long int_fld[_NUM_FLD];
2360 char *str_fld[_NUM_FLD];
2364 nb_lcore_params = 0;
2366 while ((p = strchr(p0,'(')) != NULL) {
2368 if((p0 = strchr(p,')')) == NULL)
2372 if(size >= sizeof(s))
2375 snprintf(s, sizeof(s), "%.*s", size, p);
2376 if (rte_strsplit(s, sizeof(s), str_fld, _NUM_FLD, ',') != _NUM_FLD)
2378 for (i = 0; i < _NUM_FLD; i++){
2380 int_fld[i] = strtoul(str_fld[i], &end, 0);
2381 if (errno != 0 || end == str_fld[i] || int_fld[i] > 255)
2384 if (nb_lcore_params >= MAX_LCORE_PARAMS) {
2385 printf("exceeded max number of lcore params: %hu\n",
2389 lcore_params_array[nb_lcore_params].port_id = (uint8_t)int_fld[FLD_PORT];
2390 lcore_params_array[nb_lcore_params].queue_id = (uint8_t)int_fld[FLD_QUEUE];
2391 lcore_params_array[nb_lcore_params].lcore_id = (uint8_t)int_fld[FLD_LCORE];
2394 lcore_params = lcore_params_array;
2399 parse_eth_dest(const char *optarg)
2403 uint8_t c, *dest, peer_addr[6];
2406 portid = strtoul(optarg, &port_end, 10);
2407 if (errno != 0 || port_end == optarg || *port_end++ != ',')
2408 rte_exit(EXIT_FAILURE,
2409 "Invalid eth-dest: %s", optarg);
2410 if (portid >= RTE_MAX_ETHPORTS)
2411 rte_exit(EXIT_FAILURE,
2412 "eth-dest: port %d >= RTE_MAX_ETHPORTS(%d)\n",
2413 portid, RTE_MAX_ETHPORTS);
2415 if (cmdline_parse_etheraddr(NULL, port_end,
2416 &peer_addr, sizeof(peer_addr)) < 0)
2417 rte_exit(EXIT_FAILURE,
2418 "Invalid ethernet address: %s\n",
2420 dest = (uint8_t *)&dest_eth_addr[portid];
2421 for (c = 0; c < 6; c++)
2422 dest[c] = peer_addr[c];
2423 *(uint64_t *)(val_eth + portid) = dest_eth_addr[portid];
2426 #define CMD_LINE_OPT_CONFIG "config"
2427 #define CMD_LINE_OPT_ETH_DEST "eth-dest"
2428 #define CMD_LINE_OPT_NO_NUMA "no-numa"
2429 #define CMD_LINE_OPT_NO_HW_CSUM "no-hw-csum"
2430 #define CMD_LINE_OPT_IPV6 "ipv6"
2431 #define CMD_LINE_OPT_ENABLE_JUMBO "enable-jumbo"
2432 #define CMD_LINE_OPT_HASH_ENTRY_NUM "hash-entry-num"
2434 /* Parse the argument given in the command line of the application */
2436 parse_args(int argc, char **argv)
2441 char *prgname = argv[0];
2442 static struct option lgopts[] = {
2443 {CMD_LINE_OPT_CONFIG, 1, 0, 0},
2444 {CMD_LINE_OPT_ETH_DEST, 1, 0, 0},
2445 {CMD_LINE_OPT_NO_NUMA, 0, 0, 0},
2446 {CMD_LINE_OPT_NO_HW_CSUM, 0, 0, 0},
2447 {CMD_LINE_OPT_IPV6, 0, 0, 0},
2448 {CMD_LINE_OPT_ENABLE_JUMBO, 0, 0, 0},
2449 {CMD_LINE_OPT_HASH_ENTRY_NUM, 1, 0, 0},
2455 while ((opt = getopt_long(argc, argvopt, "s:p:P",
2456 lgopts, &option_index)) != EOF) {
2460 parse_link_ip(optarg);
2464 enabled_port_mask = parse_portmask(optarg);
2465 if (enabled_port_mask == 0) {
2466 printf("invalid portmask\n");
2467 print_usage(prgname);
2472 printf("Promiscuous mode selected\n");
2478 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_CONFIG,
2479 sizeof (CMD_LINE_OPT_CONFIG))) {
2480 ret = parse_config(optarg);
2482 printf("invalid config\n");
2483 print_usage(prgname);
2488 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ETH_DEST,
2489 sizeof(CMD_LINE_OPT_ETH_DEST))) {
2490 parse_eth_dest(optarg);
2493 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_NUMA,
2494 sizeof(CMD_LINE_OPT_NO_NUMA))) {
2495 printf("numa is disabled \n");
2499 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_NO_HW_CSUM,
2500 sizeof(CMD_LINE_OPT_NO_HW_CSUM))) {
2501 printf("numa is hw ip checksum \n");
2502 port_conf.rxmode.hw_ip_checksum = 0;
2503 rx_conf.rx_free_thresh = 30;
2507 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2508 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_IPV6,
2509 sizeof(CMD_LINE_OPT_IPV6))) {
2510 printf("ipv6 is specified \n");
2515 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_ENABLE_JUMBO,
2516 sizeof (CMD_LINE_OPT_ENABLE_JUMBO))) {
2517 struct option lenopts = {"max-pkt-len", required_argument, 0, 0};
2519 printf("jumbo frame is enabled - disabling simple TX path\n");
2520 port_conf.rxmode.jumbo_frame = 1;
2522 /* if no max-pkt-len set, use the default value ETHER_MAX_LEN */
2523 if (0 == getopt_long(argc, argvopt, "", &lenopts, &option_index)) {
2524 ret = parse_max_pkt_len(optarg);
2525 if ((ret < 64) || (ret > MAX_JUMBO_PKT_LEN)){
2526 printf("invalid packet length\n");
2527 print_usage(prgname);
2530 port_conf.rxmode.max_rx_pkt_len = ret;
2532 printf("set jumbo frame max packet length to %u\n",
2533 (unsigned int)port_conf.rxmode.max_rx_pkt_len);
2535 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2536 if (!strncmp(lgopts[option_index].name, CMD_LINE_OPT_HASH_ENTRY_NUM,
2537 sizeof(CMD_LINE_OPT_HASH_ENTRY_NUM))) {
2538 ret = parse_hash_entry_number(optarg);
2539 if ((ret > 0) && (ret <= L3FWD_HASH_ENTRIES)) {
2540 hash_entry_number = ret;
2542 printf("invalid hash entry number\n");
2543 print_usage(prgname);
2551 print_usage(prgname);
2557 argv[optind-1] = prgname;
2560 optind = 0; /* reset getopt lib */
2564 #if (APP_LOOKUP_METHOD == APP_LOOKUP_EXACT_MATCH)
2566 static void convert_ipv4_5tuple(struct ipv4_5tuple* key1,
2567 union ipv4_5tuple_host* key2)
2569 key2->ip_dst = rte_cpu_to_be_32(key1->ip_dst);
2570 key2->ip_src = rte_cpu_to_be_32(key1->ip_src);
2571 key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2572 key2->port_src = rte_cpu_to_be_16(key1->port_src);
2573 key2->proto = key1->proto;
2579 static void convert_ipv6_5tuple(struct ipv6_5tuple* key1,
2580 union ipv6_5tuple_host* key2)
2583 for (i = 0; i < 16; i++)
2585 key2->ip_dst[i] = key1->ip_dst[i];
2586 key2->ip_src[i] = key1->ip_src[i];
2588 key2->port_dst = rte_cpu_to_be_16(key1->port_dst);
2589 key2->port_src = rte_cpu_to_be_16(key1->port_src);
2590 key2->proto = key1->proto;
2597 #define BYTE_VALUE_MAX 256
2598 #define ALL_32_BITS 0xffffffff
2599 #define BIT_8_TO_15 0x0000ff00
2601 populate_ipv4_few_flow_into_table(const struct rte_hash* h)
2605 uint32_t array_len = sizeof(ipv4_l3fwd_route_array)/sizeof(ipv4_l3fwd_route_array[0]);
2607 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2608 for (i = 0; i < array_len; i++) {
2609 struct ipv4_l3fwd_route entry;
2610 union ipv4_5tuple_host newkey;
2611 entry = ipv4_l3fwd_route_array[i];
2612 convert_ipv4_5tuple(&entry.key, &newkey);
2613 ret = rte_hash_add_key (h,(void *) &newkey);
2615 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2616 " to the l3fwd hash.\n", i);
2618 ipv4_l3fwd_out_if[ret] = entry.if_out;
2620 printf("Hash: Adding 0x%" PRIx32 " keys\n", array_len);
2623 #define BIT_16_TO_23 0x00ff0000
2625 populate_ipv6_few_flow_into_table(const struct rte_hash* h)
2629 uint32_t array_len = sizeof(ipv6_l3fwd_route_array)/sizeof(ipv6_l3fwd_route_array[0]);
2631 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2632 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2633 for (i = 0; i < array_len; i++) {
2634 struct ipv6_l3fwd_route entry;
2635 union ipv6_5tuple_host newkey;
2636 entry = ipv6_l3fwd_route_array[i];
2637 convert_ipv6_5tuple(&entry.key, &newkey);
2638 ret = rte_hash_add_key (h, (void *) &newkey);
2640 rte_exit(EXIT_FAILURE, "Unable to add entry %" PRIu32
2641 " to the l3fwd hash.\n", i);
2643 ipv6_l3fwd_out_if[ret] = entry.if_out;
2645 printf("Hash: Adding 0x%" PRIx32 "keys\n", array_len);
2648 #define NUMBER_PORT_USED 4
2650 populate_ipv4_many_flow_into_table(const struct rte_hash* h,
2651 unsigned int nr_flow)
2654 mask0 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_8_TO_15);
2655 for (i = 0; i < nr_flow; i++) {
2656 struct ipv4_l3fwd_route entry;
2657 union ipv4_5tuple_host newkey;
2658 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2659 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2660 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2661 /* Create the ipv4 exact match flow */
2662 memset(&entry, 0, sizeof(entry));
2663 switch (i & (NUMBER_PORT_USED -1)) {
2665 entry = ipv4_l3fwd_route_array[0];
2666 entry.key.ip_dst = IPv4(101,c,b,a);
2669 entry = ipv4_l3fwd_route_array[1];
2670 entry.key.ip_dst = IPv4(201,c,b,a);
2673 entry = ipv4_l3fwd_route_array[2];
2674 entry.key.ip_dst = IPv4(111,c,b,a);
2677 entry = ipv4_l3fwd_route_array[3];
2678 entry.key.ip_dst = IPv4(211,c,b,a);
2681 convert_ipv4_5tuple(&entry.key, &newkey);
2682 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2684 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2686 ipv4_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
2689 printf("Hash: Adding 0x%x keys\n", nr_flow);
2693 populate_ipv6_many_flow_into_table(const struct rte_hash* h,
2694 unsigned int nr_flow)
2697 mask1 = _mm_set_epi32(ALL_32_BITS, ALL_32_BITS, ALL_32_BITS, BIT_16_TO_23);
2698 mask2 = _mm_set_epi32(0, 0, ALL_32_BITS, ALL_32_BITS);
2699 for (i = 0; i < nr_flow; i++) {
2700 struct ipv6_l3fwd_route entry;
2701 union ipv6_5tuple_host newkey;
2702 uint8_t a = (uint8_t) ((i/NUMBER_PORT_USED)%BYTE_VALUE_MAX);
2703 uint8_t b = (uint8_t) (((i/NUMBER_PORT_USED)/BYTE_VALUE_MAX)%BYTE_VALUE_MAX);
2704 uint8_t c = (uint8_t) ((i/NUMBER_PORT_USED)/(BYTE_VALUE_MAX*BYTE_VALUE_MAX));
2705 /* Create the ipv6 exact match flow */
2706 memset(&entry, 0, sizeof(entry));
2707 switch (i & (NUMBER_PORT_USED - 1)) {
2708 case 0: entry = ipv6_l3fwd_route_array[0]; break;
2709 case 1: entry = ipv6_l3fwd_route_array[1]; break;
2710 case 2: entry = ipv6_l3fwd_route_array[2]; break;
2711 case 3: entry = ipv6_l3fwd_route_array[3]; break;
2713 entry.key.ip_dst[13] = c;
2714 entry.key.ip_dst[14] = b;
2715 entry.key.ip_dst[15] = a;
2716 convert_ipv6_5tuple(&entry.key, &newkey);
2717 int32_t ret = rte_hash_add_key(h,(void *) &newkey);
2719 rte_exit(EXIT_FAILURE, "Unable to add entry %u\n", i);
2721 ipv6_l3fwd_out_if[ret] = (uint8_t) entry.if_out;
2724 printf("Hash: Adding 0x%x keys\n", nr_flow);
2729 #if (APP_LOOKUP_METHOD == APP_LOOKUP_LPM)
2731 setup_lpm(int socketid)
2733 struct rte_lpm6_config config;
2738 /* create the LPM table */
2739 snprintf(s, sizeof(s), "IPV4_L3FWD_LPM_%d", socketid);
2740 ipv4_l3fwd_lookup_struct[socketid] = rte_lpm_create(s, socketid,
2741 IPV4_L3FWD_LPM_MAX_RULES, 0);
2742 if (ipv4_l3fwd_lookup_struct[socketid] == NULL)
2743 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
2744 " on socket %d\n", socketid);
2746 /* populate the LPM table */
2747 for (i = 0; i < IPV4_L3FWD_NUM_ROUTES; i++) {
2749 /* skip unused ports */
2750 if ((1 << ipv4_l3fwd_route_array[i].if_out &
2751 enabled_port_mask) == 0)
2754 ret = rte_lpm_add(ipv4_l3fwd_lookup_struct[socketid],
2755 ipv4_l3fwd_route_array[i].ip,
2756 ipv4_l3fwd_route_array[i].depth,
2757 ipv4_l3fwd_route_array[i].if_out);
2760 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2761 "l3fwd LPM table on socket %d\n",
2765 printf("LPM: Adding route 0x%08x / %d (%d)\n",
2766 (unsigned)ipv4_l3fwd_route_array[i].ip,
2767 ipv4_l3fwd_route_array[i].depth,
2768 ipv4_l3fwd_route_array[i].if_out);
2771 /* create the LPM6 table */
2772 snprintf(s, sizeof(s), "IPV6_L3FWD_LPM_%d", socketid);
2774 config.max_rules = IPV6_L3FWD_LPM_MAX_RULES;
2775 config.number_tbl8s = IPV6_L3FWD_LPM_NUMBER_TBL8S;
2777 ipv6_l3fwd_lookup_struct[socketid] = rte_lpm6_create(s, socketid,
2779 if (ipv6_l3fwd_lookup_struct[socketid] == NULL)
2780 rte_exit(EXIT_FAILURE, "Unable to create the l3fwd LPM table"
2781 " on socket %d\n", socketid);
2783 /* populate the LPM table */
2784 for (i = 0; i < IPV6_L3FWD_NUM_ROUTES; i++) {
2786 /* skip unused ports */
2787 if ((1 << ipv6_l3fwd_route_array[i].if_out &
2788 enabled_port_mask) == 0)
2791 ret = rte_lpm6_add(ipv6_l3fwd_lookup_struct[socketid],
2792 ipv6_l3fwd_route_array[i].ip,
2793 ipv6_l3fwd_route_array[i].depth,
2794 ipv6_l3fwd_route_array[i].if_out);
2797 rte_exit(EXIT_FAILURE, "Unable to add entry %u to the "
2798 "l3fwd LPM table on socket %d\n",
2802 printf("LPM: Adding route %s / %d (%d)\n",
2804 ipv6_l3fwd_route_array[i].depth,
2805 ipv6_l3fwd_route_array[i].if_out);
2810 /* Check the link status of all ports in up to 9s, and print them finally */
2812 check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
2814 #define CHECK_INTERVAL 100 /* 100ms */
2815 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
2816 uint8_t portid, count, all_ports_up, print_flag = 0;
2817 struct rte_eth_link link;
2819 printf("\nChecking link status");
2821 for (count = 0; count <= MAX_CHECK_TIME; count++) {
2823 for (portid = 0; portid < port_num; portid++) {
2824 if ((port_mask & (1 << portid)) == 0)
2826 memset(&link, 0, sizeof(link));
2827 rte_eth_link_get_nowait(portid, &link);
2828 /* print link status if flag set */
2829 if (print_flag == 1) {
2830 if (link.link_status)
2831 printf("Port %d Link Up - speed %u "
2832 "Mbps - %s\n", (uint8_t)portid,
2833 (unsigned)link.link_speed,
2834 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
2835 ("full-duplex") : ("half-duplex\n"));
2837 printf("Port %d Link Down\n",
2841 /* clear all_ports_up flag if any link down */
2842 if (link.link_status == 0) {
2847 /* after finally printing all link status, get out */
2848 if (print_flag == 1)
2851 if (all_ports_up == 0) {
2854 rte_delay_ms(CHECK_INTERVAL);
2857 /* set the print_flag if all ports up or timeout */
2858 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
2866 main(int argc, char **argv)
2871 uint32_t n_tx_queue, nb_lcores;
2872 uint8_t portid, nb_rx_queue;
2875 struct pipeline_params *params;
2878 ret = rte_eal_init(argc, argv);
2880 rte_exit(EXIT_FAILURE, "Invalid EAL parameters\n");
2884 timer_lcore = rte_lcore_id();
2886 /* parse application arguments (after the EAL ones) */
2887 ret = parse_args(argc, argv);
2889 rte_exit(EXIT_FAILURE, "Invalid L3FWD parameters\n");
2891 if (check_lcore_params() < 0)
2892 rte_exit(EXIT_FAILURE, "check_lcore_params failed\n");
2894 ret = init_lcore_rx_queues();
2896 rte_exit(EXIT_FAILURE, "init_lcore_rx_queues failed\n");
2898 params = rte_malloc(NULL, sizeof(*params), RTE_CACHE_LINE_SIZE);
2900 memcpy(params, &def_pipeline_params, sizeof(def_pipeline_params));
2901 lib_arp_init(params, NULL);
2903 /* configure the interface manager */
2906 nb_ports = rte_eth_dev_count();
2907 num_ports = nb_ports;
2908 if (nb_ports > RTE_MAX_ETHPORTS)
2909 nb_ports = RTE_MAX_ETHPORTS;
2911 if (check_port_config(nb_ports) < 0)
2912 rte_exit(EXIT_FAILURE, "check_port_config failed\n");
2914 nb_lcores = rte_lcore_count();
2917 *Configuring port_config_t structure for interface manager initialization
2919 size = RTE_CACHE_LINE_ROUNDUP(sizeof(port_config_t));
2920 port_config = rte_zmalloc(NULL, (RTE_MAX_ETHPORTS * size), RTE_CACHE_LINE_SIZE);
2921 if (port_config == NULL)
2922 rte_panic("port_config is NULL: Memory Allocation failure\n");
2924 /* initialize all ports */
2925 for (portid = 0; portid < nb_ports; portid++) {
2926 /* skip ports that are not enabled */
2927 if ((enabled_port_mask & (1 << portid)) == 0) {
2928 printf("\nSkipping disabled port %d\n", portid);
2934 printf("Initializing port %d ... ", portid );
2937 nb_rx_queue = get_port_n_rx_queues(portid);
2938 n_tx_queue = nb_lcores;
2939 if (n_tx_queue > MAX_TX_QUEUE_PER_PORT)
2940 n_tx_queue = MAX_TX_QUEUE_PER_PORT;
2942 port_config[portid].port_id = portid;
2943 port_config[portid].nrx_queue = nb_rx_queue;
2944 port_config[portid].ntx_queue = n_tx_queue;
2945 port_config[portid].state = 1;
2946 port_config[portid].promisc = promiscuous_on;
2947 port_config[portid].mempool.pool_size = MEMPOOL_SIZE;
2948 port_config[portid].mempool.buffer_size = BUFFER_SIZE + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
2949 port_config[portid].mempool.cache_size = CACHE_SIZE;
2950 port_config[portid].mempool.cpu_socket_id = rte_socket_id();
2951 memcpy (&port_config[portid].port_conf, &port_conf, sizeof(struct rte_eth_conf));
2952 memcpy (&port_config[portid].rx_conf, &rx_conf, sizeof(struct rte_eth_rxconf));
2953 memcpy (&port_config[portid].tx_conf, &tx_conf, sizeof(struct rte_eth_txconf));
2955 /* Enable TCP and UDP HW Checksum , when required */
2957 port_config[portid].tx_conf.txq_flags &=
2958 ~(ETH_TXQ_FLAGS_NOXSUMTCP|ETH_TXQ_FLAGS_NOXSUMUDP);
2961 if (ifm_port_setup (portid, &port_config[portid]))
2962 rte_panic("Port Setup Failed: %"PRIu32"\n. Try running by disabling checksum with (--no-hw-csum)", portid);
2965 check_all_ports_link_status((uint8_t)nb_ports, enabled_port_mask);
2970 populate_lpm_routes();
2972 convert_ipstr_to_numeric();
2974 /* launch per-lcore init on every lcore */
2975 rte_eal_mp_remote_launch(main_loop, NULL, CALL_MASTER);
2976 cl = cmdline_stdin_new(main_ctx, "Replay> ");
2978 rte_panic("Cannot create cmdline instance\n");
2979 cmdline_interact(cl);
2980 cmdline_stdin_exit(cl);
2982 rte_exit(0, "Bye!\n");
2983 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2984 if (rte_eal_wait_lcore(lcore_id) < 0)
2991 /**********************************************************/
2993 struct cmd_obj_clear_result {
2994 cmdline_fixed_string_t clear;
2995 cmdline_fixed_string_t l3fwd;
2996 cmdline_fixed_string_t stats;
2999 static void cmd_clear_l3fwd_stats_parsed(
3000 __rte_unused void *parsed_result,
3001 __rte_unused struct cmdline *cl,
3002 __attribute__((unused)) void *data)
3008 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_l3fwd_string =
3009 TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, l3fwd, "UDP_Replay");
3010 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_clear_string =
3011 TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, clear, "clear");
3012 cmdline_parse_token_string_t cmd_clear_l3fwd_stats_stats_string =
3013 TOKEN_STRING_INITIALIZER(struct cmd_obj_clear_result, stats, "stats");
3015 cmdline_parse_inst_t cmd_clear_l3fwd_stats = {
3016 .f = cmd_clear_l3fwd_stats_parsed, /* function to call */
3017 .data = NULL, /* 2nd arg of func */
3018 .help_str = "clears UDP_Replay stats for rx/tx",
3019 .tokens = { /* token list, NULL terminated */
3020 (void *)&cmd_clear_l3fwd_stats_l3fwd_string,
3021 (void *)&cmd_clear_l3fwd_stats_clear_string,
3022 (void *)&cmd_clear_l3fwd_stats_stats_string,
3026 /**********************************************************/
3027 struct cmd_obj_add_result {
3028 cmdline_fixed_string_t action;
3029 cmdline_fixed_string_t name;
3032 static void cmd_l3fwd_stats_parsed(
3033 __rte_unused void *parsed_result,
3034 __rte_unused struct cmdline *cl,
3035 __attribute__((unused)) void *data)
3038 /*printf("\n Rx value : Tx Value : \n");*/
3040 /*cmdline_printf(cl, "Object %s added, ip=%s\n",
3044 cmdline_parse_token_string_t cmd_l3fwd_stats_l3fwd_string =
3045 TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, action, "UDP_Replay");
3046 cmdline_parse_token_string_t cmd_l3fwd_stats_stats_string =
3047 TOKEN_STRING_INITIALIZER(struct cmd_obj_add_result, name, "stats");
3049 cmdline_parse_inst_t cmd_l3fwd_stats = {
3050 .f = cmd_l3fwd_stats_parsed, /* function to call */
3051 .data = NULL, /* 2nd arg of func */
3052 .help_str = "UDP_Replay stats for rx/tx",
3053 .tokens = { /* token list, NULL terminated */
3054 (void *)&cmd_l3fwd_stats_l3fwd_string,
3055 (void *)&cmd_l3fwd_stats_stats_string,
3060 struct cmd_quit_result {
3061 cmdline_fixed_string_t quit;
3066 __rte_unused void *parsed_result,
3068 __rte_unused void *data)
3073 static cmdline_parse_token_string_t cmd_quit_quit =
3074 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
3076 static cmdline_parse_inst_t cmd_quit = {
3077 .f = cmd_quit_parsed,
3081 (void *) &cmd_quit_quit,
3086 /**********************************************************/
3087 /****** CONTEXT (list of instruction) */
3089 cmdline_parse_ctx_t main_ctx[] = {
3090 (cmdline_parse_inst_t *)&cmd_l3fwd_stats,
3091 (cmdline_parse_inst_t *)&cmd_clear_l3fwd_stats,
3092 (cmdline_parse_inst_t *)&cmd_quit,