Temp Fix for vFW perf issue
[samplevnf.git] / VNFs / vFW / pipeline / pipeline_vfw_be.c
index 97508a7..23e2aa0 100644 (file)
@@ -24,7 +24,7 @@
  */
 
 #define EN_SWP_ACL 1
-#define EN_SWP_ARP 1
+//#define EN_SWP_ARP 1
 
 #include <stdio.h>
 #include <stdlib.h>
 #include "lib_arp.h"
 #include "lib_icmpv6.h"
 #include "pipeline_common_fe.h"
+#include "gateway.h"
 
 uint32_t timer_lcore;
 
 uint8_t firewall_flag = 1;
-uint8_t VFW_DEBUG;
+uint8_t VFW_DEBUG = 0;
 uint8_t cnxn_tracking_is_active = 1;
 /**
  * A structure defining the VFW pipeline input port per thread data.
@@ -94,13 +95,6 @@ struct pipeline_vfw {
        uint8_t traffic_type;
        uint8_t links_map[PIPELINE_MAX_PORT_IN];
        uint8_t outport_id[PIPELINE_MAX_PORT_IN];
-       /* Local ARP & ND Tables */
-       struct lib_arp_route_table_entry
-              local_lib_arp_route_table[MAX_ARP_RT_ENTRY];
-       uint8_t local_lib_arp_route_ent_cnt;
-       struct lib_nd_route_table_entry
-              local_lib_nd_route_table[MAX_ND_RT_ENTRY];
-       uint8_t local_lib_nd_route_ent_cnt;
 
 } __rte_cache_aligned;
 /**
@@ -142,7 +136,7 @@ struct mbuf_tcp_meta_data {
 
 #define IP_VERSION_4 4
 #define IP_VERSION_6 6
-#define MIX 10
+
 /* IPv6 */
 #define IP_HDR_SIZE_IPV6  40
 #define IP_HDR_DSCP_OFST_IPV6 0
@@ -176,30 +170,7 @@ __rte_cache_aligned;
   * TODO: look into "stub" table and see if that can be used
   * to avoid useless table lookup
   */
-/***** ARP local cache *****/
-uint8_t link_hw_laddr_valid[MAX_NUM_LOCAL_MAC_ADDRESS] = {
-       0, 0, 0, 0, 0, 0, 0, 0,
-       0, 0, 0, 0, 0, 0, 0, 0
-};
-
-static struct ether_addr link_hw_laddr[MAX_NUM_LOCAL_MAC_ADDRESS] = {
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} },
-       {.addr_bytes = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00} }
-};
+uint64_t arp_pkts_mask;
 
 /* Start TSC measurement */
 /* Prefetch counters and pipe before this function */
@@ -228,141 +199,6 @@ static inline void end_tsc_measure(
        }
 }
 
-static struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
-{
-       return &link_hw_laddr[out_port];
-}
-
-static uint8_t local_dest_mac_present(uint8_t out_port)
-{
-       return link_hw_laddr_valid[out_port];
-}
-
-static uint32_t local_get_nh_ipv4(
-       uint32_t ip,
-       uint32_t *port,
-       uint32_t *nhip,
-       struct pipeline_vfw *vfw_pipe)
-{
-       int i;
-
-       for (i = 0; i < vfw_pipe->local_lib_arp_route_ent_cnt; i++) {
-              if (((vfw_pipe->local_lib_arp_route_table[i].ip &
-              vfw_pipe->local_lib_arp_route_table[i].mask) ==
-              (ip & vfw_pipe->local_lib_arp_route_table[i].mask))) {
-                     *port = vfw_pipe->local_lib_arp_route_table[i].port;
-
-                     *nhip = vfw_pipe->local_lib_arp_route_table[i].nh;
-                     return 1;
-              }
-       }
-       return 0;
-}
-
-static void do_local_nh_ipv4_cache(uint32_t dest_if,
-              struct pipeline_vfw *vfw_pipe)
-{
-
-       /* Search for the entry and do local copy */
-       int i;
-
-       for (i = 0; i < MAX_ARP_RT_ENTRY; i++) {
-              if (lib_arp_route_table[i].port == dest_if) {
-
-                     struct lib_arp_route_table_entry *lentry =
-                            &vfw_pipe->
-                            local_lib_arp_route_table[vfw_pipe->
-                            local_lib_arp_route_ent_cnt];
-
-                     lentry->ip   = lib_arp_route_table[i].ip;
-                     lentry->mask = lib_arp_route_table[i].mask;
-                     lentry->port = lib_arp_route_table[i].port;
-                     lentry->nh   = lib_arp_route_table[i].nh;
-
-                     vfw_pipe->local_lib_arp_route_ent_cnt++;
-                     break;
-              }
-       }
-}
-static uint32_t local_get_nh_ipv6(
-       uint8_t *ip,
-       uint32_t *port,
-       uint8_t nhip[],
-        struct pipeline_vfw *vfw_pipe)
-{
-       uint8_t netmask_ipv6[IPV6_ADD_SIZE], netip_nd[IPV6_ADD_SIZE],
-              netip_in[IPV6_ADD_SIZE];
-       uint8_t i = 0, j = 0, k = 0, l = 0, depthflags = 0, depthflags1 = 0;
-       memset(netmask_ipv6, 0, sizeof(netmask_ipv6));
-       memset(netip_nd, 0, sizeof(netip_nd));
-       memset(netip_in, 0, sizeof(netip_in));
-
-       for (i = 0; i < vfw_pipe->local_lib_nd_route_ent_cnt; i++) {
-
-              convert_prefixlen_to_netmask_ipv6(
-                     vfw_pipe->local_lib_nd_route_table[i].depth,
-                     netmask_ipv6);
-
-              for (k = 0; k < IPV6_ADD_SIZE; k++)
-                     if (vfw_pipe->local_lib_nd_route_table[i].ipv6[k] &
-                                   netmask_ipv6[k]) {
-                            depthflags++;
-                            netip_nd[k] = vfw_pipe->
-                                   local_lib_nd_route_table[i].ipv6[k];
-                     }
-
-              for (l = 0; l < IPV6_ADD_SIZE; l++)
-                     if (ip[l] & netmask_ipv6[l]) {
-                            depthflags1++;
-                            netip_in[l] = ip[l];
-                     }
-
-
-              if ((depthflags == depthflags1) && (memcmp(netip_nd, netip_in,
-                                          sizeof(netip_nd)) == 0)) {
-
-                     *port = vfw_pipe->local_lib_nd_route_table[i].port;
-
-                     for (j = 0; j < IPV6_ADD_SIZE; j++)
-                            nhip[j] = vfw_pipe->
-                                   local_lib_nd_route_table[i].nhipv6[j];
-                     return 1;
-              }
-
-              depthflags = 0;
-              depthflags1 = 0;
-                     }
-                     return 0;
-}
-
-static void do_local_nh_ipv6_cache(uint32_t dest_if,
-              struct pipeline_vfw *vfw_pipe)
-{
-              /* Search for the entry and do local copy */
-       int i, l;
-
-       for (i = 0; i < MAX_ND_RT_ENTRY; i++) {
-
-              if (lib_nd_route_table[i].port == dest_if) {
-
-                     struct lib_nd_route_table_entry *lentry = &vfw_pipe->
-                            local_lib_nd_route_table[vfw_pipe->
-                            local_lib_nd_route_ent_cnt];
-
-                     for (l = 0; l < IPV6_ADD_SIZE; l++) {
-                            lentry->ipv6[l]   =
-                                   lib_nd_route_table[i].ipv6[l];
-                            lentry->nhipv6[l] =
-                                   lib_nd_route_table[i].nhipv6[l];
-                     }
-                     lentry->depth = lib_nd_route_table[i].depth;
-                     lentry->port  = lib_nd_route_table[i].port;
-
-                     vfw_pipe->local_lib_nd_route_ent_cnt++;
-                     break;
-                     } /* if */
-              } /* for */
-}
 /**
  * Print packet for debugging.
  *
@@ -895,15 +731,11 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
               struct pipeline_vfw *vfw_pipe)
 {
 
-       uint32_t ret;
-       int ret_mac;
        uint8_t i;
 
-       struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
 
        for (i = 0; i < 4; i++) {
               uint32_t dest_if = INVALID_DESTIF;
@@ -917,7 +749,6 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
 
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -928,120 +759,51 @@ pkt4_work_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
               uint32_t nhip = 0;
 
               uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
-
-              ret = local_get_nh_ipv4(dest_address, &dest_if,
-                            &nhip, vfw_pipe);
-              if (must_reverse) {
+              if (must_reverse)
                      rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(
-                                                 dest_if, vfw_pipe);
-                            }
 
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+       struct arp_entry_data *ret_arp_data = NULL;
+        ret_arp_data = get_dest_mac_addr_port(dest_address,
+                       &dest_if, &ehdr->d_addr);
+        meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
 
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+        if (arp_cache_dest_mac_present(dest_if)) {
+                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
+               update_nhip_access(dest_if);
+                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+                        arp_send_buffered_pkts(ret_arp_data,
+                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
 
-              }
-              meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
-              if (local_dest_mac_present(dest_if)) {
-                     ether_addr_copy(get_local_link_hw_addr(dest_if),
-                                   &ehdr->d_addr);
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
-              } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac == ARP_FOUND) {
-
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
-
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
-
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
                             }
 
                      } else {
-
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                                   printf("MAC NOT FOUND for ip 0x%"
-                                          PRIx32", dest_if %"
-                                          PRId16": %s, ",
-                                          dest_address,
-                                          dest_if, buf);
+                if (unlikely(ret_arp_data == NULL)) {
+                       if (VFW_DEBUG)
+                        printf("%s: NHIP Not Found, nhip:%x , "
+                        "outport_id: %d\n", __func__, nhip,
+                        vfw_pipe->outport_id[dest_if]);
+
+                        /* Drop the pkt */
+                        vfw_pipe->counters->
+                                 pkts_drop_without_arp_entry++;
+                        continue;
                             }
-                            /* ICMP req sent, drop packet by
-                             * changing the mask */
-                            *pkts_mask &= ~pkt_mask;
-                            vfw_pipe->
-                                   counters->pkts_drop_without_arp_entry++;
+               if (ret_arp_data->status == INCOMPLETE ||
+                           ret_arp_data->status == PROBE) {
+                                if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                       /* ICMP req sent, drop packet by
+                                               * changing the mask */
+                                       vfw_pipe->counters->
+                                               pkts_drop_without_arp_entry++;
+                                        continue;
+                                } else {
+                                        //arp_pkts_mask |= pkt_mask;
+                                       *arp_hijack_mask |= pkt_mask;
+                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                        continue;
                      }
               }
+       }
        }
 }
 
@@ -1072,15 +834,11 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
               struct pipeline_vfw *vfw_pipe)
 {
 
-       uint32_t ret;
        uint32_t dest_if = INVALID_DESTIF;
-       int ret_mac;
 
-       struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
        uint64_t pkt_mask = 1LLU << pkt_num;
 
        pkt = pkts;
@@ -1089,7 +847,6 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
 
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -1100,118 +857,50 @@ pkt_work_vfw_arp_ipv4_packets(struct rte_mbuf *pkts,
               uint32_t nhip = 0;
 
               uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
-
-              ret = local_get_nh_ipv4(dest_address, &dest_if,
-                            &nhip, vfw_pipe);
-              if (must_reverse) {
+              if (must_reverse)
                      rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(
-                                                 dest_if, vfw_pipe);
-                            }
 
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+       struct arp_entry_data *ret_arp_data = NULL;
+                     ret_arp_data = get_dest_mac_addr_port(dest_address,
+                                   &dest_if, &ehdr->d_addr);
+                       meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
 
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+        if (arp_cache_dest_mac_present(dest_if)) {
 
-              }
-              meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
-              if (local_dest_mac_present(dest_if)) {
-                     ether_addr_copy(get_local_link_hw_addr(dest_if),
-                                   &ehdr->d_addr);
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
-              } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac) {
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
-
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
-
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
-                            }
+                ether_addr_copy(get_link_hw_addr(dest_if), &ehdr->s_addr);
+               update_nhip_access(dest_if);
+                if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+                        arp_send_buffered_pkts(ret_arp_data,
+                                 &ehdr->d_addr, vfw_pipe->outport_id[dest_if]);
 
+                            }
                      } else {
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC NOT FOUND for ip 0x%"
-                                                 PRIx32", dest_if %"
-                                                 PRId16": %s, ",
-                                                 dest_address,
-                                                 dest_if, buf);
+                if (unlikely(ret_arp_data == NULL)) {
+
+                       if (VFW_DEBUG)
+                        printf("%s: NHIP Not Found, nhip:%x , "
+                        "outport_id: %d\n", __func__, nhip,
+                        vfw_pipe->outport_id[dest_if]);
+
+                        vfw_pipe->counters->
+                                pkts_drop_without_arp_entry++;
+                        return;
                             }
-                            /* ICMP req sent, drop packet by
-                             * changing the mask */
-                            *pkts_mask &= ~pkt_mask;
-                            vfw_pipe->
-                                   counters->pkts_drop_without_arp_entry++;
+               if (ret_arp_data->status == INCOMPLETE ||
+                           ret_arp_data->status == PROBE) {
+                                if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                       /* ICMP req sent, drop packet by
+                                               * changing the mask */
+                                       vfw_pipe->counters->
+                                               pkts_drop_without_arp_entry++;
+                                        return;
+                                } else {
+                                        arp_pkts_mask |= pkt_mask;
+                                        arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                        return;
                      }
               }
+       }
 
        }
 }
@@ -1244,12 +933,10 @@ pkt4_work_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
               struct pipeline_vfw *vfw_pipe)
 {
        uint8_t nh_ipv6[IPV6_ADD_SIZE];
-       uint32_t ret;
        struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
        uint8_t i;
 
        for (i = 0; i < 4; i++) {
@@ -1263,7 +950,6 @@ pkt4_work_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
                      continue;
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -1275,104 +961,50 @@ pkt4_work_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
               uint8_t dest_address[IPV6_ADD_SIZE];
 
               memset(nhip, 0, IPV6_ADD_SIZE);
-
-              rte_mov16(dest_address, ihdr->dst_addr);
-              ret = local_get_nh_ipv6(&dest_address[0], &dest_if,
-                            &nhip[0], vfw_pipe);
-              if (must_reverse) {
+              if (must_reverse)
                      rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
-
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
-
-              }
-
-              meta_data_addr->output_port = vfw_pipe->outport_id[dest_if];
 
+              rte_mov16(dest_address, ihdr->dst_addr);
               memset(nh_ipv6, 0, IPV6_ADD_SIZE);
-              if (get_dest_mac_address_ipv6_port(
+              struct nd_entry_data *ret_nd_data = NULL;
+              ret_nd_data = get_dest_mac_address_ipv6_port(
                                    &dest_address[0],
                                    &dest_if,
                                    &hw_addr,
-                                   &nh_ipv6[0])) {
-                     ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
+                                   &nh_ipv6[0]);
 
-                     if (vfw_debug >= DEBUG_LEVEL_4) {
-                            char buf[HW_ADDR_SIZE];
-
-                            ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                            printf("MAC found for  dest_if %d: %s, ",
-                                          dest_if, buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->s_addr);
-                            printf("new eth hdr src: %s, ", buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->d_addr);
-                            printf("new eth hdr dst: %s\n", buf);
-                     }
+               meta_data_addr->output_port = vfw_pipe->
+                                    outport_id[dest_if];
+              if (nd_cache_dest_mac_present(dest_if)) {
+                    ether_addr_copy(get_link_hw_addr(dest_if),
+                                   &ehdr->s_addr);
+                   update_nhip_access(dest_if);
 
+                    if (unlikely(ret_nd_data && ret_nd_data->num_pkts)) {
+                        nd_send_buffered_pkts(ret_nd_data,
+                               &ehdr->d_addr, meta_data_addr->output_port);
+                    }
               } else {
-                     printf("deleting ipv6\n");
-                     *pkts_mask &= ~pkt_mask;
-                     /*Next Neighbor is not yet implemented
-                      * for ipv6.*/
-                     vfw_pipe->counters->
-                            pkts_drop_without_arp_entry++;
+                    if (unlikely(ret_nd_data == NULL)) {
+                         *pkts_mask &= ~pkt_mask;
+                         vfw_pipe->counters->
+                               pkts_drop_without_arp_entry++;
+                          continue;
+                    }
+                   if (ret_nd_data->status == INCOMPLETE ||
+                         ret_nd_data->status == PROBE) {
+                         if (ret_nd_data->num_pkts >= NUM_DESC) {
+                                /* Drop the pkt */
+                                *pkts_mask &= ~pkt_mask;
+                                vfw_pipe->counters->
+                                       pkts_drop_without_arp_entry++;
+                               continue;
+                          } else {
+                                arp_pkts_mask |= pkt_mask;
+                                nd_queue_unresolved_packet(ret_nd_data, pkt);
+                                continue;
+                          }
+                    }
               }
 
        }
@@ -1406,12 +1038,10 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
               struct pipeline_vfw *vfw_pipe)
 {
        uint8_t nh_ipv6[IPV6_ADD_SIZE];
-       uint32_t ret;
        struct ether_addr hw_addr;
        struct mbuf_tcp_meta_data *meta_data_addr;
        struct ether_hdr *ehdr;
        struct rte_mbuf *pkt;
-       uint16_t phy_port;
 
        uint32_t dest_if = INVALID_DESTIF;
        /* bitmask representing only this packet */
@@ -1423,7 +1053,6 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
 
               int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
 
-              phy_port = pkt->port;
               meta_data_addr = (struct mbuf_tcp_meta_data *)
                      RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
               ehdr = rte_vfw_get_ether_addr(pkt);
@@ -1435,104 +1064,48 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
               uint8_t dest_address[IPV6_ADD_SIZE];
 
               memset(nhip, 0, IPV6_ADD_SIZE);
-
-              rte_mov16(dest_address, ihdr->dst_addr);
-              ret = local_get_nh_ipv6(&dest_address[0], &dest_if,
-                            &nhip[0], vfw_pipe);
-              if (must_reverse) {
+              if (must_reverse)
                      rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          *pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
-
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   *pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
-
-              }
-
-              meta_data_addr->output_port = vfw_pipe->outport_id[dest_if];
-
+              rte_mov16(dest_address, ihdr->dst_addr);
               memset(nh_ipv6, 0, IPV6_ADD_SIZE);
-              if (get_dest_mac_address_ipv6_port(
+              struct nd_entry_data *ret_nd_data = NULL;
+              ret_nd_data = get_dest_mac_address_ipv6_port(
                                    &dest_address[0],
                                    &dest_if,
                                    &hw_addr,
-                                   &nh_ipv6[0])) {
-                     ether_addr_copy(&hw_addr, &ehdr->d_addr);
+                                   &nh_ipv6[0]);
+             meta_data_addr->output_port = vfw_pipe->
+                                    outport_id[dest_if];
+              if (nd_cache_dest_mac_present(dest_if)) {
                      ether_addr_copy(get_link_hw_addr(dest_if),
                                    &ehdr->s_addr);
+                   update_nhip_access(dest_if);
 
-                     if (vfw_debug >= DEBUG_LEVEL_4) {
-                            char buf[HW_ADDR_SIZE];
-
-                            ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                            printf("MAC found for  dest_if %d: %s, ",
-                                          dest_if, buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->s_addr);
-                            printf("new eth hdr src: %s, ", buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->d_addr);
-                            printf("new eth hdr dst: %s\n", buf);
+                    if (unlikely(ret_nd_data && ret_nd_data->num_pkts)) {
+                        nd_send_buffered_pkts(ret_nd_data,
+                               &ehdr->d_addr, meta_data_addr->output_port);
                      }
-
               } else {
-                     printf("deleting ipv6\n");
-                     *pkts_mask &= ~pkt_mask;
-                     /*Next Neighbor is not yet implemented
-                      * for ipv6.*/
-                     vfw_pipe->counters->
-                            pkts_drop_without_arp_entry++;
+                    if (unlikely(ret_nd_data == NULL)) {
+                        *pkts_mask &= ~pkt_mask;
+                       vfw_pipe->counters->
+                               pkts_drop_without_arp_entry++;
+                        return;
+                    }
+                   if (ret_nd_data->status == INCOMPLETE ||
+                          ret_nd_data->status == PROBE) {
+                          if (ret_nd_data->num_pkts >= NUM_DESC) {
+                                /* Drop the pkt */
+                                *pkts_mask &= ~pkt_mask;
+                                vfw_pipe->counters->
+                                    pkts_drop_without_arp_entry++;
+                                return;
+                          } else {
+                                arp_pkts_mask |= pkt_mask;
+                                nd_queue_unresolved_packet(ret_nd_data, pkt);
+                                return;
+                          }
+                    }
               }
 
        }
@@ -1542,338 +1115,258 @@ pkt_work_vfw_arp_ipv6_packets(struct rte_mbuf *pkts,
 #else
 
 /**
- * walk every valid mbuf (denoted by pkts_mask) and apply arp to the packet.
+ * walk every valid mbuf (denoted by pkts_mask) and forward the packet.
  * To support synproxy, some (altered) packets may need to be sent back where
  * they came from. The ip header has already been adjusted, but the ethernet
  * header has not, so this must be performed here.
- * Return an updated pkts_mask, since arp may drop some packets
+ * Return an updated pkts_mask and arp_hijack_mask since arp may drop some packets
  *
  * @param pkts
- *  A pointer to the packet.
+ *  A pointer to the packet array.
  * @param pkts_mask
- *  Packet mask
- * @param synproxy_reply_mask
- *  Reply Packet mask for Synproxy
+ *  Packets mask to be processed
+ * @param arp_hijack_mask
+ *  Packets to be hijacked for arp buffering
  * @param vfw_pipe
  *  A pointer to VFW pipeline.
  */
-static uint64_t
-rte_vfw_arp_ipv4_packets(struct rte_mbuf **pkts,
-              uint64_t pkts_mask,
-              uint64_t synproxy_reply_mask,
-              struct pipeline_vfw *vfw_pipe)
+static void vfw_fwd_pkts_ipv4(struct rte_mbuf **pkts, uint64_t *pkts_mask,
+               uint64_t *arp_hijack_mask, struct pipeline_vfw *vfw_pipe)
 {
-       uint64_t pkts_to_arp = pkts_mask;
+       uint64_t pkts_to_arp = *pkts_mask;
 
-       uint32_t ret;
-       uint32_t dest_if = INVALID_DESTIF;
-       int ret_mac;
-       for (; pkts_to_arp;) {
-              struct ether_addr hw_addr;
-              struct mbuf_tcp_meta_data *meta_data_addr;
-              struct ether_hdr *ehdr;
-              struct rte_mbuf *pkt;
-              uint16_t phy_port;
-
-              uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
-              /* bitmask representing only this packet */
-              uint64_t pkt_mask = 1LLU << pos;
-              /* remove this packet from remaining list */
-              pkts_to_arp &= ~pkt_mask;
-              pkt = pkts[pos];
-              int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
+       for (; pkts_to_arp;) {
 
-              phy_port = pkt->port;
-              meta_data_addr = (struct mbuf_tcp_meta_data *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
-              ehdr = rte_vfw_get_ether_addr(pkt);
+               struct mbuf_tcp_meta_data *meta_data_addr;
+               struct ether_hdr *ehdr;
+               struct rte_mbuf *pkt;
+               uint32_t src_phy_port;
 
+               uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
+               /* bitmask representing only this packet */
+               uint64_t pkt_mask = 1LLU << pos;
+               /* remove this packet from remaining list */
+               pkts_to_arp &= ~pkt_mask;
+               pkt = pkts[pos];
 
-              struct ipv4_hdr *ihdr = (struct ipv4_hdr *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
-              uint32_t nhip = 0;
+               if(VFW_DEBUG) {
+                       printf("----------------\n");
+                       print_pkt(pkt);
+               }
 
-              uint32_t dest_address = rte_bswap32(ihdr->dst_addr);
+               meta_data_addr = (struct mbuf_tcp_meta_data *)
+                       RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
 
-              ret = local_get_nh_ipv4(dest_address, &dest_if,
-                            &nhip, vfw_pipe);
-              if (must_reverse) {
-                     rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(
-                                                 dest_if, vfw_pipe);
-                            }
+               ehdr = (struct ether_hdr *)
+                       RTE_MBUF_METADATA_UINT32_PTR(pkt, ETHERNET_START);
 
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 &dest_address,
-                                                 IP_VERSION_4);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv4_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+               src_phy_port = pkt->port;
+               uint32_t dst_phy_port = INVALID_DESTIF;
 
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port(&dest_address,
-                                          IP_VERSION_4);
-                            if (dest_if == INVALID_DESTIF) {
-                                   pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv4_cache(dest_if, vfw_pipe);
-                     }
+               if(is_phy_port_privte(src_phy_port))
+                   dst_phy_port = prv_to_pub_map[src_phy_port];
+               else
+                   dst_phy_port = pub_to_prv_map[src_phy_port];
 
-              }
-              meta_data_addr->output_port =  vfw_pipe->outport_id[dest_if];
-              if (local_dest_mac_present(dest_if)) {
-                     ether_addr_copy(get_local_link_hw_addr(dest_if),
-                                   &ehdr->d_addr);
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
-              } else {
-                     ret_mac = get_dest_mac_addr_port(dest_address,
-                                   &dest_if, &hw_addr);
-                     if (ret_mac) {
-                            link_hw_laddr_valid[dest_if] = 1;
-                            memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                          sizeof(struct ether_addr));
-
-                            ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                            ether_addr_copy(get_link_hw_addr(dest_if),
-                                          &ehdr->s_addr);
-
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &hw_addr);
-                                   printf("MAC found for ip 0x%"
-                                          PRIx32", dest_if %d: %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->s_addr);
-                                   printf("new eth hdr src: %s, ", buf);
-                                   ether_format_addr(buf, sizeof(buf),
-                                                 &ehdr->d_addr);
-                                   printf("new eth hdr dst: %s\n", buf);
-                            }
 
-                     } else {
-                            if (unlikely(ret_mac == 0))
-                                   request_arp(meta_data_addr->output_port,
-                                          nhip);
-
-                            if (vfw_debug >= DEBUG_LEVEL_4) {
-                                   char buf[HW_ADDR_SIZE];
-
-                            ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                            printf("MAC NOT FOUND for ip 0x%"
-                                          PRIx32", dest_if %"
-                                          PRId16": %s, ",
-                                          dest_address,
-                                          dest_if, buf);
-                     }
-                     /* ICMP req sent, drop packet by
-                      * changing the mask */
-                     pkts_mask &= ~pkt_mask;
-                     vfw_pipe->
-                            counters->pkts_drop_without_arp_entry++;
-              }
-}
+               if(likely(is_gateway())){
+                       struct ipv4_hdr *ipv4hdr = (struct ipv4_hdr *)
+                               RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
 
-       }
+                       /* Gateway Proc Starts */
 
-       return pkts_mask;
-}
-/**
- * walk every valid mbuf (denoted by pkts_mask) and apply arp to the packet.
- * To support synproxy, some (altered) packets may need to be sent back where
- * they came from. The ip header has already been adjusted, but the ethernet
- * header has not, so this must be performed here.
- * Return an updated pkts_mask, since arp may drop some packets
- *
- * @param pkts
- *  A pointer to the packet.
- * @param pkts_mask
- *  Packet mask
- * @param synproxy_reply_mask
- *  Reply Packet mask for Synproxy
- * @param vfw_pipe
- *  A pointer to VFW pipeline.
- */
+                       struct arp_entry_data *ret_arp_data = NULL;
+                       struct ether_addr dst_mac;
+                       uint32_t nhip = 0;
+                       uint32_t dst_ip_addr = rte_bswap32(ipv4hdr->dst_addr);
 
-       static uint64_t
-rte_vfw_arp_ipv6_packets(struct rte_mbuf **pkts,
-              uint64_t pkts_mask,
-              uint64_t synproxy_reply_mask,
-              struct pipeline_vfw *vfw_pipe)
-{
-       uint64_t pkts_to_arp = pkts_mask;
-       uint8_t nh_ipv6[IPV6_ADD_SIZE];
-       uint32_t ret;
-       uint32_t dest_if = INVALID_DESTIF;
+                       gw_get_route_nh_port_ipv4(dst_ip_addr, &dst_phy_port, &nhip, dst_phy_port);
 
-       for (; pkts_to_arp;) {
-              struct ether_addr hw_addr;
-              struct mbuf_tcp_meta_data *meta_data_addr;
-              struct ether_hdr *ehdr;
-              struct rte_mbuf *pkt;
-              uint16_t phy_port;
+                       ret_arp_data = get_dest_mac_addr_ipv4(nhip, dst_phy_port, &dst_mac);
 
-              uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
-              /* bitmask representing only this packet */
-              uint64_t pkt_mask = 1LLU << pos;
-              /* remove this packet from remaining list */
-              pkts_to_arp &= ~pkt_mask;
-              pkt = pkts[pos];
-              int must_reverse = ((synproxy_reply_mask & pkt_mask) != 0);
+                       /* Gateway Proc Ends */
 
-              phy_port = pkt->port;
-              meta_data_addr = (struct mbuf_tcp_meta_data *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
-              ehdr = rte_vfw_get_ether_addr(pkt);
+                       if (likely(arp_cache_dest_mac_present(dst_phy_port))) {
 
-              struct ipv6_hdr *ihdr = (struct ipv6_hdr *)
-                     RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
+                               ether_addr_copy(&dst_mac, &ehdr->d_addr);
+                               ether_addr_copy(get_link_hw_addr(dst_phy_port), &ehdr->s_addr);
 
-              uint8_t nhip[IPV6_ADD_SIZE];
-              uint8_t dest_address[IPV6_ADD_SIZE];
+                               meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
 
-              memset(nhip, 0, IPV6_ADD_SIZE);
+                               update_nhip_access(dst_phy_port);
 
-              rte_mov16(dest_address, ihdr->dst_addr);
-              ret = local_get_nh_ipv6(&dest_address[0], &dest_if,
-                            &nhip[0], vfw_pipe);
-              if (must_reverse) {
-                     rte_sp_exchange_mac_addresses(ehdr);
-                     if (is_phy_port_privte(phy_port)) {
-                            if (!ret) {
-                                   dest_if = get_pub_to_prv_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
+                               if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
 
-                     } else {
-                            if (!ret) {
-                                   dest_if = get_prv_to_pub_port(
-                                                 (uint32_t *)
-                                                 &dest_address[0],
-                                                 IP_VERSION_6);
-                                   if (dest_if == INVALID_DESTIF) {
-                                          pkts_mask &= ~pkt_mask;
-                                          vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                                   }
-                                   do_local_nh_ipv6_cache(dest_if,
-                                                 vfw_pipe);
-                            }
-                     }
-
-              } else if (is_phy_port_privte(phy_port)) {
-                     if (!ret) {
-                            dest_if = get_prv_to_pub_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
-
-              } else {
-                     if (!ret) {
-                            dest_if = get_pub_to_prv_port((uint32_t *)
-                                          &dest_address[0], IP_VERSION_6);
-                            if (dest_if == INVALID_DESTIF) {
-                                   pkts_mask &= ~pkt_mask;
-                                   vfw_pipe->counters->
-                                          pkts_drop_without_arp_entry++;
-
-                            }
-                            do_local_nh_ipv6_cache(dest_if, vfw_pipe);
-                     }
+                                       arp_send_buffered_pkts(ret_arp_data, &ehdr->d_addr,
+                                                       vfw_pipe->outport_id[dst_phy_port]);
+                               }
 
-              }
+                       } else {
+                               if (unlikely(ret_arp_data == NULL)) {
 
-              meta_data_addr->output_port = vfw_pipe->outport_id[dest_if];
+                                       printf("NHIP Not Found\n");
 
-              memset(nh_ipv6, 0, IPV6_ADD_SIZE);
-              if (get_dest_mac_address_ipv6_port(
-                                   &dest_address[0],
-                                   &dest_if,
-                                   &hw_addr,
-                                   &nh_ipv6[0])) {
-                     ether_addr_copy(&hw_addr, &ehdr->d_addr);
-                     ether_addr_copy(get_link_hw_addr(dest_if),
-                                   &ehdr->s_addr);
+                                       /* Drop the pkt */
+                                       vfw_pipe->counters->
+                                               pkts_drop_without_arp_entry++;
+                                       continue;
+                               }
+                               if (ret_arp_data->status == INCOMPLETE ||
+                                               ret_arp_data->status == PROBE) {
+                                       if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                               /* ICMP req sent, drop packet by
+                                                * changing the mask */
+                                               vfw_pipe->counters->pkts_drop_without_arp_entry++;
+                                               continue;
+                                       } else {
+                                               *arp_hijack_mask |= pkt_mask;
+                                               arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                               continue;
+                                       }
+                               }
+                       }
+               } else {
+                       /* IP Pkt forwarding based on  pub/prv mapping */
+                       meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
 
-                     if (vfw_debug >= DEBUG_LEVEL_4) {
-                            char buf[HW_ADDR_SIZE];
-
-                            ether_format_addr(buf, sizeof(buf),
-                                          &hw_addr);
-                            printf("MAC found for  dest_if %d: %s, ",
-                                          dest_if, buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->s_addr);
-                            printf("new eth hdr src: %s, ", buf);
-                            ether_format_addr(buf, sizeof(buf),
-                                          &ehdr->d_addr);
-                            printf("new eth hdr dst: %s\n", buf);
-                     }
+                       if(VFW_DEBUG) {
+                               printf("IP_PKT_FWD: src_phy_port=%d, dst_phy_port=%d\n",
+                                               src_phy_port, dst_phy_port);
+                       }
+               }
 
-              } else {
-                     printf("deleting ipv6\n");
-                     pkts_mask &= ~pkt_mask;
-                     /*Next Neighbor is not yet implemented
-                      * for ipv6.*/
-                     vfw_pipe->counters->
-                            pkts_drop_without_arp_entry++;
-              }
+               if(VFW_DEBUG)
+                       print_pkt(pkt);
+       }
 
+}
 
-       }
+/**
+ * walk every valid mbuf (denoted by pkts_mask) and forward the packet.
+ * To support synproxy, some (altered) packets may need to be sent back where
+ * they came from. The ip header has already been adjusted, but the ethernet
+ * header has not, so this must be performed here.
+ * Return an updated pkts_mask and arp_hijack_mask since arp may drop some packets
+ *
+ * @param pkts
+ *  A pointer to the packet array.
+ * @param pkts_mask
+ *  Packets mask to be processed
+ * @param arp_hijack_mask
+ *  Packets to be hijacked for arp buffering
+ * @param vfw_pipe
+ *  A pointer to VFW pipeline.
+ */
+static void vfw_fwd_pkts_ipv6(struct rte_mbuf **pkts, uint64_t *pkts_mask,
+                       uint64_t *arp_hijack_mask, struct pipeline_vfw *vfw_pipe)
+{
+       uint64_t pkts_to_arp = *pkts_mask;
 
-       return pkts_mask;
+       for (; pkts_to_arp;) {
+
+               struct mbuf_tcp_meta_data *meta_data_addr;
+               struct ether_hdr *ehdr;
+               struct rte_mbuf *pkt;
+               uint32_t src_phy_port;
+
+               struct nd_entry_data *ret_nd_data = NULL;
+
+               uint8_t pos = (uint8_t) __builtin_ctzll(pkts_to_arp);
+               /* bitmask representing only this packet */
+               uint64_t pkt_mask = 1LLU << pos;
+               /* remove this packet from remaining list */
+               pkts_to_arp &= ~pkt_mask;
+               pkt = pkts[pos];
+
+               if(VFW_DEBUG) {
+                       printf("----------------\n");
+                       print_pkt(pkt);
+               }
+
+               meta_data_addr = (struct mbuf_tcp_meta_data *)
+                       RTE_MBUF_METADATA_UINT32_PTR(pkt, META_DATA_OFFSET);
+
+               ehdr = (struct ether_hdr *)
+                       RTE_MBUF_METADATA_UINT32_PTR(pkt, ETHERNET_START);
+
+               src_phy_port = pkt->port;
+               uint32_t dst_phy_port = INVALID_DESTIF;
+
+               if(is_gateway()){
+                       struct ipv6_hdr *ipv6hdr = (struct ipv6_hdr *)
+                               RTE_MBUF_METADATA_UINT32_PTR(pkt, IP_START);
+
+                       /* Gateway Proc Starts */
+
+                       struct ether_addr dst_mac;
+                       uint32_t dst_phy_port = INVALID_DESTIF;
+                       uint8_t nhipv6[IPV6_ADD_SIZE];
+                       uint8_t dest_ipv6_address[IPV6_ADD_SIZE];
+                       memset(nhipv6, 0, IPV6_ADD_SIZE);
+                       src_phy_port = pkt->port;
+                       rte_mov16(dest_ipv6_address, (uint8_t *)ipv6hdr->dst_addr);
+
+                       gw_get_nh_port_ipv6(dest_ipv6_address, &dst_phy_port, nhipv6);
+
+                       ret_nd_data = get_dest_mac_addr_ipv6(nhipv6, dst_phy_port, &dst_mac);
+
+                       /* Gateway Proc Ends */
+
+                       if (nd_cache_dest_mac_present(dst_phy_port)) {
+
+                               ether_addr_copy(&dst_mac, &ehdr->d_addr);
+                               ether_addr_copy(get_link_hw_addr(dst_phy_port), &ehdr->s_addr);
+
+                               meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
+
+                               update_nhip_access(dst_phy_port);
+
+                               if (unlikely(ret_nd_data && ret_nd_data->num_pkts)) {
+                                       nd_send_buffered_pkts(ret_nd_data, &ehdr->d_addr,
+                                                       vfw_pipe->outport_id[dst_phy_port]);
+                               }
+
+                       } else {
+                               if (unlikely(ret_nd_data == NULL)) {
+
+                                       printf("NHIP Not Found\n");
+
+                                       /* Drop the pkt */
+                                       vfw_pipe->counters->pkts_drop_without_arp_entry++;
+                                       continue;
+                               }
+                               if (ret_nd_data->status == INCOMPLETE ||
+                                               ret_nd_data->status == PROBE) {
+                                       if (ret_nd_data->num_pkts >= NUM_DESC) {
+                                               /* ICMP req sent, drop packet by
+                                                * changing the mask */
+                                               vfw_pipe->counters->pkts_drop_without_arp_entry++;
+                                               continue;
+                                       } else {
+                                               *arp_hijack_mask |= pkt_mask;
+                                               nd_queue_unresolved_packet(ret_nd_data, pkt);
+                                               continue;
+                                       }
+                               }
+                       }
+
+               } else {
+                       /* IP Pkt forwarding based on  pub/prv mapping */
+                       if(is_phy_port_privte(src_phy_port))
+                               dst_phy_port = prv_to_pub_map[src_phy_port];
+                       else
+                               dst_phy_port = pub_to_prv_map[src_phy_port];
+
+                       meta_data_addr->output_port = vfw_pipe->outport_id[dst_phy_port];
+
+                       if(VFW_DEBUG) {
+                               printf("IP_PKT_FWD: src_phy_port=%d, dst_phy_port=%d\n",
+                                               src_phy_port, dst_phy_port);
+                       }
+               }
+               if(VFW_DEBUG)
+                       print_pkt(pkt);
+       }
 }
 
 #endif
@@ -2069,8 +1562,9 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
 
        uint64_t packet_mask_in = RTE_LEN2MASK(n_pkts, uint64_t);
        uint64_t pkts_drop_mask;
-       uint64_t hijack_mask = 0;
-       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
+       uint64_t synp_hijack_mask = 0;
+       uint64_t arp_hijack_mask = 0;
+//       uint64_t synproxy_reply_mask;       /* for synproxy */
        uint64_t keep_mask = packet_mask_in;
 
        uint64_t conntrack_mask = 0, connexist_mask = 0;
@@ -2150,8 +1644,8 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
        if (likely(cnxn_tracking_is_active)) {
               rte_ct_cnxn_tracker_batch_lookup_type(ct, pkts,
                             &keep_mask, &ct_helper, IPv4_HEADER_SIZE);
-              synproxy_reply_mask = ct_helper.reply_pkt_mask;
-              hijack_mask = ct_helper.hijack_mask;
+//              synproxy_reply_mask = ct_helper.reply_pkt_mask;
+              synp_hijack_mask = ct_helper.hijack_mask;
 
        }
 
@@ -2189,9 +1683,9 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
 #else
        rte_prefetch0((void*)in_port_dir_a);
        rte_prefetch0((void*)prv_to_pub_map);
-       rte_prefetch0((void*) & vfw_pipe->local_lib_arp_route_table);
-       keep_mask = rte_vfw_arp_ipv4_packets(pkts, keep_mask,
-                     synproxy_reply_mask, vfw_pipe);
+
+       vfw_fwd_pkts_ipv4(pkts, &keep_mask, &arp_hijack_mask, vfw_pipe);
+
 #endif
 
        if (vfw_debug > 1) {
@@ -2203,7 +1697,15 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
                                    (void *)keep_mask);
        }
 
-       /* Update mask before returning, so that bad packets are dropped */
+          /* Hijack the Synproxy and ARP buffered packets */
+
+       if (unlikely(arp_hijack_mask || synp_hijack_mask)) {
+
+//                printf("Pkts hijacked arp = %lX, synp = %lX\n",
+//                                   arp_hijack_mask, synp_hijack_mask);
+
+                rte_pipeline_ah_packet_hijack(p,(arp_hijack_mask | synp_hijack_mask));
+        }
 
        pkts_drop_mask = packet_mask_in & ~keep_mask;
 
@@ -2212,9 +1714,6 @@ vfw_port_in_action_ipv4(struct rte_pipeline *p,
               rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
        }
 
-       if (unlikely(hijack_mask != 0))
-              rte_pipeline_ah_packet_hijack(p, hijack_mask);
-
        vfw_pipe->counters->num_batch_pkts_sum += n_pkts;
        vfw_pipe->counters->num_pkts_measurements++;
 
@@ -2254,8 +1753,10 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
 
        uint64_t packet_mask_in = RTE_LEN2MASK(n_pkts, uint64_t);
        uint64_t pkts_drop_mask;
-       uint64_t hijack_mask = 0;
-       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
+       uint64_t synp_hijack_mask = 0;
+       uint64_t arp_hijack_mask = 0;
+//       uint64_t hijack_mask = 0;
+//       uint64_t synproxy_reply_mask = 0;       /* for synproxy */
        uint64_t keep_mask = packet_mask_in;
 
        uint64_t conntrack_mask = 0, connexist_mask = 0;
@@ -2331,8 +1832,8 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
        if (likely(cnxn_tracking_is_active)) {
               rte_ct_cnxn_tracker_batch_lookup_type(ct, pkts,
                             &keep_mask, &ct_helper, IPv6_HEADER_SIZE);
-              synproxy_reply_mask = ct_helper.reply_pkt_mask;
-              hijack_mask = ct_helper.hijack_mask;
+//              synproxy_reply_mask = ct_helper.reply_pkt_mask;
+              synp_hijack_mask = ct_helper.hijack_mask;
 
        }
 
@@ -2344,7 +1845,7 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
                                    ETHERNET_START));
        }
        rte_prefetch0((void*)in_port_dir_a);
-       rte_prefetch0(vfw_pipe->local_lib_nd_route_table);
//      rte_prefetch0(vfw_pipe->local_lib_nd_route_table);
        uint32_t i;
 
        for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4) {
@@ -2369,9 +1870,9 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
        }
 #else
        rte_prefetch0((void*)in_port_dir_a);
-       rte_prefetch0((void*) & vfw_pipe->local_lib_arp_route_table);
-       keep_mask = rte_vfw_arp_ipv6_packets(pkts, keep_mask,
-                     synproxy_reply_mask, vfw_pipe);
+
+       vfw_fwd_pkts_ipv6(pkts, &keep_mask, &arp_hijack_mask, vfw_pipe);
+
 #endif
 
        if (vfw_debug > 1) {
@@ -2383,6 +1884,16 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
                                    (void *)keep_mask);
        }
 
+       /* Hijack the Synproxy and ARP buffered packets */
+
+        if (unlikely(arp_hijack_mask || synp_hijack_mask)) {
+
+//                printf("Pkts hijacked arp = %lX, synp = %lX\n",
+//                                   arp_hijack_mask, synp_hijack_mask);
+
+                rte_pipeline_ah_packet_hijack(p,(arp_hijack_mask | synp_hijack_mask));
+        }
+
        /* Update mask before returning, so that bad packets are dropped */
 
        pkts_drop_mask = packet_mask_in & ~keep_mask;
@@ -2392,9 +1903,6 @@ vfw_port_in_action_ipv6(struct rte_pipeline *p,
               rte_pipeline_ah_packet_drop(p, pkts_drop_mask);
        }
 
-       if (unlikely(hijack_mask != 0))
-              rte_pipeline_ah_packet_hijack(p, hijack_mask);
-
        vfw_pipe->counters->num_batch_pkts_sum += n_pkts;
        vfw_pipe->counters->num_pkts_measurements++;
 
@@ -2462,7 +1970,7 @@ pipeline_vfw_parse_args(struct pipeline_vfw *vfw_pipe,
               if (strcmp(arg_name, "n_flows") == 0) {
                      int n_flows = atoi(arg_value);
 
-                     if (n_flows == 0)
+                     if ((n_flows == 0) || (n_flows > 8000000))
                             return -1;
 
                      /* must be power of 2, round up if not */
@@ -2555,7 +2063,7 @@ static void
        strncpy(pipe->name, params->name, sizeof(pipe->name));
        pipe->log_level = params->log_level;
        pipe_vfw->n_flows = 4096;       /* small default value */
-       pipe_vfw->traffic_type = MIX;
+       pipe_vfw->traffic_type = IP_VERSION_4;
        pipe_vfw->pipeline_num = 0xff;
        for (i = 0; i < PIPELINE_MAX_PORT_IN; i++) {
               pipe_vfw->links_map[i] = 0xff;