Merge "[l2l3 stack] implements new arp state machine & arp buffering"
[samplevnf.git] / VNFs / vCGNAPT / pipeline / pipeline_cgnapt_be.c
index f9b0eb1..af37127 100644 (file)
 /* To maintain all cgnapt pipeline pointers used for all stats */
 struct pipeline_cgnapt *all_pipeline_cgnapt[128];
 uint8_t n_cgnapt_pipeline;
+struct pipeline_cgnapt *global_pnat;
 
+uint64_t arp_pkts_mask;
+extern struct arp_entry_data *arp_data_ptr[16];
 /* To know egress or ingress port */
 static uint8_t cgnapt_in_port_egress_prv[PIPELINE_MAX_PORT_IN];
 static uint8_t cgnapt_prv_que_port_index[PIPELINE_MAX_PORT_IN];
@@ -140,6 +143,7 @@ struct rte_hash_parameters napt_common_table_hash_params = {
 };
 
 /***** ARP local cache *****/
+
 uint8_t link_hw_laddr_valid[MAX_NUM_LOCAL_MAC_ADDRESS] = {
        0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0
@@ -178,7 +182,6 @@ static uint32_t local_get_nh_ipv4(
        uint32_t *port,
        uint32_t *nhip,
        struct pipeline_cgnapt *p_nat);
-
 static void do_local_nh_ipv4_cache(
        uint32_t dest_if,
        struct pipeline_cgnapt *p_nat);
@@ -220,6 +223,11 @@ uint64_t nextPowerOf2(uint64_t n)
        return n;
 }
 
+void remove_local_cache(uint8_t port)
+{
+       link_hw_laddr_valid[port] = 0;
+}
+
 /**
  * Function to get MAC addr of local link
  *
@@ -230,10 +238,10 @@ uint64_t nextPowerOf2(uint64_t n)
  *  Outport Link MAC addr
  */
 
-struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
-{
-       return &link_hw_laddr[out_port];
-}
+//struct ether_addr *get_local_link_hw_addr(uint8_t out_port)
+//{
+//     return &link_hw_laddr[out_port];
+//}
 
 /**
  * Function to get MAC addr from array instead of hash table
@@ -910,6 +918,33 @@ void sw_checksum(struct rte_mbuf *pkt, enum PKT_TYPE ver)
        }
 }
 
+void print_pkt_info(uint8_t *eth_dest, struct ether_addr *hw_addr, 
+               uint32_t dest_address, uint32_t port_id, struct rte_mbuf *pkt)
+{
+
+if (CGNAPT_DEBUG > 2) {
+       printf("MAC Found ip 0x%x, port %d - %02x:%02x:%02x:%02x:%02x:%02x  \n",
+       dest_address, port_id, hw_addr->addr_bytes[0], hw_addr->addr_bytes[1],
+       hw_addr->addr_bytes[2], hw_addr->addr_bytes[3], hw_addr->addr_bytes[4],
+       hw_addr->addr_bytes[5]);
+
+       printf("Dest MAC before - %02x:%02x:%02x:%02x:%02x:%02x      \n",
+               eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3], eth_dest[4],
+               eth_dest[5]);
+}
+
+if (CGNAPT_DEBUG > 2) {
+       printf("Dest MAC after - "
+               "%02x:%02x:%02x:%02x:%02x:%02x      \n",
+               eth_dest[0], eth_dest[1],
+               eth_dest[2], eth_dest[3],
+               eth_dest[4], eth_dest[5]);
+}
+
+if (CGNAPT_DEBUG > 4)
+       print_pkt(pkt);
+}
+
 static uint8_t check_arp_icmp(
        struct rte_mbuf *pkt,
        uint64_t pkt_mask,
@@ -925,7 +960,7 @@ static uint8_t check_arp_icmp(
 
        /* ARP outport number */
        uint16_t out_port = p_nat->p.n_ports_out - 1;
-
+       printf("check_arp_icmp called*****\n");
        uint8_t *protocol;
        uint32_t prot_offset;
 
@@ -2169,11 +2204,34 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
                        }
 
                        *outport_id = p_nat->outport_id[dest_if];
-                       int ret;
-                       ret = get_dest_mac_addr_port(dest_address,
-                               &dest_if, &hw_addr);
+                       struct arp_entry_data *ret_arp_data;
+                       ret_arp_data = get_dest_mac_addr_port(dest_address,
+                               &dest_if, (struct ether_addr *)&hw_addr);
+
+                       if (unlikely(ret_arp_data == NULL)) {
+
+                               printf("%s: NHIP Not Found, nhip: %x, "
+                               "outport_id: %d\n", __func__, nhip,
+                               *outport_id);
+
+                               /* Drop the pkt */
+                               p_nat->invalid_packets |= pkt_mask;
+                               p_nat->naptDroppedPktCount++;
+
+                               #ifdef CGNAPT_DEBUGGING
+                               p_nat->naptDroppedPktCount4++;
+                               #endif
+                               continue;
+                       }
+
+                       if (ret_arp_data->status == COMPLETE) {
+                               
+                               if (ret_arp_data->num_pkts) {
+                                       p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                                       arp_send_buffered_pkts(ret_arp_data,
+                                                &hw_addr, *outport_id);
+                               }
 
-                       if (ret == ARP_FOUND) {
                                memcpy(eth_dest, &hw_addr,
                                        sizeof(struct ether_addr));
                                memcpy(eth_src, get_link_hw_addr(dest_if),
@@ -2208,23 +2266,22 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
                                        print_pkt(pkts[pkt_index]);
                                #endif
 
-                       } else{
-                               if (ret == ARP_NOT_FOUND) {
-                                       /* Commented code may be required
-                                        * for future use, Please keep it */
-                                       //request_arp(*outport_id, nhip,
-                                       //      p_nat->p.p);
-                                       printf("%s: ARP Not Found, nhip: %x, "
-                                       "outport_id: %d\n", __func__, nhip,
-                                       *outport_id);
-                               }
+                       } else if (ret_arp_data->status == INCOMPLETE || 
+                               ret_arp_data->status == PROBE) {
+                               if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                       /* Drop the pkt */
+                                       p_nat->invalid_packets |= pkt_mask;
+                                       p_nat->naptDroppedPktCount++;
 
-                               p_nat->invalid_packets |= pkt_mask;
-                               p_nat->naptDroppedPktCount++;
-                               #ifdef CGNAPT_DEBUGGING
-                               p_nat->naptDroppedPktCount4++;
-                               #endif
-                               continue;
+                                       #ifdef CGNAPT_DEBUGGING
+                                       p_nat->naptDroppedPktCount4++;
+                                       #endif
+                                       continue;
+                               } else {
+                                       arp_queue_unresolved_packet(ret_arp_data,
+                                               pkts[pkt_index]);
+                                       continue;
+                               }
                        }
 
                        #ifdef CGNAPT_DBG_PRNT
@@ -2396,61 +2453,76 @@ static int cgnapt_in_port_ah_mix(struct rte_pipeline *rte_p,
                                };
 
                                *outport_id = p_nat->outport_id[dest_if];
-                               int ret;
-                               ret = get_dest_mac_addr_port(dest_address,
-                                       &dest_if, &hw_addr);
+                               struct arp_entry_data *ret_arp_data;
+                               ret_arp_data = get_dest_mac_addr_port(dest_address,
+                                       &dest_if, (struct ether_addr *)&hw_addr);
+
+                               if (unlikely(ret_arp_data == NULL)) {
+
+                                       printf("%s: NHIP Not Found, nhip: %x, "
+                                       "outport_id: %d\n", __func__, nhip,
+                                       *outport_id);
+
+                                       /* Drop the pkt */
+                                       p_nat->invalid_packets |= pkt_mask;
+                                       p_nat->naptDroppedPktCount++;
+
+                                       #ifdef CGNAPT_DEBUGGING
+                                       p_nat->naptDroppedPktCount4++;
+                                       #endif
+                                       continue;
+                               }
+
+                               if (ret_arp_data->status == COMPLETE) {
+
+                                       if (ret_arp_data->num_pkts) {
+                                               p_nat->naptedPktCount +=
+                                                        ret_arp_data->num_pkts;
+                                               arp_send_buffered_pkts(ret_arp_data,
+                                                        &hw_addr, *outport_id);
+                                       }
 
-                               if (ret == ARP_FOUND) {
                                        memcpy(eth_dest, &hw_addr,
                                                sizeof(struct ether_addr));
                                        memcpy(eth_src, get_link_hw_addr(
                                                dest_if),
                                                sizeof(struct ether_addr));
-                               #ifdef CGNAPT_DBG_PRNT
-                               if (CGNAPT_DEBUG > 2) {
-                               printf("MAC found for ip 0x%x, port %d - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                               dest_address, *outport_id,
-                                hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-                                hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-                                hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
+                                       #ifdef CGNAPT_DBG_PRNT
+                                       if (CGNAPT_DEBUG > 2) {
+                                       printf("MAC found for ip 0x%x, port %d - "
+                                       "%02x:%02x:%02x:%02x:%02x:%02x\n",
+                                       dest_address, *outport_id,
+                                       hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
+                                       hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
+                                       hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
 
-                               printf("Dest MAC before - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
+                                       printf("Dest MAC before - "
+                                       "%02x:%02x:%02x:%02x:%02x:%02x\n",
                                         eth_dest[0], eth_dest[1], eth_dest[2],
                                         eth_dest[3], eth_dest[4], eth_dest[5]);
-                               }
-                               #endif
+                                       }
+                                       #endif
 
-                               #ifdef CGNAPT_DBG_PRNT
-                               if (CGNAPT_DEBUG > 2) {
-                               printf("Dest MAC after - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
+                                       #ifdef CGNAPT_DBG_PRNT
+                                       if (CGNAPT_DEBUG > 2) {
+                                       printf("Dest MAC after - "
+                                       "%02x:%02x:%02x:%02x:%02x:%02x\n",
                                         eth_dest[0], eth_dest[1], eth_dest[2],
                                         eth_dest[3], eth_dest[4], eth_dest[5]);
-                               }
-                               #endif
+                                       }
+                                       #endif
 
-                               #ifdef CGNAPT_DBG_PRNT
-                               if (CGNAPT_DEBUG > 4)
-                                       print_pkt(pkts[pkt_index]);
-                               #endif
+                                       #ifdef CGNAPT_DBG_PRNT
+                                       if (CGNAPT_DEBUG > 4)
+                                               print_pkt(pkts[pkt_index]);
+                                       #endif
 
-                               } else {
-                                       if (ret == ARP_NOT_FOUND) {
-                                       printf("%s: ARP Not Found, nhip: %x, "
-                                       "outport_id: %d\n", __func__, nhip,
-                                       *outport_id);
-                                       }
-                                               //request_arp(*outport_id,
-                                               //      nhip, p_nat->p.p);
-                               p_nat->invalid_packets |= pkt_mask;
-                               p_nat->naptDroppedPktCount++;
-                               #ifdef CGNAPT_DEBUGGING
-                               p_nat->naptDroppedPktCount4++;
-                               #endif
-                               continue;
-                       }
+                               } else if (ret_arp_data->status == INCOMPLETE || 
+                                       ret_arp_data->status == PROBE) {
+                                       arp_queue_unresolved_packet(ret_arp_data,
+                                               pkts[pkt_index]);
+                                       continue;
+                               }
 
                        if (*protocol == IP_PROTOCOL_ICMP) {
                                // Query ID reverse translation done here
@@ -2547,7 +2619,7 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
        p_nat->pkt_burst_cnt = 0;       /* for dynamic napt */
        p_nat->valid_packets = rte_p->pkts_mask;        /*n_pkts; */
        p_nat->invalid_packets = 0;
-
+       arp_pkts_mask = 0;
        #ifdef CGNAPT_DBG_PRNT
        if (CGNAPT_DEBUG > 1)
                printf("cgnapt_key hit fn: %" PRIu32 "\n", n_pkts);
@@ -2567,6 +2639,7 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
 
        if (unlikely(p_nat->valid_packets == 0)) {
                /* no suitable packet for lookup */
+               printf("no suitable valid packets\n");
                rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
                return p_nat->valid_packets;
        }
@@ -2600,12 +2673,20 @@ static int cgnapt_in_port_ah_ipv4_prv(struct rte_pipeline *rte_p,
                                                [p_nat->lkup_indx[j]]);
        }
 
+       //prefetch();
+
+
        for (i = 0; i < (n_pkts & (~0x3LLU)); i += 4)
                pkt4_work_cgnapt_ipv4_prv(pkts, i, arg, p_nat);
 
        for (; i < n_pkts; i++)
                pkt_work_cgnapt_ipv4_prv(pkts, i, arg, p_nat);
 
+       if (arp_pkts_mask) {
+               p_nat->valid_packets &= ~(arp_pkts_mask);
+               rte_pipeline_ah_packet_hijack(rte_p, arp_pkts_mask);
+       }
+
        if (p_nat->invalid_packets) {
                /* get rid of invalid packets */
                rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
@@ -2682,7 +2763,7 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
        p_nat->pkt_burst_cnt = 0;       /* for dynamic napt */
        p_nat->valid_packets = rte_p->pkts_mask;        /*n_pkts; */
        p_nat->invalid_packets = 0;
-
+       arp_pkts_mask = 0;
        #ifdef CGNAPT_DBG_PRNT
        if (CGNAPT_DEBUG > 1)
                printf("cgnapt_key hit fn: %" PRIu32 "\n", n_pkts);
@@ -2701,6 +2782,7 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
        p_nat->valid_packets &= ~(p_nat->invalid_packets);
 
        if (unlikely(p_nat->valid_packets == 0)) {
+               printf("no valid packets in pub\n");
                /* no suitable packet for lookup */
                rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
                return p_nat->valid_packets;
@@ -2741,6 +2823,11 @@ static int cgnapt_in_port_ah_ipv4_pub(struct rte_pipeline *rte_p,
        for (; i < n_pkts; i++)
                pkt_work_cgnapt_ipv4_pub(pkts, i, arg, p_nat);
 
+       if (arp_pkts_mask) {
+               rte_pipeline_ah_packet_hijack(rte_p, arp_pkts_mask);
+               p_nat->valid_packets &= ~(arp_pkts_mask);
+       }
+
        if (p_nat->invalid_packets) {
                /* get rid of invalid packets */
                rte_pipeline_ah_packet_drop(rte_p, p_nat->invalid_packets);
@@ -3706,6 +3793,7 @@ pkt_work_cgnapt_key_ipv4_pub(
  *  A pointer to main CGNAPT structure
  *
  */
+uint64_t last_update;
 void
 pkt_work_cgnapt_ipv4_prv(
        struct rte_mbuf **pkts,
@@ -3849,54 +3937,31 @@ pkt_work_cgnapt_ipv4_prv(
                #endif
                return;
        }
-
+       last_update = rte_rdtsc();
        dest_address = rte_bswap32(*dst_addr);
-       /*Multiport Changes */
        uint32_t nhip = 0;
-       uint32_t ret;
-       ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-       if (!ret) {
-               dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
-
-               if (dest_if == INVALID_DESTIF) {
-                       p_nat->invalid_packets |= pkt_mask;
-                       p_nat->naptDroppedPktCount++;
-                       #ifdef CGNAPT_DEBUGGING
-                       p_nat->naptDroppedPktCount6++;
-                       #endif
-                       return;
-               }
-
-               do_local_nh_ipv4_cache(dest_if, p_nat);
-       }
-
+       struct arp_entry_data *ret_arp_data = NULL;
+       ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
        *outport_id = p_nat->outport_id[dest_if];
 
-       #ifdef CGNAPT_DBG_PRNT
-       if (CGNAPT_DEBUG > 2)
-               printf("Egress: \tphy_port:%d\t get_prv_to_pub():%d "
-                               "\tout_port:%d\n", pkt->port, dest_if,
-                               *outport_id);
-       #endif
+       if (arp_cache_dest_mac_present(dest_if)) {
+               ether_addr_copy(get_link_hw_addr(dest_if),(struct ether_addr *)eth_src);
+               arp_data_ptr[dest_if]->n_last_update = time(NULL);
 
-       if (local_dest_mac_present(dest_if)) {
-               memcpy(eth_dest,
-                               get_local_link_hw_addr(dest_if),
-                               sizeof(struct ether_addr));
-               memcpy(eth_src, get_link_hw_addr(dest_if),
-                               sizeof(struct ether_addr));
+               if (unlikely(ret_arp_data && ret_arp_data->num_pkts)) {
+                       printf("sending buffered packets\n");
+                       p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                       arp_send_buffered_pkts(ret_arp_data,
+                                (struct ether_addr *)eth_dest, *outport_id);
+
+               }
        } else {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-               if (unlikely(ret != ARP_FOUND)) {
+               if (unlikely(ret_arp_data == NULL)) {
 
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                               printf("%s: ARP Not Found, nhip: %x, "
-                               "outport_id: %d\n", __func__, nhip,
-                               *outport_id);
-                       }
+                       printf("%s: NHIP Not Found, nhip:%x , "
+                       "outport_id: %d\n", __func__, nhip,
+                       *outport_id);
 
                        /* Drop the pkt */
                        p_nat->invalid_packets |= pkt_mask;
@@ -3906,41 +3971,26 @@ pkt_work_cgnapt_ipv4_prv(
                        p_nat->naptDroppedPktCount4++;
                        #endif
                        return;
-
-               }
-
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
-                       printf("MAC found for ip 0x%x, port %d - %02x:%02x: "
-                       "%02x:%02x:%02x:%02x\n", dest_address,
-                       *outport_id,
-                       hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-                       hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-                       hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]);
-
-                       printf("Dest MAC before - %02x:%02x:%02x: "
-                       "%02x:%02x:%02x\n", eth_dest[0], eth_dest[1],
-                       eth_dest[2], eth_dest[3], eth_dest[4], eth_dest[5]);
                }
 
-               #endif
-
-               memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
-
-               link_hw_laddr_valid[dest_if] = 1;
-               memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                               sizeof(struct ether_addr));
+               if (ret_arp_data->status == INCOMPLETE || 
+                          ret_arp_data->status == PROBE) {
+                               if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                       /* Drop the pkt */
+                                       p_nat->invalid_packets |= pkt_mask;
+                                       p_nat->naptDroppedPktCount++;
 
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
-                       printf("Dest MAC after - %02x:%02x:%02x:%02x:%02x"
-                       ":%02x\n", eth_dest[0], eth_dest[1], eth_dest[2],
-                       eth_dest[3], eth_dest[4], eth_dest[5]);
+                                       #ifdef CGNAPT_DEBUGGING
+                                       p_nat->naptDroppedPktCount4++;
+                                       #endif
+                                       return;
+                               } else {
+                                       arp_pkts_mask |= pkt_mask;
+                                       arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                       return;
+                               }
                }
-               #endif
 
-               memcpy(eth_src, get_link_hw_addr(dest_if),
-                               sizeof(struct ether_addr));
        }
 
        {
@@ -4234,55 +4284,33 @@ pkt_work_cgnapt_ipv4_pub(
                        #endif
                        return;
                }
+       }
 
        dest_address = entry->data.u.prv_ip;
+       struct arp_entry_data *ret_arp_data = NULL;
+       ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+       *outport_id = p_nat->outport_id[dest_if];
 
-       ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-       if (!ret) {
-               dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
+       if (arp_cache_dest_mac_present(dest_if)) {
+               ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+               arp_data_ptr[dest_if]->n_last_update = time(NULL);
 
-               if (dest_if == INVALID_DESTIF) {
-                       p_nat->invalid_packets |= pkt_mask;
-                       p_nat->naptDroppedPktCount++;
-                       #ifdef CGNAPT_DEBUGGING
-                       p_nat->naptDroppedPktCount6++;
-                       #endif
-                       return;
+               if (ret_arp_data && ret_arp_data->num_pkts) {
+                       printf("sending buffered packets\n");
+                       p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                       arp_send_buffered_pkts(ret_arp_data,
+                                (struct ether_addr *)eth_dest, *outport_id);
                }
 
-               do_local_nh_ipv4_cache(dest_if, p_nat);
-       }
-
-               *outport_id = p_nat->outport_id[dest_if];
-
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2)
-                       printf("Ingress: \tphy_port:%d\t get_pub_to_prv():%d "
-                       "\tout_port%d\n", pkt->port, dest_if, *outport_id);
-               #endif
-       }
-
-       if (local_dest_mac_present(dest_if)) {
-               memcpy(eth_dest,
-                                get_local_link_hw_addr(dest_if),
-                                sizeof(struct ether_addr));
-               memcpy(eth_src, get_link_hw_addr(dest_if),
-                                sizeof(struct ether_addr));
        } else {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-               if (unlikely(ret != ARP_FOUND)) {
-
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
-                               /* Commented code may be required for debug
-                                * and future use, Please keep it */
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                               printf("%s: ARP Not Found, nhip: %x, "
-                               "outport_id: %d\n", __func__, nhip,
-                               *outport_id);
+               if (unlikely(ret_arp_data == NULL)) {
 
-                       }
+                       /* Commented code may be required for debug
+                        * and future use, Please keep it */
+                       printf("%s: NHIP Not Found, nhip: %x, "
+                       "outport_id: %d\n", __func__, nhip,
+                       *outport_id);
 
                        /* Drop the pkt */
                        p_nat->invalid_packets |= pkt_mask;
@@ -4294,42 +4322,24 @@ pkt_work_cgnapt_ipv4_pub(
                        return;
 
                }
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
-                       printf
-                               ("MAC found for ip 0x%x, port %d - %02x:%02x: "
-                               "%02x:%02x:%02x:%02x\n", dest_address,
-                               *outport_id,
-                               hw_addr.addr_bytes[0], hw_addr.addr_bytes[1],
-                               hw_addr.addr_bytes[2], hw_addr.addr_bytes[3],
-                               hw_addr.addr_bytes[4], hw_addr.addr_bytes[5]
-                               );
-
-                       printf
-                               ("Dest MAC before - %02x:%02x:%02x:%02x "
-                               ":%02x:%02x\n", eth_dest[0], eth_dest[1],
-                               eth_dest[2], eth_dest[3], eth_dest[4],
-                               eth_dest[5]);
-               }
-               #endif
 
-               memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
-
-               link_hw_laddr_valid[dest_if] = 1;
-               memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                sizeof(struct ether_addr));
+               if (ret_arp_data->status == INCOMPLETE || 
+                       ret_arp_data->status == PROBE) {
+                       if (ret_arp_data->num_pkts >= NUM_DESC) {
+                               /* Drop the pkt */
+                               p_nat->invalid_packets |= pkt_mask;
+                               p_nat->naptDroppedPktCount++;
 
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
-                       printf("Dest MAC after - "
-                       "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                       eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
-                       eth_dest[4], eth_dest[5]);
+                               #ifdef CGNAPT_DEBUGGING
+                               p_nat->naptDroppedPktCount4++;
+                               #endif
+                               return;
+                       } else {
+                               arp_pkts_mask |= pkt_mask;
+                               arp_queue_unresolved_packet(ret_arp_data, pkt);
+                               return;
+                       }
                }
-               #endif
-
-               memcpy(eth_src, get_link_hw_addr(dest_if),
-                                sizeof(struct ether_addr));
        }
 
        {
@@ -4466,7 +4476,7 @@ pkt_work_cgnapt_ipv4_pub(
                if (ct_position < 0){
                        p_nat->invalid_packets |= pkt_mask;
 
-                        p_nat->naptDroppedPktCount++;
+                       p_nat->naptDroppedPktCount++;
                        return;
                }
                        #ifdef ALGDBG
@@ -4708,101 +4718,60 @@ pkt4_work_cgnapt_ipv4_prv(
                                #endif
                                continue;
                        }
-
-                       dest_address = rte_bswap32(*dst_addr);
-               ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-               if (!ret) {
-                       dest_if = get_prv_to_pub_port(&dest_address,
-                                       IP_VERSION_4);
-               if (dest_if == INVALID_DESTIF) {
-                       p_nat->invalid_packets |= pkt_mask;
-                       p_nat->naptDroppedPktCount++;
-                       #ifdef CGNAPT_DEBUGGING
-                       p_nat->naptDroppedPktCount6++;
-                       #endif
-                       continue;
-               }
-                       do_local_nh_ipv4_cache(dest_if, p_nat);
                }
-                       *outport_id = p_nat->outport_id[dest_if];
 
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2)
-                       printf("Egress: \tphy_port:%d\t "
-                       "get_prv_to_pub():%d \tout_port:%d\n",
-                                pkt->port, dest_if, *outport_id);
-               #endif
-               }
+               dest_address = rte_bswap32(*dst_addr);
+               struct arp_entry_data *ret_arp_data = NULL;
+               uint64_t start, end;
+               ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+               *outport_id = p_nat->outport_id[dest_if];
+               if (arp_cache_dest_mac_present(dest_if)) {
+                       ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+                       arp_data_ptr[dest_if]->n_last_update = time(NULL);
+               
+                       if (ret_arp_data && ret_arp_data->num_pkts) {
+                               printf("sending buffered packets\n");
+                               p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                               arp_send_buffered_pkts(ret_arp_data,
+                                        (struct ether_addr *)eth_dest, *outport_id);
+                       }
 
-               if (local_dest_mac_present(dest_if)) {
-                       memcpy(eth_dest,
-                                        get_local_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
-                       memcpy(eth_src,
-                                        get_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
                } else {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
 
-               if (unlikely(ret != ARP_FOUND)) {
+                       if (unlikely(ret_arp_data == NULL)) {
 
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
                                printf("%s: ARP Not Found, nhip: %x, "
                                "outport_id: %d\n", __func__, nhip,
                                *outport_id);
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                       }
 
-                       /* Drop the pkt */
-                       p_nat->invalid_packets |= pkt_mask;
-                       p_nat->naptDroppedPktCount++;
-
-                       #ifdef CGNAPT_DEBUGGING
-                       p_nat->naptDroppedPktCount4++;
-                       #endif
-                       continue;
+                               /* Drop the pkt */
+                               p_nat->invalid_packets |= pkt_mask;
+                               p_nat->naptDroppedPktCount++;
 
-               }
-                       #ifdef CGNAPT_DBG_PRNT
-                       if (CGNAPT_DEBUG > 2) {
-                               printf("MAC found for ip 0x%x, port %d - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                                       dest_address,
-                                       *outport_id,
-                                       hw_addr.addr_bytes[0],
-                                       hw_addr.addr_bytes[1],
-                                       hw_addr.addr_bytes[2],
-                                       hw_addr.addr_bytes[3],
-                                       hw_addr.addr_bytes[4],
-                                       hw_addr.addr_bytes[5]
-                                       );
+                               #ifdef CGNAPT_DEBUGGING
+                               p_nat->naptDroppedPktCount4++;
+                               #endif
+                               continue;
 
-                               printf("Dest MAC before - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                               eth_dest[0], eth_dest[1], eth_dest[2],
-                               eth_dest[3], eth_dest[4], eth_dest[5]);
                        }
-                       #endif
-
-                       memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-                       link_hw_laddr_valid[dest_if] = 1;
-                       memcpy(&link_hw_laddr[dest_if], &hw_addr,
-                                        sizeof(struct ether_addr));
+                       if (ret_arp_data->status == INCOMPLETE || 
+                               ret_arp_data->status == PROBE) {
+                               if (ret_arp_data->num_pkts >= NUM_DESC) {
+                                       /* Drop the pkt */
+                                       p_nat->invalid_packets |= pkt_mask;
+                                       p_nat->naptDroppedPktCount++;
 
-                       #ifdef CGNAPT_DBG_PRNT
-                       if (CGNAPT_DEBUG > 2) {
-                               printf("Dest MAC after - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                               eth_dest[0], eth_dest[1], eth_dest[2],
-                               eth_dest[3], eth_dest[4], eth_dest[5]);
+                                       #ifdef CGNAPT_DEBUGGING
+                                       p_nat->naptDroppedPktCount4++;
+                                       #endif
+                                       continue;
+                               } else {
+                                       arp_pkts_mask |= pkt_mask;
+                                       arp_queue_unresolved_packet(ret_arp_data, pkt);
+                                       continue;
+                               }
                        }
-                       #endif
-
-                       memcpy(eth_src,
-                                        get_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
                }
 
                {
@@ -5050,7 +5019,7 @@ pkt4_work_cgnapt_ipv4_pub(
                                #ifdef CGNAPT_DEBUGGING
                                p_nat->naptDroppedPktCount3++;
                                #endif
-
+                               printf("causing p_nat->naptDroppedPktCount3\n");
                                continue;
                        }
 
@@ -5114,54 +5083,30 @@ pkt4_work_cgnapt_ipv4_pub(
                                #endif
                                continue;
                        }
-
-                       dest_address = entry->data.u.prv_ip;
-       ret = local_get_nh_ipv4(dest_address, &dest_if, &nhip, p_nat);
-       if (!ret) {
-               dest_if = get_prv_to_pub_port(&dest_address, IP_VERSION_4);
-
-               if (dest_if == INVALID_DESTIF) {
-                       p_nat->invalid_packets |= pkt_mask;
-                       p_nat->naptDroppedPktCount++;
-                       #ifdef CGNAPT_DEBUGGING
-                       p_nat->naptDroppedPktCount6++;
-                       #endif
-                       continue;
                }
+               dest_address = entry->data.u.prv_ip;
+               struct arp_entry_data *ret_arp_data = NULL;
+               ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)eth_dest);
+               *outport_id = p_nat->outport_id[dest_if];
 
-               do_local_nh_ipv4_cache(dest_if, p_nat);
-       }
-
-                       *outport_id = p_nat->outport_id[dest_if];
-
-                       #ifdef CGNAPT_DBG_PRNT
-                       if (CGNAPT_DEBUG > 2)
-                               printf("Ingress: \tphy_port:%d\t "
-                               "get_pub_to_prv():%d \tout_port%d\n",
-                                        pkt->port, dest_if,
-                                        *outport_id);
-                       #endif
+       if (arp_cache_dest_mac_present(dest_if)) {
+               ether_addr_copy(get_link_hw_addr(dest_if), (struct ether_addr *)eth_src);
+               arp_data_ptr[dest_if]->n_last_update = time(NULL);
+               
+               if (ret_arp_data && ret_arp_data->num_pkts) {
+                       printf("sending buffered packets\n");
+                       p_nat->naptedPktCount += ret_arp_data->num_pkts;
+                       arp_send_buffered_pkts(ret_arp_data,
+                                (struct ether_addr *)eth_dest, *outport_id);
                }
 
-               if (local_dest_mac_present(dest_if)) {
-                       memcpy(eth_dest,
-                                        get_local_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
-                       memcpy(eth_src,
-                                        get_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
-               } else {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+       } else {
 
-               if (unlikely(ret != ARP_FOUND)) {
+               if (unlikely(ret_arp_data == NULL)) {
 
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
-                               printf("%s: ARP Not Found, nhip: %x, "
-                               "outport_id: %d\n", __func__, nhip,
-                               *outport_id);
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                       }
+                       printf("%s: NHIP Not Found, nhip: %x, "
+                       "outport_id: %d\n", __func__, nhip,
+                       *outport_id);
 
                        /* Drop the pkt */
                        p_nat->invalid_packets |= pkt_mask;
@@ -5171,47 +5116,26 @@ pkt4_work_cgnapt_ipv4_pub(
                        p_nat->naptDroppedPktCount4++;
                        #endif
                        continue;
-
                }
-                       #ifdef CGNAPT_DBG_PRNT
-                       if (CGNAPT_DEBUG > 2) {
-                               printf("MAC found for ip 0x%x, port %d - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                               dest_address, *outport_id,
-                               hw_addr.addr_bytes[0],
-                               hw_addr.addr_bytes[1],
-                               hw_addr.addr_bytes[2],
-                               hw_addr.addr_bytes[3],
-                               hw_addr.addr_bytes[4],
-                               hw_addr.addr_bytes[5]
-                               );
-
-                               printf("Dest MAC before - "
-                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                               eth_dest[0], eth_dest[1], eth_dest[2],
-                               eth_dest[3], eth_dest[4], eth_dest[5]);
-                       }
-                       #endif
-
-                       memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-                       link_hw_laddr_valid[dest_if] = 1;
-                       memcpy(&link_hw_laddr[dest_if],
-                                        &hw_addr, sizeof(struct ether_addr));
+               if (ret_arp_data->status == INCOMPLETE || 
+                       ret_arp_data->status == PROBE) {
+                       if (ret_arp_data->num_pkts >= NUM_DESC) {
+                               /* Drop the pkt */
+                               p_nat->invalid_packets |= pkt_mask;
+                               p_nat->naptDroppedPktCount++;
 
-                       #ifdef CGNAPT_DBG_PRNT
-                       if (CGNAPT_DEBUG > 2) {
-                               printf("Dest MAC after - %02x:%02x:%02x: "
-                               "%02x:%02x:%02x\n",
-                               eth_dest[0], eth_dest[1], eth_dest[2],
-                               eth_dest[3], eth_dest[4], eth_dest[5]);
+                               #ifdef CGNAPT_DEBUGGING
+                               p_nat->naptDroppedPktCount4++;
+                               #endif
+                               continue;
+                       } else {
+                               arp_pkts_mask |= pkt_mask;
+                               arp_queue_unresolved_packet(ret_arp_data, pkt);
+                               continue;
                        }
-                       #endif
-
-                       memcpy(eth_src,
-                                        get_link_hw_addr(dest_if),
-                                        sizeof(struct ether_addr));
                }
+       }
 
                {
                        /* Ingress */
@@ -6159,7 +6083,6 @@ pkt_work_cgnapt_ipv6_prv(
        __rte_unused void *arg,
        struct pipeline_cgnapt *p_nat)
 {
-
        /* index into hash table entries */
        int hash_table_entry = p_nat->lkup_indx[pkt_num];
 
@@ -6351,23 +6274,17 @@ pkt_work_cgnapt_ipv6_prv(
        #endif
 
        if (local_dest_mac_present(dest_if)) {
-               memcpy(eth_dest,
-                               get_local_link_hw_addr(dest_if),
-                               sizeof(struct ether_addr));
                memcpy(eth_src, get_link_hw_addr(dest_if),
                                sizeof(struct ether_addr));
        } else {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+               struct arp_entry_data *ret_arp_data;
+               ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)&hw_addr);
 
-               if (unlikely(ret != ARP_FOUND)) {
+               if (unlikely(ret_arp_data == NULL)) {
 
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
-                               printf("%s: ARP Not Found, nhip: %x, "
-                               "outport_id: %d\n", __func__, nhip,
-                               *outport_id);
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                       }
+                       printf("%s: NHIP Not Found, nhip: %x, "
+                       "outport_id: %d\n", __func__, nhip,
+                       *outport_id);
 
                        /* Drop the pkt */
                        p_nat->invalid_packets |= pkt_mask;
@@ -6377,10 +6294,12 @@ pkt_work_cgnapt_ipv6_prv(
                        p_nat->naptDroppedPktCount4++;
                        #endif
                        return;
-
                }
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
+
+               if (ret_arp_data->status == COMPLETE) {
+
+                       #ifdef CGNAPT_DBG_PRNT
+                       if (CGNAPT_DEBUG > 2) {
                        printf("MAC found for ip 0x%x, port %d - %02x:%02x: "
                        "%02x:%02x:%02x:%02x\n", dest_address,
                        *outport_id,
@@ -6392,22 +6311,28 @@ pkt_work_cgnapt_ipv6_prv(
                        "%02x:%02x\n", eth_dest[0], eth_dest[1],
                        eth_dest[2], eth_dest[3],
                        eth_dest[4], eth_dest[5]);
-               }
-               #endif
+                       }
+                       #endif
 
-               memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
+                       //memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
-               #ifdef CGNAPT_DBG_PRNT
-               if (CGNAPT_DEBUG > 2) {
-                       printf("Dest MAC after - "
-                       "%02x:%02x:%02x:%02x:%02x:%02x\n",
-                       eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
-                       eth_dest[4], eth_dest[5]);
-               }
-               #endif
+                       #ifdef CGNAPT_DBG_PRNT
+                       if (CGNAPT_DEBUG > 2) {
+                               printf("Dest MAC after - "
+                               "%02x:%02x:%02x:%02x:%02x:%02x\n",
+                               eth_dest[0], eth_dest[1], eth_dest[2], eth_dest[3],
+                               eth_dest[4], eth_dest[5]);
+                       }
+                       #endif
 
-               memcpy(eth_src, get_link_hw_addr(dest_if),
+                       memcpy(eth_src, get_link_hw_addr(dest_if),
                                 sizeof(struct ether_addr));
+               } else if (ret_arp_data->status == INCOMPLETE || 
+                       ret_arp_data->status == PROBE) {
+                       arp_queue_unresolved_packet(ret_arp_data,
+                               pkt);
+                       return;
+               }
        }
 
        {
@@ -6874,20 +6799,15 @@ pkt4_work_cgnapt_ipv6_prv(
 
                memset(nh_ipv6, 0, 16);
 
-               {
-               int ret;
-               ret = get_dest_mac_addr_port(dest_address, &dest_if, &hw_addr);
+       {
+               struct arp_entry_data *ret_arp_data;
+               ret_arp_data = get_dest_mac_addr_port(dest_address, &dest_if, (struct ether_addr *)&hw_addr);
 
-               if (unlikely(ret != ARP_FOUND)) {
+               if (unlikely(ret_arp_data == NULL)) {
 
-                       if (unlikely(ret == ARP_NOT_FOUND)) {
-                               /* Commented code may be required for debug
-                                * and future use, Please keep it */
-                               //request_arp(*outport_id, nhip, p_nat->p.p);
-                               printf("%s: ARP Not Found, nhip: %x, "
-                               "outport_id: %d\n", __func__, nhip,
-                               *outport_id);
-                       }
+                       printf("%s: NHIP Not Found, nhip: %x, "
+                       "outport_id: %d\n", __func__, nhip,
+                       *outport_id);
 
                        /* Drop the pkt */
                        p_nat->invalid_packets |= pkt_mask;
@@ -6897,9 +6817,10 @@ pkt4_work_cgnapt_ipv6_prv(
                        p_nat->naptDroppedPktCount4++;
                        #endif
                        continue;
-
                }
 
+               if (ret_arp_data->status == COMPLETE) {
+
                        #ifdef CGNAPT_DBG_PRNT
                        if (CGNAPT_DEBUG > 2) {
                                printf("MAC found for ip 0x%x, port %d - "
@@ -6920,7 +6841,7 @@ pkt4_work_cgnapt_ipv6_prv(
                        }
                        #endif
 
-                       memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
+                       //memcpy(eth_dest, &hw_addr, sizeof(struct ether_addr));
 
                        #ifdef CGNAPT_DBG_PRNT
                        if (CGNAPT_DEBUG > 2) {
@@ -6931,10 +6852,15 @@ pkt4_work_cgnapt_ipv6_prv(
                        }
                        #endif
 
-                       memcpy(eth_src,
-                                        get_link_hw_addr(dest_if),
+                       memcpy(eth_src, get_link_hw_addr(dest_if),
                                         sizeof(struct ether_addr));
+               } else if (ret_arp_data->status == INCOMPLETE || 
+                       ret_arp_data->status == PROBE) {
+                       arp_queue_unresolved_packet(ret_arp_data,
+                               pkt);
+                       continue;
                }
+       }
 
                {
                        /* Egress */
@@ -7050,8 +6976,7 @@ pkt4_work_cgnapt_ipv6_pub(
                uint8_t dest_addr_ipv6[16];
                uint8_t nh_ipv6[16];
                uint32_t dest_if = INVALID_DESTIF;
-               /* Ingress */
-               {
+               { /*start of Ingress */
 
                        if (unlikely(protocol == IP_PROTOCOL_UDP
                                && rte_be_to_cpu_16(*src_port) == 53)) {
@@ -7086,7 +7011,7 @@ pkt4_work_cgnapt_ipv6_pub(
                        }
 
                        *outport_id = p_nat->outport_id[dest_if];
-               }
+               }/* end of ingress */
 
                #ifdef CGNAPT_DEBUGGING
                static int static_count;
@@ -7147,7 +7072,7 @@ pkt4_work_cgnapt_ipv6_pub(
                }
 
                {
-                       /* Ingress */
+               /* start of Ingress */
 
                        convert_ipv4_to_ipv6(pkt, &ipv4_hdr);
 
@@ -7175,7 +7100,7 @@ pkt4_work_cgnapt_ipv6_pub(
                        #endif
 
                        p_nat->inaptedPktCount++;
-               }
+               } /* end of ingress */
 
                p_nat->naptedPktCount++;
 
@@ -7185,7 +7110,7 @@ pkt4_work_cgnapt_ipv6_pub(
                        else
                                sw_checksum(pkt, pkt_type);
                #endif
-       }
+       } /* end of for loop */
 }
 
 /**
@@ -7891,12 +7816,11 @@ pkt_miss_cgnapt(struct pipeline_cgnapt_entry_key *key,
                                        printf("Add Dynamic NAT entry failed "
                                        "in pkt!!!\n");
                                #endif
-               } else {
+                       } else {
                                #ifdef CGNAPT_DEBUGGING
                                p_nat->missedpktcount11++;
                                #endif
-               }
-
+                       }
                }
 
        } else if (!is_phy_port_privte(phy_port)) {
@@ -8389,7 +8313,6 @@ pipeline_cgnapt_parse_args(struct pipeline_cgnapt *p,
        return 0;
 
 }
-
 /**
  * Function to initialize the pipeline
  *
@@ -8417,6 +8340,7 @@ static void *pipeline_cgnapt_init(struct pipeline_params *params, void *arg)
        size = RTE_CACHE_LINE_ROUNDUP(sizeof(struct pipeline_cgnapt));
        p = rte_zmalloc(NULL, size, RTE_CACHE_LINE_SIZE);
        p_nat = (struct pipeline_cgnapt *)p;
+       global_pnat = p_nat;
        if (p == NULL)
                return NULL;
 
@@ -9260,10 +9184,10 @@ pipeline_cgnapt_msg_req_entry_addm_pair(
        }
        #endif
 
-       if (CGNAPT_DEBUG > 2)
-               printf("key.ip %x, key.port %d", key.ip, key.port);
-               printf("key.pid %d, in_type %d,", key.pid, type);
-               printf("entry_type %d\n", entry.data.type);
+       //if (CGNAPT_DEBUG > 2)
+               //printf("key.ip %x, key.port %d", key.ip, key.port);
+               //printf("key.pid %d, in_type %d,", key.pid, type);
+               //printf("entry_type %d\n", entry.data.type);
 
        int32_t position = rte_hash_add_key(napt_common_table, &key);