VNFs/DPPD-PROX/packet_utils.c

   1 /*
   2 // Copyright (c) 2010-2020 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 #include <rte_lcore.h>
  18 #include <rte_hash.h>
  19 #include <rte_hash_crc.h>
  20 #include <rte_lpm.h>
  21
  22 #include "task_base.h"
  23 #include "lconf.h"
  24 #include "prefetch.h"
  25 #include "log.h"
  26 #include "defines.h"
  27 #include "handle_master.h"
  28 #include "prox_port_cfg.h"
  29 #include "packet_utils.h"
  30 #include "prox_shared.h"
  31 #include "prox_lua.h"
  32 #include "hash_entry_types.h"
  33 #include "prox_compat.h"
  34 #include "tx_pkt.h"
  35
  36 static inline int find_ip(struct ether_hdr_arp *pkt, uint16_t len, uint32_t *ip_dst)
  37 {
  38         prox_rte_vlan_hdr *vlan_hdr;
  39         prox_rte_ether_hdr *eth_hdr = (prox_rte_ether_hdr*)pkt;
  40         prox_rte_ipv4_hdr *ip;
  41         uint16_t ether_type = eth_hdr->ether_type;
  42         uint16_t l2_len = sizeof(prox_rte_ether_hdr);
  43
  44         // Unstack VLAN tags
  45         while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(prox_rte_vlan_hdr) < len)) {
  46                 vlan_hdr = (prox_rte_vlan_hdr *)((uint8_t *)pkt + l2_len);
  47                 l2_len +=4;
  48                 ether_type = vlan_hdr->eth_proto;
  49         }
  50
  51         switch (ether_type) {
  52         case ETYPE_MPLSU:
  53         case ETYPE_MPLSM:
  54                 // In case of MPLS, next hop MAC is based on MPLS, not destination IP
  55                 l2_len = 0;
  56                 break;
  57         case ETYPE_IPv4:
  58                 break;
  59         case ETYPE_EoGRE:
  60         case ETYPE_ARP:
  61         case ETYPE_IPv6:
  62                 l2_len = 0;
  63                 break;
  64         default:
  65                 l2_len = 0;
  66                 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
  67                 break;
  68         }
  69
  70         if (l2_len && (l2_len + sizeof(prox_rte_ipv4_hdr) <= len)) {
  71                 prox_rte_ipv4_hdr *ip = (prox_rte_ipv4_hdr *)((uint8_t *)pkt + l2_len);
  72                 // TODO: implement LPM => replace ip_dst by next hop IP DST
  73                 *ip_dst = ip->dst_addr;
  74                 return 0;
  75         }
  76         return -1;
  77 }
  78
  79 /* This implementation could be improved: instead of checking each time we send a packet whether we need also
  80    to send an ARP, we should only check whether the MAC is valid.
  81    We should check arp_update_time in the master process. This would also require the generating task to clear its arp ring
  82    to avoid sending many ARP while starting after a long stop.
  83    We could also check for arp_timeout in the master so that dataplane has only to check whether MAC is available
  84    but this would require either thread safety, or the exchange of information between master and generating core.
  85 */
  86
  87 static inline int add_key_and_send_arp(struct rte_hash *ip_hash, uint32_t *ip_dst, struct arp_table *entries,  uint64_t tsc, uint64_t hz, uint32_t arp_update_time, prox_next_hop_index_type nh, uint64_t **time)
  88 {
  89         int ret = rte_hash_add_key(ip_hash, (const void *)ip_dst);
  90         if (unlikely(ret < 0)) {
  91                 // No reason to send ARP, as reply would be anyhow ignored
  92                 plogx_err("Unable to add ip "IPv4_BYTES_FMT" in mac_hash\n", IP4(*ip_dst));
  93                 return DROP_MBUF;
  94         } else {
  95                 entries[ret].ip = *ip_dst;
  96                 entries[ret].nh = nh;
  97                 *time = &entries[ret].arp_update_time;
  98         }
  99         return SEND_ARP;
 100 }
 101
 102 static inline int update_mac_and_send_mbuf(struct arp_table *entry, prox_rte_ether_addr *mac, uint64_t tsc, uint64_t hz, uint32_t arp_update_time, uint64_t **time)
 103 {
 104         if (likely((tsc < entry->arp_update_time) && (tsc < entry->arp_timeout))) {
 105                 memcpy(mac, &entry->mac, sizeof(prox_rte_ether_addr));
 106                 return SEND_MBUF;
 107         } else if (tsc > entry->arp_update_time) {
 108                 // long time since we have sent an arp, send arp
 109                 *time = &entry->arp_update_time;
 110                 if (tsc < entry->arp_timeout){
 111                         // MAC is valid in the table => send also the mbuf
 112                         memcpy(mac, &entry->mac, sizeof(prox_rte_ether_addr));
 113                         return SEND_MBUF_AND_ARP;
 114                 } else {
 115                         // MAC still unknown, or timed out => only send ARP
 116                         return SEND_ARP;
 117                 }
 118         }
 119         // MAC is unknown and we already sent an ARP recently, drop mbuf and wait for ARP reply
 120         return DROP_MBUF;
 121 }
 122
 123 int write_dst_mac(struct task_base *tbase, struct rte_mbuf *mbuf, uint32_t *ip_dst, uint64_t **time, uint64_t tsc)
 124 {
 125         const uint64_t hz = rte_get_tsc_hz();
 126         struct ether_hdr_arp *packet = rte_pktmbuf_mtod(mbuf, struct ether_hdr_arp *);
 127         prox_rte_ether_addr *mac = &packet->ether_hdr.d_addr;
 128         prox_next_hop_index_type next_hop_index;
 129         static uint64_t last_tsc = 0, n_no_route = 0;
 130
 131         struct l3_base *l3 = &(tbase->l3);
 132
 133         // First find the next hop
 134         if (l3->ipv4_lpm) {
 135                 // A routing table was configured
 136                 // If a gw (gateway_ipv4) is also specified, it is used as default gw only i.e. lowest priority (shortest prefix)
 137                 // This is implemented automatically through lpm
 138                 uint16_t len = rte_pktmbuf_pkt_len(mbuf);
 139                 if (find_ip(packet, len, ip_dst) != 0) {
 140                         // Unable to find IP address => non IP packet => send it as it
 141                         return SEND_MBUF;
 142                 }
 143                 if (unlikely(rte_lpm_lookup(l3->ipv4_lpm, rte_bswap32(*ip_dst), &next_hop_index) != 0)) {
 144                         // Prevent printing too many messages
 145                         n_no_route++;
 146                         if (tsc > last_tsc + rte_get_tsc_hz()) {
 147                                 plog_err("No route to IP "IPv4_BYTES_FMT" (%ld times)\n", IP4(*ip_dst), n_no_route);
 148                                 last_tsc = tsc;
 149                                 n_no_route = 0;
 150                         }
 151                         return DROP_MBUF;
 152                 }
 153                 struct arp_table *entry = &l3->next_hops[next_hop_index];
 154
 155                 if (entry->ip) {
 156                         *ip_dst = entry->ip;
 157                         return update_mac_and_send_mbuf(entry, mac, tsc, hz, l3->arp_update_time, time);
 158                 }
 159
 160                 // no next ip: this is a local route
 161                 // Find IP in lookup table. Send ARP if not found
 162                 int ret = rte_hash_lookup(l3->ip_hash, (const void *)ip_dst);
 163                 if (unlikely(ret < 0)) {
 164                         // IP not found, try to send an ARP
 165                         return add_key_and_send_arp(l3->ip_hash, ip_dst, l3->arp_table, tsc, hz, l3->arp_update_time, MAX_HOP_INDEX, time);
 166                 } else {
 167                         return update_mac_and_send_mbuf(&l3->arp_table[ret], mac, tsc, hz, l3->arp_update_time, time);
 168                 }
 169                 return 0;
 170         }
 171         // No Routing table specified: only a local ip and maybe a gateway
 172         // Old default behavior: if a gw is specified, ALL packets go to this gateway (even those we could send w/o the gw
 173         if (l3->gw.ip) {
 174                 if (likely((l3->flags & FLAG_DST_MAC_KNOWN) && (tsc < l3->gw.arp_update_time) && (tsc < l3->gw.arp_timeout))) {
 175                         memcpy(mac, &l3->gw.mac, sizeof(prox_rte_ether_addr));
 176                         return SEND_MBUF;
 177                 } else if (tsc > l3->gw.arp_update_time) {
 178                         // long time since we have successfully sent an arp, send arp
 179                         // If sending ARP failed (ring full) then arp_update_time is not updated to avoid having to wait 1 sec to send ARP REQ again
 180                         *time = &l3->gw.arp_update_time;
 181                         *ip_dst = l3->gw.ip;
 182                         if ((l3->flags & FLAG_DST_MAC_KNOWN) && (tsc < l3->gw.arp_timeout)){
 183                                 // MAC is valid in the table => send also the mbuf
 184                                 memcpy(mac, &l3->gw.mac, sizeof(prox_rte_ether_addr));
 185                                 return SEND_MBUF_AND_ARP;
 186                         } else {
 187                                 // MAC still unknown, or timed out => only send ARP
 188                                 return SEND_ARP;
 189                         }
 190                 } else {
 191                         // MAC is unknown and we already sent an ARP recently, drop mbuf and wait for ARP reply
 192                         return DROP_MBUF;
 193                 }
 194         }
 195
 196         uint16_t len = rte_pktmbuf_pkt_len(mbuf);
 197         if (find_ip(packet, len, ip_dst) != 0) {
 198                 // Unable to find IP address => non IP packet => send it as it
 199                 return SEND_MBUF;
 200         }
 201         if (likely(l3->n_pkts < 4)) {
 202                 for (unsigned int idx = 0; idx < l3->n_pkts; idx++) {
 203                         if (*ip_dst == l3->optimized_arp_table[idx].ip) {
 204                                  // IP address already in table
 205                                 return update_mac_and_send_mbuf(&l3->optimized_arp_table[idx], mac, tsc, hz, l3->arp_update_time, time);
 206                         }
 207                 }
 208                 // IP address not found in table
 209                 l3->optimized_arp_table[l3->n_pkts].ip = *ip_dst;
 210                 *time = &l3->optimized_arp_table[l3->n_pkts].arp_update_time;
 211                 l3->n_pkts++;
 212
 213                 if (l3->n_pkts < 4) {
 214                         return SEND_ARP;
 215                 }
 216
 217                 // We have too many IP addresses to search linearly; lets use hash table instead => copy all entries in hash table
 218                 for (uint32_t idx = 0; idx < l3->n_pkts; idx++) {
 219                         uint32_t ip = l3->optimized_arp_table[idx].ip;
 220                         int ret = rte_hash_add_key(l3->ip_hash, (const void *)&ip);
 221                         if (ret < 0) {
 222                                 // This should not happen as few entries so far.
 223                                 // If it happens, we still send the ARP as easier:
 224                                 //      If the ARP corresponds to this error, the ARP reply will be ignored
 225                                 //      If ARP does not correspond to this error/ip, then ARP reply will be handled.
 226                                 plogx_err("Unable add ip "IPv4_BYTES_FMT" in mac_hash (already %d entries)\n", IP4(ip), idx);
 227                         } else {
 228                                 memcpy(&l3->arp_table[ret], &l3->optimized_arp_table[idx], sizeof(struct arp_table));
 229                         }
 230                 }
 231                 return SEND_ARP;
 232         } else {
 233                 // Find IP in lookup table. Send ARP if not found
 234                 int ret = rte_hash_lookup(l3->ip_hash, (const void *)ip_dst);
 235                 if (unlikely(ret < 0)) {
 236                         // IP not found, try to send an ARP
 237                         return add_key_and_send_arp(l3->ip_hash, ip_dst, &l3->arp_table[ret], tsc, hz, l3->arp_update_time, MAX_HOP_INDEX, time);
 238                 } else {
 239                         // IP has been found
 240                         return update_mac_and_send_mbuf(&l3->arp_table[ret], mac, tsc, hz, l3->arp_update_time, time);
 241                 }
 242         }
 243         // Should not happen
 244         return DROP_MBUF;
 245 }
 246
 247 void task_init_l3(struct task_base *tbase, struct task_args *targ)
 248 {
 249         static char hash_name[30];
 250         uint32_t n_entries = MAX_ARP_ENTRIES * 4;
 251         const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
 252         sprintf(hash_name, "A%03d_%03d_mac_table", targ->lconf->id, targ->id);
 253
 254         hash_name[0]++;
 255
 256         struct rte_hash_parameters hash_params = {
 257                 .name = hash_name,
 258                 .entries = n_entries,
 259                 .key_len = sizeof(uint32_t),
 260                 .hash_func = rte_hash_crc,
 261                 .hash_func_init_val = 0,
 262         };
 263         tbase->l3.ip_hash = rte_hash_create(&hash_params);
 264         PROX_PANIC(tbase->l3.ip_hash == NULL, "Failed to set up ip hash table\n");
 265
 266         tbase->l3.arp_table = (struct arp_table *)prox_zmalloc(n_entries * sizeof(struct arp_table), socket_id);
 267         PROX_PANIC(tbase->l3.arp_table == NULL, "Failed to allocate memory for %u entries in arp table\n", n_entries);
 268         plog_info("\tarp table, with %d entries of size %ld\n", n_entries, sizeof(struct l3_base));
 269
 270         targ->lconf->ctrl_func_p[targ->task] = handle_ctrl_plane_pkts;
 271         targ->lconf->ctrl_timeout = freq_to_tsc(targ->ctrl_freq);
 272         tbase->l3.gw.ip = rte_cpu_to_be_32(targ->gateway_ipv4);
 273         tbase->flags |= TASK_L3;
 274         tbase->l3.core_id = targ->lconf->id;
 275         tbase->l3.task_id = targ->id;
 276         tbase->l3.tmaster = targ->tmaster;
 277         tbase->l3.seed = (uint)rte_rdtsc();
 278         if (targ->arp_timeout != 0)
 279                 tbase->l3.arp_timeout = targ->arp_timeout;
 280         else
 281                 tbase->l3.arp_timeout = DEFAULT_ARP_TIMEOUT;
 282         if (targ->arp_update_time != 0)
 283                 tbase->l3.arp_update_time = targ->arp_update_time;
 284         else
 285                 tbase->l3.arp_update_time = DEFAULT_ARP_UPDATE_TIME;
 286 }
 287
 288 void task_start_l3(struct task_base *tbase, struct task_args *targ)
 289 {
 290         const int NB_ARP_MBUF = 1024;
 291         const int ARP_MBUF_SIZE = 2048;
 292         const int NB_CACHE_ARP_MBUF = 256;
 293         const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
 294
 295         struct prox_port_cfg *port = find_reachable_port(targ);
 296         if (port && (tbase->l3.arp_pool == NULL)) {
 297                 static char name[] = "arp0_pool";
 298                 tbase->l3.reachable_port_id = port - prox_port_cfg;
 299                 if (targ->local_ipv4) {
 300                         tbase->local_ipv4 = rte_be_to_cpu_32(targ->local_ipv4);
 301                         register_ip_to_ctrl_plane(tbase->l3.tmaster, tbase->local_ipv4, tbase->l3.reachable_port_id, targ->lconf->id, targ->id);
 302                 }
 303                 if (strcmp(targ->route_table, "") != 0) {
 304                         struct lpm4 *lpm;
 305                         int ret;
 306
 307                         PROX_PANIC(tbase->local_ipv4 == 0, "missing local_ipv4 will route table is specified in L3 mode\n");
 308
 309                         // LPM might be modified runtime => do not share with other cores
 310                         ret = lua_to_lpm4(prox_lua(), GLOBAL, targ->route_table, socket_id, &lpm);
 311                         PROX_PANIC(ret, "Failed to load IPv4 LPM:\n%s\n", get_lua_to_errors());
 312
 313                         tbase->l3.ipv4_lpm = lpm->rte_lpm;
 314                         tbase->l3.next_hops = prox_zmalloc(sizeof(*tbase->l3.next_hops) * MAX_HOP_INDEX, socket_id);
 315                         PROX_PANIC(tbase->l3.next_hops == NULL, "Could not allocate memory for next hop\n");
 316
 317                         for (uint32_t i = 0; i < MAX_HOP_INDEX; i++) {
 318                                 if (!lpm->next_hops[i].ip_dst)
 319                                         continue;
 320                                 tbase->l3.nb_gws++;
 321                                 tbase->l3.next_hops[i].ip = rte_bswap32(lpm->next_hops[i].ip_dst);
 322                                 int tx_port = lpm->next_hops[i].mac_port.out_idx;
 323                                 // gen only supports one port right now .... hence port = 0
 324                                 if ((tx_port > targ->nb_txports - 1) && (tx_port > targ->nb_txrings - 1)) {
 325                                         PROX_PANIC(1, "Routing Table contains port %d but only %d tx port/ %d ring:\n", tx_port, targ->nb_txports, targ->nb_txrings);
 326                                 }
 327                         }
 328                         plog_info("Using routing table %s in l3 mode, with %d gateways\n", targ->route_table, tbase->l3.nb_gws);
 329
 330                         // Last but one "next_hop_index" is not a gateway but direct routes
 331                         tbase->l3.next_hops[tbase->l3.nb_gws].ip = 0;
 332                         ret = rte_lpm_add(tbase->l3.ipv4_lpm, targ->local_ipv4, targ->local_prefix, tbase->l3.nb_gws++);
 333                         PROX_PANIC(ret, "Failed to add local_ipv4 "IPv4_BYTES_FMT"/%d to lpm\n", IP4(tbase->local_ipv4), targ->local_prefix);
 334                         // Last "next_hop_index" is default gw
 335                         tbase->l3.next_hops[tbase->l3.nb_gws].ip = rte_bswap32(targ->gateway_ipv4);
 336                         if (targ->gateway_ipv4) {
 337                                 ret = rte_lpm_add(tbase->l3.ipv4_lpm, targ->gateway_ipv4, 0, tbase->l3.nb_gws++);
 338                                 PROX_PANIC(ret, "Failed to add gateway_ipv4 "IPv4_BYTES_FMT"/%d to lpm\n", IP4(tbase->l3.gw.ip), 0);
 339                         }
 340                 }
 341
 342                 master_init_vdev(tbase->l3.tmaster, tbase->l3.reachable_port_id, targ->lconf->id, targ->id);
 343                 name[3]++;
 344                 struct rte_mempool *ret = rte_mempool_create(name, NB_ARP_MBUF, ARP_MBUF_SIZE, NB_CACHE_ARP_MBUF,
 345                         sizeof(struct rte_pktmbuf_pool_private), rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
 346                         rte_socket_id(), 0);
 347                 PROX_PANIC(ret == NULL, "Failed to allocate ARP memory pool on socket %u with %u elements\n",
 348                         rte_socket_id(), NB_ARP_MBUF);
 349                 plog_info("\t\tMempool %p (%s) size = %u * %u cache %u, socket %d\n", ret, name, NB_ARP_MBUF,
 350                         ARP_MBUF_SIZE, NB_CACHE_ARP_MBUF, rte_socket_id());
 351                 tbase->l3.arp_pool = ret;
 352         }
 353 }
 354
 355 void task_set_gateway_ip(struct task_base *tbase, uint32_t ip)
 356 {
 357         tbase->l3.gw.ip = ip;
 358         tbase->flags &= ~FLAG_DST_MAC_KNOWN;
 359 }
 360
 361 void task_set_local_ip(struct task_base *tbase, uint32_t ip)
 362 {
 363         tbase->local_ipv4 = ip;
 364 }
 365
 366 static void reset_arp_update_time(struct l3_base *l3, uint32_t ip)
 367 {
 368         uint32_t idx;
 369         plogx_dbg("MAC entry for IP "IPv4_BYTES_FMT" timeout in kernel\n", IP4(ip));
 370
 371         if (l3->ipv4_lpm) {
 372                 int ret = rte_hash_lookup(l3->ip_hash, (const void *)&ip);
 373                 if (ret >= 0)
 374                         l3->arp_table[ret].arp_update_time = 0;
 375         } else if (ip == l3->gw.ip) {
 376                 l3->gw.arp_update_time = 0;
 377         } else if (l3->n_pkts < 4) {
 378                 for (idx = 0; idx < l3->n_pkts; idx++) {
 379                         uint32_t ip_dst = l3->optimized_arp_table[idx].ip;
 380                         if (ip_dst == ip)
 381                                 break;
 382                 }
 383                 if (idx < l3->n_pkts) {
 384                         l3->optimized_arp_table[idx].arp_update_time = 0;
 385                 }
 386         } else {
 387                 int ret = rte_hash_lookup(l3->ip_hash, (const void *)&ip);
 388                 if (ret >= 0)
 389                         l3->arp_table[ret].arp_update_time = 0;
 390         }
 391         return;
 392 }
 393
 394 static prox_next_hop_index_type get_nh_index(struct task_base *tbase, uint32_t gw_ip)
 395 {
 396         // Check if gateway already exists
 397         for (prox_next_hop_index_type i = 0; i < tbase->l3.nb_gws; i++) {
 398                 if (tbase->l3.next_hops[i].ip == gw_ip) {
 399                         return i;
 400                 }
 401         }
 402         if (tbase->l3.nb_gws < MAX_HOP_INDEX) {
 403                 tbase->l3.next_hops[tbase->l3.nb_gws].ip = gw_ip;
 404                 tbase->l3.nb_gws++;
 405                 return tbase->l3.nb_gws - 1;
 406         } else
 407                 return MAX_HOP_INDEX;
 408 }
 409 void handle_ctrl_plane_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
 410 {
 411         uint8_t out[1];
 412         const uint64_t hz = rte_get_tsc_hz();
 413         uint32_t ip, ip_dst, idx, gateway_ip, prefix;
 414         prox_next_hop_index_type gateway_index;
 415         int j, ret, modified_route;
 416         uint16_t command;
 417         struct ether_hdr_arp *hdr;
 418         struct l3_base *l3 = &tbase->l3;
 419         uint64_t tsc= rte_rdtsc();
 420         uint64_t arp_timeout = l3->arp_timeout * hz / 1000;
 421         uint32_t nh;
 422
 423         for (j = 0; j < n_pkts; ++j) {
 424                 PREFETCH0(mbufs[j]);
 425         }
 426         for (j = 0; j < n_pkts; ++j) {
 427                 PREFETCH0(rte_pktmbuf_mtod(mbufs[j], void *));
 428         }
 429
 430         for (j = 0; j < n_pkts; ++j) {
 431                 out[0] = OUT_HANDLED;
 432                 command = mbufs[j]->udata64 & 0xFFFF;
 433                 plogx_dbg("\tReceived %s mbuf %p\n", actions_string[command], mbufs[j]);
 434                 switch(command) {
 435                 case ROUTE_ADD_FROM_CTRL:
 436                         ip = ctrl_ring_get_ip(mbufs[j]);
 437                         gateway_ip = ctrl_ring_get_gateway_ip(mbufs[j]);
 438                         prefix = ctrl_ring_get_prefix(mbufs[j]);
 439                         gateway_index = get_nh_index(tbase, gateway_ip);
 440                         if (gateway_index >= MAX_HOP_INDEX) {
 441                                 plog_err("Unable to find or define gateway index - too many\n");
 442                                 return;
 443                         }
 444                         modified_route = rte_lpm_is_rule_present(tbase->l3.ipv4_lpm, rte_bswap32(ip), prefix, &nh);
 445                         ret = rte_lpm_add(tbase->l3.ipv4_lpm, rte_bswap32(ip), prefix, gateway_index);
 446                         if (ret < 0) {
 447                                 plog_err("Failed to add route to "IPv4_BYTES_FMT"/%d using "IPv4_BYTES_FMT"(index = %d)\n", IP4(ip), prefix, IP4(gateway_ip), gateway_index);
 448                         } else if (modified_route)
 449                                 plogx_dbg("Modified route to "IPv4_BYTES_FMT"/%d using "IPv4_BYTES_FMT"(index = %d) (was using "IPv4_BYTES_FMT"(index = %d)\n", IP4(ip), prefix, IP4(gateway_ip), gateway_index, IP4(tbase->l3.next_hops[nh].ip), nh);
 450                         else {
 451                                 plogx_dbg("Added new route to "IPv4_BYTES_FMT"/%d using "IPv4_BYTES_FMT"(index = %d)\n", IP4(ip), prefix, IP4(gateway_ip), gateway_index);
 452                         }
 453                         break;
 454                 case ROUTE_DEL_FROM_CTRL:
 455                         ip = ctrl_ring_get_ip(mbufs[j]);
 456                         prefix = ctrl_ring_get_prefix(mbufs[j]);
 457
 458                         ret = rte_lpm_is_rule_present(tbase->l3.ipv4_lpm, rte_bswap32(ip), prefix, &nh);
 459                         if (ret > 0) {
 460                                 ret = rte_lpm_delete(tbase->l3.ipv4_lpm, rte_bswap32(ip), prefix);
 461                                 if (ret < 0) {
 462                                         plog_err("Failed to add rule\n");
 463                                 }
 464                                 plog_info("Deleting route to "IPv4_BYTES_FMT"/%d\n", IP4(ip), prefix);
 465                         }
 466                         break;
 467                 case UPDATE_FROM_CTRL:
 468                         hdr = rte_pktmbuf_mtod(mbufs[j], struct ether_hdr_arp *);
 469                         ip = (mbufs[j]->udata64 >> 32) & 0xFFFFFFFF;
 470
 471                         if (prox_rte_is_zero_ether_addr(&hdr->arp.data.sha)) {
 472                                 // MAC timeout or deleted from kernel table => reset update_time
 473                                 // This will cause us to send new ARP request
 474                                 // However, as arp_timeout not touched, we should continue sending our regular IP packets
 475                                 reset_arp_update_time(l3, ip);
 476                                 return;
 477                         } else
 478                                 plogx_dbg("\tUpdating MAC entry for IP "IPv4_BYTES_FMT" with MAC "MAC_BYTES_FMT"\n",
 479                                         IP4(ip), MAC_BYTES(hdr->arp.data.sha.addr_bytes));
 480
 481                         if (l3->ipv4_lpm) {
 482                                 uint32_t nh;
 483                                 struct arp_table *entry;
 484                                 ret = rte_hash_add_key(l3->ip_hash, (const void *)&ip);
 485                                 if (ret < 0) {
 486                                         plogx_info("Unable add ip "IPv4_BYTES_FMT" in mac_hash\n", IP4(ip));
 487                                 } else if ((nh = l3->arp_table[ret].nh) != MAX_HOP_INDEX) {
 488                                         entry = &l3->next_hops[nh];
 489                                         memcpy(&entry->mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 490                                         entry->arp_timeout = tsc + arp_timeout;
 491                                         update_arp_update_time(l3, &entry->arp_update_time, l3->arp_update_time);
 492                                 } else {
 493                                         memcpy(&l3->arp_table[ret].mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 494                                         l3->arp_table[ret].arp_timeout = tsc + arp_timeout;
 495                                         update_arp_update_time(l3, &l3->arp_table[ret].arp_update_time, l3->arp_update_time);
 496                                 }
 497                         }
 498                         else if (ip == l3->gw.ip) {
 499                                 // MAC address of the gateway
 500                                 memcpy(&l3->gw.mac, &hdr->arp.data.sha, 6);
 501                                 l3->flags |= FLAG_DST_MAC_KNOWN;
 502                                 l3->gw.arp_timeout = tsc + arp_timeout;
 503                                 update_arp_update_time(l3, &l3->gw.arp_update_time, l3->arp_update_time);
 504                         } else if (l3->n_pkts < 4) {
 505                                 // Few packets tracked - should be faster to loop through them thean using a hash table
 506                                 for (idx = 0; idx < l3->n_pkts; idx++) {
 507                                         ip_dst = l3->optimized_arp_table[idx].ip;
 508                                         if (ip_dst == ip)
 509                                                 break;
 510                                 }
 511                                 if (idx < l3->n_pkts) {
 512                                         memcpy(&l3->optimized_arp_table[idx].mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 513                                         l3->optimized_arp_table[idx].arp_timeout = tsc + arp_timeout;
 514                                         update_arp_update_time(l3, &l3->optimized_arp_table[idx].arp_update_time, l3->arp_update_time);
 515                                 }
 516                         } else {
 517                                 ret = rte_hash_add_key(l3->ip_hash, (const void *)&ip);
 518                                 if (ret < 0) {
 519                                         plogx_info("Unable add ip "IPv4_BYTES_FMT" in mac_hash\n", IP4(ip));
 520                                 } else {
 521                                         memcpy(&l3->arp_table[ret].mac, &(hdr->arp.data.sha), sizeof(prox_rte_ether_addr));
 522                                         l3->arp_table[ret].arp_timeout = tsc + arp_timeout;
 523                                         update_arp_update_time(l3, &l3->arp_table[ret].arp_update_time, l3->arp_update_time);
 524                                 }
 525                         }
 526                         tx_drop(mbufs[j]);
 527                         break;
 528                 case ARP_REPLY_FROM_CTRL:
 529                 case ARP_REQ_FROM_CTRL:
 530                         out[0] = 0;
 531                         // tx_ctrlplane_pkt does not drop packets
 532                         plogx_dbg("\tForwarding (ARP) packet from master\n");
 533                         tbase->aux->tx_ctrlplane_pkt(tbase, &mbufs[j], 1, out);
 534                         TASK_STATS_ADD_TX_NON_DP(&tbase->aux->stats, 1);
 535                         break;
 536                 case ICMP_FROM_CTRL:
 537                         out[0] = 0;
 538                         // tx_ctrlplane_pkt does not drop packets
 539                         plogx_dbg("\tForwarding (PING) packet from master\n");
 540                         tbase->aux->tx_ctrlplane_pkt(tbase, &mbufs[j], 1, out);
 541                         TASK_STATS_ADD_TX_NON_DP(&tbase->aux->stats, 1);
 542                         break;
 543                 case PKT_FROM_TAP:
 544                         out[0] = 0;
 545                         // tx_ctrlplane_pkt does not drop packets
 546                         plogx_dbg("\tForwarding TAP packet from master\n");
 547                         tbase->aux->tx_ctrlplane_pkt(tbase, &mbufs[j], 1, out);
 548                         TASK_STATS_ADD_TX_NON_DP(&tbase->aux->stats, 1);
 549                         break;
 550                 }
 551         }
 552 }