/*
-// Copyright (c) 2010-2017 Intel Corporation
+// Copyright (c) 2010-2020 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
#include "stats.h"
#include "log.h"
#include "mbuf_utils.h"
-#include "input.h" /* Needed for callback on dump */
+#include "prefetch.h"
+#include "arp.h"
+#include "tx_pkt.h"
+#include "handle_master.h"
+#include "input.h"
+#include "prox_ipv6.h" /* Needed for callback on dump */
+
+#define TCP_PORT_BGP rte_cpu_to_be_16(179)
/* _param version of the rx_pkt_hw functions are used to create two
instances of very similar variations of these functions. The
packets are received if the dequeue step involves finding 32 packets.
*/
-#define MIN_PMD_RX 32
+#define MIN_PMD_RX 32
+#define PROX_L3 1
+#define PROX_NDP 2
static uint16_t rx_pkt_hw_port_queue(struct port_queue *pq, struct rte_mbuf **mbufs, int multi)
{
rx_params_hw->last_read_portid = (rx_params_hw->last_read_portid + 1) & rx_params_hw->rxport_mask;
}
-static uint16_t rx_pkt_hw_param(struct task_base *tbase, struct rte_mbuf ***mbufs, int multi,
- void (*next)(struct rx_params_hw *rx_param_hw))
+static inline void dump_l3(struct task_base *tbase, struct rte_mbuf *mbuf)
+{
+ if (unlikely(tbase->aux->task_rt_dump.n_print_rx)) {
+ if ((tbase->aux->task_rt_dump.input == NULL) || (tbase->aux->task_rt_dump.input->reply == NULL)) {
+ plogdx_info(mbuf, "RX: ");
+ } else {
+ struct input *input = tbase->aux->task_rt_dump.input;
+ char tmp[128];
+ int strlen;
+#if RTE_VERSION >= RTE_VERSION_NUM(1,8,0,0)
+ int port_id = mbuf->port;
+#else
+ int port_id = mbuf->pkt.in_port;
+#endif
+ strlen = snprintf(tmp, sizeof(tmp), "pktdump,%d,%d\n", port_id,
+ rte_pktmbuf_pkt_len(mbuf));
+ input->reply(input, tmp, strlen);
+ input->reply(input, rte_pktmbuf_mtod(mbuf, char *), rte_pktmbuf_pkt_len(mbuf));
+ input->reply(input, "\n", 1);
+ }
+ tbase->aux->task_rt_dump.n_print_rx --;
+ if (0 == tbase->aux->task_rt_dump.n_print_rx) {
+ task_base_del_rx_pkt_function(tbase, rx_pkt_dump);
+ }
+ }
+ if (unlikely(tbase->aux->task_rt_dump.n_trace)) {
+ plogdx_info(mbuf, "RX: ");
+ tbase->aux->task_rt_dump.n_trace--;
+ }
+}
+
+static inline void handle_ipv4(struct task_base *tbase, struct rte_mbuf **mbufs, int i, prox_rte_ipv4_hdr *pip, int *skip)
+{
+ prox_rte_tcp_hdr *tcp = (prox_rte_tcp_hdr *)(pip + 1);
+ if (pip->next_proto_id == IPPROTO_ICMP) {
+ dump_l3(tbase, mbufs[i]);
+ tx_ring(tbase, tbase->l3.ctrl_plane_ring, ICMP_TO_MASTER, mbufs[i]);
+ (*skip)++;
+ } else if ((tcp->src_port == TCP_PORT_BGP) || (tcp->dst_port == TCP_PORT_BGP)) {
+ dump_l3(tbase, mbufs[i]);
+ tx_ring(tbase, tbase->l3.ctrl_plane_ring, BGP_TO_MASTER, mbufs[i]);
+ (*skip)++;
+ } else if (unlikely(*skip)) {
+ mbufs[i - *skip] = mbufs[i];
+ }
+}
+static inline int handle_l3(struct task_base *tbase, uint16_t nb_rx, struct rte_mbuf ***mbufs_ptr)
+{
+ struct rte_mbuf **mbufs = *mbufs_ptr;
+ int i;
+ struct ether_hdr_arp *hdr_arp[MAX_PKT_BURST];
+ prox_rte_ether_hdr *hdr;
+ prox_rte_ipv4_hdr *pip;
+ prox_rte_vlan_hdr *vlan;
+ int skip = 0;
+
+ for (i = 0; i < nb_rx; i++) {
+ PREFETCH0(mbufs[i]);
+ }
+
+ for (i = 0; i < nb_rx; i++) {
+ hdr_arp[i] = rte_pktmbuf_mtod(mbufs[i], struct ether_hdr_arp *);
+ PREFETCH0(hdr_arp[i]);
+ }
+ for (i = 0; i < nb_rx; i++) {
+ if (likely(hdr_arp[i]->ether_hdr.ether_type == ETYPE_IPv4)) {
+ hdr = (prox_rte_ether_hdr *)hdr_arp[i];
+ pip = (prox_rte_ipv4_hdr *)(hdr + 1);
+ handle_ipv4(tbase, mbufs, i, pip, &skip);
+ } else {
+ switch (hdr_arp[i]->ether_hdr.ether_type) {
+ case ETYPE_VLAN:
+ hdr = (prox_rte_ether_hdr *)hdr_arp[i];
+ vlan = (prox_rte_vlan_hdr *)(hdr + 1);
+ if (vlan->eth_proto == ETYPE_IPv4) {
+ pip = (prox_rte_ipv4_hdr *)(vlan + 1);
+ handle_ipv4(tbase, mbufs, i, pip, &skip);
+ } else if (vlan->eth_proto == ETYPE_ARP) {
+ dump_l3(tbase, mbufs[i]);
+ tx_ring(tbase, tbase->l3.ctrl_plane_ring, ARP_PKT_FROM_NET_TO_MASTER, mbufs[i]);
+ skip++;
+ }
+ break;
+ case ETYPE_ARP:
+ dump_l3(tbase, mbufs[i]);
+ tx_ring(tbase, tbase->l3.ctrl_plane_ring, ARP_PKT_FROM_NET_TO_MASTER, mbufs[i]);
+ skip++;
+ break;
+ default:
+ if (unlikely(skip)) {
+ mbufs[i - skip] = mbufs[i];
+ }
+ }
+ }
+ }
+ return skip;
+}
+
+static inline int handle_ndp(struct task_base *tbase, uint16_t nb_rx, struct rte_mbuf ***mbufs_ptr)
+{
+ struct rte_mbuf **mbufs = *mbufs_ptr;
+ int i;
+ prox_rte_ether_hdr *hdr[MAX_PKT_BURST];
+ int skip = 0;
+
+ for (i = 0; i < nb_rx; i++) {
+ PREFETCH0(mbufs[i]);
+ }
+ for (i = 0; i < nb_rx; i++) {
+ hdr[i] = rte_pktmbuf_mtod(mbufs[i], prox_rte_ether_hdr *);
+ PREFETCH0(hdr[i]);
+ }
+ for (i = 0; i < nb_rx; i++) {
+ prox_rte_ipv6_hdr *ipv6_hdr = (prox_rte_ipv6_hdr *)(hdr[i] + 1);
+ if (unlikely((hdr[i]->ether_type == ETYPE_IPv6) && (ipv6_hdr->proto == ICMPv6))) {
+ dump_l3(tbase, mbufs[i]);
+ tx_ring(tbase, tbase->l3.ctrl_plane_ring, NDP_PKT_FROM_NET_TO_MASTER, mbufs[i]);
+ skip++;
+ } else if (unlikely(skip)) {
+ mbufs[i - skip] = mbufs[i];
+ }
+ }
+ return skip;
+}
+
+static uint16_t rx_pkt_hw_param(struct task_base *tbase, struct rte_mbuf ***mbufs_ptr, int multi,
+ void (*next)(struct rx_params_hw *rx_param_hw), int l3_ndp)
{
uint8_t last_read_portid;
- uint16_t nb_rx;
+ uint16_t nb_rx, ret;
+ int skip = 0;
START_EMPTY_MEASSURE();
- *mbufs = tbase->ws_mbuf->mbuf[0] +
+ *mbufs_ptr = tbase->ws_mbuf->mbuf[0] +
(RTE_ALIGN_CEIL(tbase->ws_mbuf->idx[0].prod, 2) & WS_MBUF_MASK);
last_read_portid = tbase->rx_params_hw.last_read_portid;
struct port_queue *pq = &tbase->rx_params_hw.rx_pq[last_read_portid];
- nb_rx = rx_pkt_hw_port_queue(pq, *mbufs, multi);
+ nb_rx = rx_pkt_hw_port_queue(pq, *mbufs_ptr, multi);
next(&tbase->rx_params_hw);
+ if (l3_ndp == PROX_L3)
+ skip = handle_l3(tbase, nb_rx, mbufs_ptr);
+ else if (l3_ndp == PROX_NDP)
+ skip = handle_ndp(tbase, nb_rx, mbufs_ptr);
+
+ if (skip)
+ TASK_STATS_ADD_RX_NON_DP(&tbase->aux->stats, skip);
if (likely(nb_rx > 0)) {
TASK_STATS_ADD_RX(&tbase->aux->stats, nb_rx);
- return nb_rx;
+ return nb_rx - skip;
}
TASK_STATS_ADD_IDLE(&tbase->aux->stats, rte_rdtsc() - cur_tsc);
return 0;
}
-static inline uint16_t rx_pkt_hw1_param(struct task_base *tbase, struct rte_mbuf ***mbufs, int multi)
+static inline uint16_t rx_pkt_hw1_param(struct task_base *tbase, struct rte_mbuf ***mbufs_ptr, int multi, int l3_ndp)
{
uint16_t nb_rx, n;
+ int skip = 0;
START_EMPTY_MEASSURE();
- *mbufs = tbase->ws_mbuf->mbuf[0] +
+ *mbufs_ptr = tbase->ws_mbuf->mbuf[0] +
(RTE_ALIGN_CEIL(tbase->ws_mbuf->idx[0].prod, 2) & WS_MBUF_MASK);
nb_rx = rte_eth_rx_burst(tbase->rx_params_hw1.rx_pq.port,
tbase->rx_params_hw1.rx_pq.queue,
- *mbufs, MAX_PKT_BURST);
+ *mbufs_ptr, MAX_PKT_BURST);
if (multi) {
n = nb_rx;
while ((n != 0) && (MAX_PKT_BURST - nb_rx >= MIN_PMD_RX)) {
n = rte_eth_rx_burst(tbase->rx_params_hw1.rx_pq.port,
tbase->rx_params_hw1.rx_pq.queue,
- *mbufs + nb_rx, MIN_PMD_RX);
+ *mbufs_ptr + nb_rx, MIN_PMD_RX);
nb_rx += n;
PROX_PANIC(nb_rx > 64, "Received %d packets while expecting maximum %d\n", n, MIN_PMD_RX);
}
}
- if (likely(nb_rx > 0)) {
- TASK_STATS_ADD_RX(&tbase->aux->stats, nb_rx);
- return nb_rx;
+ if (unlikely(nb_rx == 0)) {
+ TASK_STATS_ADD_IDLE(&tbase->aux->stats, rte_rdtsc() - cur_tsc);
+ return 0;
}
- TASK_STATS_ADD_IDLE(&tbase->aux->stats, rte_rdtsc() - cur_tsc);
- return 0;
+
+ if (l3_ndp == PROX_L3)
+ skip = handle_l3(tbase, nb_rx, mbufs_ptr);
+ else if (l3_ndp == PROX_NDP)
+ skip = handle_ndp(tbase, nb_rx, mbufs_ptr);
+
+ if (skip)
+ TASK_STATS_ADD_RX_NON_DP(&tbase->aux->stats, skip);
+
+ TASK_STATS_ADD_RX(&tbase->aux->stats, nb_rx);
+ return nb_rx - skip;
}
uint16_t rx_pkt_hw(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw_param(tbase, mbufs, 0, next_port);
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port, 0);
}
uint16_t rx_pkt_hw_pow2(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw_param(tbase, mbufs, 0, next_port_pow2);
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port_pow2, 0);
}
uint16_t rx_pkt_hw1(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw1_param(tbase, mbufs, 0);
+ return rx_pkt_hw1_param(tbase, mbufs, 0, 0);
}
uint16_t rx_pkt_hw_multi(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw_param(tbase, mbufs, 1, next_port);
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port, 0);
}
uint16_t rx_pkt_hw_pow2_multi(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw_param(tbase, mbufs, 1, next_port_pow2);
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port_pow2, 0);
}
uint16_t rx_pkt_hw1_multi(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
- return rx_pkt_hw1_param(tbase, mbufs, 1);
+ return rx_pkt_hw1_param(tbase, mbufs, 1, 0);
+}
+
+uint16_t rx_pkt_hw_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port, PROX_L3);
+}
+
+uint16_t rx_pkt_hw_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port, PROX_NDP);
+}
+
+uint16_t rx_pkt_hw_pow2_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port_pow2, PROX_L3);
+}
+
+uint16_t rx_pkt_hw_pow2_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 0, next_port_pow2, PROX_NDP);
+}
+
+uint16_t rx_pkt_hw1_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw1_param(tbase, mbufs, 0, PROX_L3);
+}
+
+uint16_t rx_pkt_hw1_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw1_param(tbase, mbufs, 0, PROX_NDP);
+}
+
+uint16_t rx_pkt_hw_multi_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port, PROX_L3);
+}
+
+uint16_t rx_pkt_hw_multi_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port, PROX_NDP);
+}
+
+uint16_t rx_pkt_hw_pow2_multi_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port_pow2, PROX_L3);
+}
+
+uint16_t rx_pkt_hw_pow2_multi_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw_param(tbase, mbufs, 1, next_port_pow2, PROX_NDP);
+}
+
+uint16_t rx_pkt_hw1_multi_l3(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw1_param(tbase, mbufs, 1, PROX_L3);
+}
+
+uint16_t rx_pkt_hw1_multi_ndp(struct task_base *tbase, struct rte_mbuf ***mbufs)
+{
+ return rx_pkt_hw1_param(tbase, mbufs, 1, PROX_NDP);
}
/* The following functions implement ring access */
-static uint16_t ring_deq(struct rte_ring *r, struct rte_mbuf **mbufs)
+uint16_t ring_deq(struct rte_ring *r, struct rte_mbuf **mbufs)
{
void **v_mbufs = (void **)mbufs;
#ifdef BRAS_RX_BULK
{
uint16_t ret;
- if (tbase->aux->rx_prev_idx + 1 == tbase->aux->rx_prev_count) {
- ret = tbase->aux->rx_pkt_prev[tbase->aux->rx_prev_idx](tbase, mbufs);
- } else {
- tbase->aux->rx_prev_idx++;
- ret = tbase->aux->rx_pkt_prev[tbase->aux->rx_prev_idx](tbase, mbufs);
- tbase->aux->rx_prev_idx--;
- }
+ tbase->aux->rx_prev_idx++;
+ ret = tbase->aux->rx_pkt_prev[tbase->aux->rx_prev_idx - 1](tbase, mbufs);
+ tbase->aux->rx_prev_idx--;
return ret;
}
uint32_t n_dump = tbase->aux->task_rt_dump.n_print_rx;
n_dump = ret < n_dump? ret : n_dump;
- if (tbase->aux->task_rt_dump.input->reply == NULL) {
+ if ((tbase->aux->task_rt_dump.input == NULL) || (tbase->aux->task_rt_dump.input->reply == NULL)) {
for (uint32_t i = 0; i < n_dump; ++i) {
- plogd_info((*mbufs)[i], "RX: ");
+ plogdx_info((*mbufs)[i], "RX: ");
}
}
else {
uint16_t rx_pkt_trace(struct task_base *tbase, struct rte_mbuf ***mbufs)
{
+ tbase->aux->task_rt_dump.cur_trace = 0;
uint16_t ret = call_prev_rx_pkt(tbase, mbufs);
if (ret) {
uint32_t n_trace = tbase->aux->task_rt_dump.n_trace;
n_trace = ret < n_trace? ret : n_trace;
- tbase->aux->task_rt_dump.cur_trace = n_trace;
+ n_trace = n_trace <= MAX_RING_BURST ? n_trace : MAX_RING_BURST;
for (uint32_t i = 0; i < n_trace; ++i) {
uint8_t *pkt = rte_pktmbuf_mtod((*mbufs)[i], uint8_t *);
tbase->aux->task_rt_dump.pkt_cpy_len[i] = rte_pktmbuf_pkt_len((*mbufs)[i]);
tbase->aux->task_rt_dump.pkt_mbuf_addr[i] = (*mbufs)[i];
}
+ tbase->aux->task_rt_dump.cur_trace += n_trace;
tbase->aux->task_rt_dump.n_trace -= n_trace;
/* Unset by TX when n_trace = 0 */
{
uint16_t ret = call_prev_rx_pkt(tbase, mbufs);
- tbase->aux->rx_bucket[ret]++;
+ if (likely(ret < RX_BUCKET_SIZE))
+ tbase->aux->rx_bucket[ret]++;
+ else
+ tbase->aux->rx_bucket[RX_BUCKET_SIZE - 1]++;
return ret;
}
return ret;
}
-
-uint16_t rx_pkt_all(struct task_base *tbase, struct rte_mbuf ***mbufs)
-{
- uint16_t tot = 0;
- uint16_t ret = 0;
- struct rte_mbuf **new_mbufs;
- struct rte_mbuf **dst = tbase->aux->all_mbufs;
-
- /* In case we receive less than MAX_PKT_BURST packets in one
- iteration, do no perform any copying of mbuf pointers. Use
- the buffer itself instead. */
- ret = call_prev_rx_pkt(tbase, &new_mbufs);
- if (ret < MAX_PKT_BURST/2) {
- *mbufs = new_mbufs;
- return ret;
- }
-
- memcpy(dst + tot, new_mbufs, ret * sizeof(*dst));
- tot += ret;
- *mbufs = dst;
-
- do {
- ret = call_prev_rx_pkt(tbase, &new_mbufs);
- memcpy(dst + tot, new_mbufs, ret * sizeof(*dst));
- tot += ret;
- } while (ret == MAX_PKT_BURST/2 && tot < MAX_RX_PKT_ALL - MAX_PKT_BURST);
-
- if (tot >= MAX_RX_PKT_ALL - MAX_PKT_BURST) {
- plog_err("Could not receive all packets - buffer full\n");
- }
-
- return tot;
-}
Code Review / samplevnf.git / blobdiff