2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
17 #include <rte_ethdev.h>
18 #include <rte_version.h>
22 #include "task_base.h"
25 #include "prox_assert.h"
27 #include "mbuf_utils.h"
28 #include "handle_master.h"
30 static void buf_pkt_single(struct task_base *tbase, struct rte_mbuf *mbuf, const uint8_t out)
32 const uint16_t prod = tbase->ws_mbuf->idx[out].prod++;
33 tbase->ws_mbuf->mbuf[out][prod & WS_MBUF_MASK] = mbuf;
36 static inline void buf_pkt_all(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
38 for (uint16_t j = 0; j < n_pkts; ++j) {
39 if (unlikely(out[j] >= OUT_HANDLED)) {
40 rte_pktmbuf_free(mbufs[j]);
41 if (out[j] == OUT_HANDLED)
42 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, 1);
44 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
47 buf_pkt_single(tbase, mbufs[j], out[j]);
53 int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
56 int first = 0, ret, ok = 0, rc;
57 const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
58 struct rte_mbuf *arp_mbuf = NULL; // used when one need to send both an ARP and a mbuf
60 uint64_t tsc = rte_rdtsc();
62 for (int j = 0; j < n_pkts; j++) {
63 if ((out) && (out[j] >= OUT_HANDLED))
65 if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst, &time, tsc)) != SEND_MBUF)) {
67 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, j - first, out);
73 // We re-use the mbuf - no need to create a arp_mbuf and delete the existing mbuf
74 mbufs[j]->port = tbase->l3.reachable_port_id;
75 if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, mbufs[j], tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
76 update_arp_update_time(&tbase->l3, time, 1000);
78 update_arp_update_time(&tbase->l3, time, 100);
80 case SEND_MBUF_AND_ARP:
81 // We send the mbuf and an ARP - we need to allocate another mbuf for ARP
82 ret = rte_mempool_get(tbase->l3.arp_pool, (void **)&arp_mbuf);
83 if (likely(ret == 0)) {
84 arp_mbuf->port = tbase->l3.reachable_port_id;
85 if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, arp_mbuf, tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
86 update_arp_update_time(&tbase->l3, time, 1000);
88 update_arp_update_time(&tbase->l3, time, 100);
90 plog_err("Failed to get a mbuf from arp mempool\n");
91 // We still send the initial mbuf
93 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + j, 1, out);
97 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
102 if (n_pkts - first) {
103 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, n_pkts - first, out);
109 /* The following help functions also report stats. Therefore we need
110 to pass the task_base struct. */
111 static inline int txhw_drop(const struct port_queue *port_queue, struct rte_mbuf **mbufs, uint16_t n_pkts, struct task_base *tbase)
116 /* TX vector mode can't transmit more than 32 packets */
117 if (n_pkts > MAX_PMD_TX) {
118 ntx = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, MAX_PMD_TX);
119 ntx += rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs + ntx, n_pkts - ntx);
121 ntx = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, n_pkts);
123 TASK_STATS_ADD_TX(&tbase->aux->stats, ntx);
127 plog_dbg("Failed to send %d packets from %p\n", ret, mbufs[0]);
128 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts - ntx);
129 if (tbase->tx_pkt == tx_pkt_bw) {
130 uint32_t drop_bytes = 0;
132 drop_bytes += mbuf_wire_size(mbufs[ntx]);
133 rte_pktmbuf_free(mbufs[ntx++]);
134 } while (ntx < n_pkts);
135 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
139 rte_pktmbuf_free(mbufs[ntx++]);
140 } while (ntx < n_pkts);
146 static inline int txhw_no_drop(const struct port_queue *port_queue, struct rte_mbuf **mbufs, uint16_t n_pkts, struct task_base *tbase)
151 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
153 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, n_pkts);
161 static inline int ring_enq_drop(struct rte_ring *ring, struct rte_mbuf *const *mbufs, uint16_t n_pkts, __attribute__((unused)) struct task_base *tbase)
164 /* return 0 on succes, -ENOBUFS on failure */
165 // Rings can be single or multiproducer (ctrl rings are multi producer)
166 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
167 if (unlikely(rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts))) {
169 if (unlikely(rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts, NULL) == 0)) {
172 if (tbase->tx_pkt == tx_pkt_bw) {
173 uint32_t drop_bytes = 0;
174 for (uint16_t i = 0; i < n_pkts; ++i) {
175 drop_bytes += mbuf_wire_size(mbufs[i]);
176 rte_pktmbuf_free(mbufs[i]);
178 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
179 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts);
182 for (uint16_t i = 0; i < n_pkts; ++i)
183 rte_pktmbuf_free(mbufs[i]);
184 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts);
188 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
193 static inline int ring_enq_no_drop(struct rte_ring *ring, struct rte_mbuf *const *mbufs, uint16_t n_pkts, __attribute__((unused)) struct task_base *tbase)
196 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
197 while (rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts)) {
199 while (rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts, NULL) == 0) {
203 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
207 void flush_queues_hw(struct task_base *tbase)
211 for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
212 prod = tbase->ws_mbuf->idx[i].prod;
213 cons = tbase->ws_mbuf->idx[i].cons;
216 tbase->ws_mbuf->idx[i].prod = 0;
217 tbase->ws_mbuf->idx[i].cons = 0;
218 txhw_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
222 tbase->flags &= ~FLAG_TX_FLUSH;
225 void flush_queues_sw(struct task_base *tbase)
229 for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
230 prod = tbase->ws_mbuf->idx[i].prod;
231 cons = tbase->ws_mbuf->idx[i].cons;
234 tbase->ws_mbuf->idx[i].prod = 0;
235 tbase->ws_mbuf->idx[i].cons = 0;
236 ring_enq_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
239 tbase->flags &= ~FLAG_TX_FLUSH;
242 void flush_queues_no_drop_hw(struct task_base *tbase)
246 for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
247 prod = tbase->ws_mbuf->idx[i].prod;
248 cons = tbase->ws_mbuf->idx[i].cons;
251 tbase->ws_mbuf->idx[i].prod = 0;
252 tbase->ws_mbuf->idx[i].cons = 0;
253 txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
257 tbase->flags &= ~FLAG_TX_FLUSH;
260 void flush_queues_no_drop_sw(struct task_base *tbase)
264 for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
265 prod = tbase->ws_mbuf->idx[i].prod;
266 cons = tbase->ws_mbuf->idx[i].cons;
269 tbase->ws_mbuf->idx[i].prod = 0;
270 tbase->ws_mbuf->idx[i].cons = 0;
271 ring_enq_no_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
274 tbase->flags &= ~FLAG_TX_FLUSH;
277 /* "try" functions try to send packets to sw/hw w/o failing or blocking;
278 They return if ring/queue is full and are used by aggregators.
279 "try" functions do not have drop/no drop flavors
280 They are only implemented in never_discard mode (as by default they
281 use only one outgoing ring. */
282 uint16_t tx_try_self(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
285 tx_pkt_never_discard_self(tbase, mbufs, n_pkts, NULL);
288 tx_pkt_never_discard_self(tbase, mbufs, 64, NULL);
293 uint16_t tx_try_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
295 const int bulk_size = 64;
296 uint16_t ret = bulk_size, sent = 0, n_bulks;
297 n_bulks = n_pkts >> __builtin_ctz(bulk_size);
299 for (int i = 0; i < n_bulks; i++) {
300 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
301 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, bulk_size);
303 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, bulk_size, NULL);
307 if (ret != bulk_size)
310 if ((ret == bulk_size) && (n_pkts & (bulk_size - 1))) {
311 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
312 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, (n_pkts & (bulk_size - 1)));
314 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, (n_pkts & (bulk_size - 1)), NULL);
319 TASK_STATS_ADD_TX(&tbase->aux->stats, sent);
323 uint16_t tx_try_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
325 const int bulk_size = 64;
326 uint16_t ret = bulk_size, n_bulks, sent = 0;
327 n_bulks = n_pkts >> __builtin_ctz(bulk_size);
329 const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
330 for (int i = 0; i < n_bulks; i++) {
331 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, bulk_size);
334 if (ret != bulk_size)
337 if ((ret == bulk_size) && (n_pkts & (bulk_size - 1))) {
338 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, (n_pkts & (bulk_size - 1)));
342 TASK_STATS_ADD_TX(&tbase->aux->stats, sent);
346 int tx_pkt_no_drop_never_discard_hw1_lat_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
348 return txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
351 int tx_pkt_no_drop_never_discard_hw1_thrpt_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
353 static uint8_t fake_out[MAX_PKT_BURST] = {0};
355 if (n_pkts == MAX_PKT_BURST) {
356 // First xmit what was queued
359 prod = tbase->ws_mbuf->idx[0].prod;
360 cons = tbase->ws_mbuf->idx[0].cons;
362 if ((uint16_t)(prod - cons)){
363 tbase->flags &= ~FLAG_TX_FLUSH;
364 tbase->ws_mbuf->idx[0].prod = 0;
365 tbase->ws_mbuf->idx[0].cons = 0;
366 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], tbase->ws_mbuf->mbuf[0] + (cons & WS_MBUF_MASK), (uint16_t)(prod - cons), tbase);
368 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
370 ret+= tx_pkt_no_drop_hw(tbase, mbufs, n_pkts, fake_out);
375 int tx_pkt_never_discard_hw1_lat_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
377 return txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
380 int tx_pkt_never_discard_hw1_thrpt_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
382 static uint8_t fake_out[MAX_PKT_BURST] = {0};
384 if (n_pkts == MAX_PKT_BURST) {
385 // First xmit what was queued
388 prod = tbase->ws_mbuf->idx[0].prod;
389 cons = tbase->ws_mbuf->idx[0].cons;
391 if ((uint16_t)(prod - cons)){
392 tbase->flags &= ~FLAG_TX_FLUSH;
393 tbase->ws_mbuf->idx[0].prod = 0;
394 tbase->ws_mbuf->idx[0].cons = 0;
395 ret+= txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], tbase->ws_mbuf->mbuf[0] + (cons & WS_MBUF_MASK), (uint16_t)(prod - cons), tbase);
397 ret+= txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
399 ret+= tx_pkt_hw(tbase, mbufs, n_pkts, fake_out);
404 /* Transmit to hw using tx_params_hw_sw structure
405 This function is used to transmit to hw when tx_params_hw_sw should be used
406 i.e. when the task needs to transmit both to hw and sw */
407 int tx_pkt_no_drop_never_discard_hw1_no_pointer(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
409 txhw_no_drop(&tbase->tx_params_hw_sw.tx_port_queue, mbufs, n_pkts, tbase);
413 int tx_pkt_no_drop_never_discard_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
415 return ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
418 int tx_pkt_never_discard_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
420 return ring_enq_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
423 static uint16_t tx_pkt_free_dropped(__attribute__((unused)) struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
427 /* The most probable and most important optimize case is if
428 the no packets should be dropped. */
429 for (i = 0; i + 8 < n_pkts; i += 8) {
430 v |= *((uint64_t*)(&out[i]));
432 for (; i < n_pkts; ++i) {
437 /* At least some packets need to be dropped, so the
438 mbufs array needs to be updated. */
440 uint16_t n_discard = 0;
441 for (uint16_t i = 0; i < n_pkts; ++i) {
442 if (unlikely(out[i] >= OUT_HANDLED)) {
443 rte_pktmbuf_free(mbufs[i]);
444 n_discard += out[i] == OUT_DISCARD;
447 mbufs[n_kept++] = mbufs[i];
449 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, n_discard);
450 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, n_pkts - n_kept - n_discard);
456 int tx_pkt_no_drop_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
458 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
462 ret = txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_kept, tbase);
466 int tx_pkt_no_drop_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
468 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
472 ret = ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_kept, tbase);
476 int tx_pkt_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
478 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
481 return txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_kept, tbase);
485 int tx_pkt_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
487 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
490 return ring_enq_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_kept, tbase);
494 int tx_pkt_self(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
496 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
498 TASK_STATS_ADD_TX(&tbase->aux->stats, n_kept);
499 tbase->ws_mbuf->idx[0].nb_rx = n_kept;
500 struct rte_mbuf **tx_mbuf = tbase->ws_mbuf->mbuf[0] + (tbase->ws_mbuf->idx[0].prod & WS_MBUF_MASK);
501 for (uint16_t i = 0; i < n_kept; ++i) {
502 tx_mbuf[i] = mbufs[i];
507 int tx_pkt_never_discard_self(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
509 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
510 tbase->ws_mbuf->idx[0].nb_rx = n_pkts;
511 struct rte_mbuf **tx_mbuf = tbase->ws_mbuf->mbuf[0] + (tbase->ws_mbuf->idx[0].prod & WS_MBUF_MASK);
512 for (uint16_t i = 0; i < n_pkts; ++i) {
513 tx_mbuf[i] = mbufs[i];
518 int tx_pkt_no_drop_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
521 buf_pkt_all(tbase, mbufs, n_pkts, out);
523 const uint8_t nb_bufs = tbase->tx_params_hw.nb_txports;
526 for (uint8_t i = 0; i < nb_bufs; ++i) {
527 prod = tbase->ws_mbuf->idx[i].prod;
528 cons = tbase->ws_mbuf->idx[i].cons;
530 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
531 tbase->flags &= ~FLAG_TX_FLUSH;
532 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
533 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
539 int tx_pkt_no_drop_sw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
542 buf_pkt_all(tbase, mbufs, n_pkts, out);
544 const uint8_t nb_bufs = tbase->tx_params_sw.nb_txrings;
547 for (uint8_t i = 0; i < nb_bufs; ++i) {
548 prod = tbase->ws_mbuf->idx[i].prod;
549 cons = tbase->ws_mbuf->idx[i].cons;
551 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
552 tbase->flags &= ~FLAG_TX_FLUSH;
553 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
554 ret += ring_enq_no_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
560 int tx_pkt_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
563 buf_pkt_all(tbase, mbufs, n_pkts, out);
565 const uint8_t nb_bufs = tbase->tx_params_hw.nb_txports;
568 for (uint8_t i = 0; i < nb_bufs; ++i) {
569 prod = tbase->ws_mbuf->idx[i].prod;
570 cons = tbase->ws_mbuf->idx[i].cons;
572 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
573 tbase->flags &= ~FLAG_TX_FLUSH;
574 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
575 ret += txhw_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
581 int tx_pkt_sw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
584 buf_pkt_all(tbase, mbufs, n_pkts, out);
586 const uint8_t nb_bufs = tbase->tx_params_sw.nb_txrings;
588 for (uint8_t i = 0; i < nb_bufs; ++i) {
589 prod = tbase->ws_mbuf->idx[i].prod;
590 cons = tbase->ws_mbuf->idx[i].cons;
592 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
593 tbase->flags &= ~FLAG_TX_FLUSH;
594 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
595 ret+= ring_enq_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
601 static inline void trace_one_rx_pkt(struct task_base *tbase, struct rte_mbuf *mbuf)
604 /* For each packet being transmitted, find which
605 buffer represent the packet as it was before
608 uint32_t len = sizeof(tbase->aux->task_rt_dump.pkt_mbuf_addr)/sizeof(tbase->aux->task_rt_dump.pkt_mbuf_addr[0]);
609 for (;j < len; ++j) {
610 if (tbase->aux->task_rt_dump.pkt_mbuf_addr[j] == mbuf)
614 #if RTE_VERSION >= RTE_VERSION_NUM(1,8,0,0)
617 rte_pktmbuf_data_len(&tmp) = tbase->aux->task_rt_dump.pkt_cpy_len[j];
618 rte_pktmbuf_pkt_len(&tmp) = tbase->aux->task_rt_dump.pkt_cpy_len[j];
619 tmp.buf_addr = tbase->aux->task_rt_dump.pkt_cpy[j];
620 plogdx_info(&tmp, "Trace RX: ");
624 static inline void trace_one_tx_pkt(struct task_base *tbase, struct rte_mbuf *mbuf, uint8_t *out, uint32_t i)
629 plogdx_info(mbuf, "Handled: ");
632 plogdx_info(mbuf, "Dropped: ");
635 plogdx_info(mbuf, "TX[%d]: ", out[i]);
638 } else if (tbase->aux->tx_pkt_orig == tx_pkt_drop_all) {
639 plogdx_info(mbuf, "Dropped: ");
641 plogdx_info(mbuf, "TX[0]: ");
644 static void unset_trace(struct task_base *tbase)
646 if (0 == tbase->aux->task_rt_dump.n_trace) {
647 if (tbase->tx_pkt == tx_pkt_l3) {
648 tbase->aux->tx_pkt_l2 = tbase->aux->tx_pkt_orig;
649 tbase->aux->tx_pkt_orig = NULL;
651 tbase->tx_pkt = tbase->aux->tx_pkt_orig;
652 tbase->aux->tx_pkt_orig = NULL;
654 tbase->aux->task_rt_dump.cur_trace = 0;
655 task_base_del_rx_pkt_function(tbase, rx_pkt_trace);
659 int tx_pkt_trace(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
662 if (tbase->aux->task_rt_dump.cur_trace == 0) {
663 // No packet received since dumping...
664 tbase->aux->task_rt_dump.n_print_tx = tbase->aux->task_rt_dump.n_trace;
665 if (tbase->aux->task_rt_dump.n_trace < n_pkts) {
666 tbase->aux->task_rt_dump.n_trace = 0;
667 tbase->aux->task_rt_dump.cur_trace = 0;
668 task_base_del_rx_pkt_function(tbase, rx_pkt_trace);
670 tbase->aux->task_rt_dump.n_trace -= n_pkts;
672 ret = tx_pkt_dump(tbase, mbufs, n_pkts, out);
673 tbase->aux->task_rt_dump.n_print_tx = 0;
676 plog_info("Tracing %d pkts\n", tbase->aux->task_rt_dump.cur_trace);
677 uint32_t cur_trace = (n_pkts < tbase->aux->task_rt_dump.cur_trace) ? n_pkts: tbase->aux->task_rt_dump.cur_trace;
678 for (uint32_t i = 0; i < cur_trace; ++i) {
679 trace_one_rx_pkt(tbase, mbufs[i]);
680 trace_one_tx_pkt(tbase, mbufs[i], out, i);
683 ret = tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
689 int tx_pkt_dump(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
691 uint32_t n_dump = tbase->aux->task_rt_dump.n_print_tx;
694 n_dump = n_pkts < n_dump? n_pkts : n_dump;
695 for (uint32_t i = 0; i < n_dump; ++i) {
699 plogdx_info(mbufs[i], "Handled: ");
702 plogdx_info(mbufs[i], "Dropped: ");
705 plogdx_info(mbufs[i], "TX[%d]: ", out[i]);
709 plogdx_info(mbufs[i], "TX: ");
711 tbase->aux->task_rt_dump.n_print_tx -= n_dump;
713 ret = tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
715 if (0 == tbase->aux->task_rt_dump.n_print_tx) {
716 if (tbase->tx_pkt == tx_pkt_l3) {
717 tbase->aux->tx_pkt_l2 = tbase->aux->tx_pkt_orig;
718 tbase->aux->tx_pkt_orig = NULL;
720 tbase->tx_pkt = tbase->aux->tx_pkt_orig;
721 tbase->aux->tx_pkt_orig = NULL;
727 /* Gather the distribution of the number of packets that have been
728 xmitted from one TX call. Since the value is only modified by the
729 task that xmits the packet, no atomic operation is needed. */
730 int tx_pkt_distr(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
732 if (likely(n_pkts < TX_BUCKET_SIZE))
733 tbase->aux->tx_bucket[n_pkts]++;
735 tbase->aux->tx_bucket[TX_BUCKET_SIZE - 1]++;
736 return tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
739 int tx_pkt_bw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
741 uint32_t tx_bytes = 0;
742 uint32_t drop_bytes = 0;
744 for (uint16_t i = 0; i < n_pkts; ++i) {
745 if (!out || out[i] < OUT_HANDLED)
746 tx_bytes += mbuf_wire_size(mbufs[i]);
748 drop_bytes += mbuf_wire_size(mbufs[i]);
751 TASK_STATS_ADD_TX_BYTES(&tbase->aux->stats, tx_bytes);
752 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
753 return tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
756 int tx_pkt_drop_all(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
758 for (uint16_t j = 0; j < n_pkts; ++j) {
759 rte_pktmbuf_free(mbufs[j]);
762 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, n_pkts);
764 for (uint16_t j = 0; j < n_pkts; ++j) {
765 if (out[j] == OUT_HANDLED)
766 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, 1);
768 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
773 static inline void dump_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
775 uint32_t n_dump = tbase->aux->task_rt_dump.n_print_tx;
776 uint32_t n_trace = tbase->aux->task_rt_dump.n_trace;
778 if (unlikely(n_dump)) {
779 n_dump = n_pkts < n_dump? n_pkts : n_dump;
780 for (uint32_t i = 0; i < n_dump; ++i) {
781 plogdx_info(mbufs[i], "TX: ");
783 tbase->aux->task_rt_dump.n_print_tx -= n_dump;
784 } else if (unlikely(n_trace)) {
785 n_trace = n_pkts < n_trace? n_pkts : n_trace;
786 for (uint32_t i = 0; i < n_trace; ++i) {
787 plogdx_info(mbufs[i], "TX: ");
789 tbase->aux->task_rt_dump.n_trace -= n_trace;
793 // ctrlplane packets are slow path, hence cost of checking if dump ortrace is needed in not too important
794 // easier to have this implementation than an implementation similar to dataplane tx
795 int tx_ctrlplane_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
797 dump_pkts(tbase, mbufs, n_pkts);
798 return txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
801 int tx_ctrlplane_sw(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
803 dump_pkts(tbase, mbufs, n_pkts);
804 return ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
807 static inline int tx_ring_all(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
809 if (tbase->aux->task_rt_dump.cur_trace) {
810 trace_one_rx_pkt(tbase, mbuf);
812 mbuf->udata64 = ((uint64_t)ip << 32) | (core_id << 16) | (task_id << 8) | command;
813 return rte_ring_enqueue(ring, mbuf);
816 int tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
818 plogx_dbg("\tSending command %s with ip %d.%d.%d.%d to ring %p using mbuf %p, core %d and task %d - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, core_id, task_id, rte_ring_free_count(ring));
819 int ret = tx_ring_all(tbase, ring, command, mbuf, core_id, task_id, ip);
820 if (unlikely(ret != 0)) {
821 plogx_dbg("\tFail to send command %s with ip %d.%d.%d.%d to ring %p using mbuf %p, core %d and task %d - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, core_id, task_id, rte_ring_free_count(ring));
822 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
823 rte_pktmbuf_free(mbuf);
828 void tx_ring_ip(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint32_t ip)
830 plogx_dbg("\tSending command %s with ip %d.%d.%d.%d to ring %p using mbuf %p - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, rte_ring_free_count(ring));
831 int ret = tx_ring_all(tbase, ring, command, mbuf, 0, 0, ip);
832 if (unlikely(ret != 0)) {
833 plogx_dbg("\tFail to send command %s with ip %d.%d.%d.%d to ring %p using mbuf %p - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, rte_ring_free_count(ring));
834 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
835 rte_pktmbuf_free(mbuf);
839 void tx_ring(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf)
841 plogx_dbg("\tSending command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
842 int ret = tx_ring_all(tbase, ring, command, mbuf, 0, 0, 0);
843 if (unlikely(ret != 0)) {
844 plogx_dbg("\tFail to send command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
845 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
846 rte_pktmbuf_free(mbuf);
850 void tx_ring_route(struct task_base *tbase, struct rte_ring *ring, int add, struct rte_mbuf *mbuf, uint32_t ip, uint32_t gateway_ip, uint32_t prefix)
854 command = ROUTE_ADD_FROM_CTRL;
856 command = ROUTE_DEL_FROM_CTRL;
858 plogx_dbg("\tSending command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
859 ctrl_ring_set_command(mbuf, command);
860 ctrl_ring_set_ip(mbuf, ip);
861 ctrl_ring_set_gateway_ip(mbuf, gateway_ip);
862 ctrl_ring_set_prefix(mbuf, prefix);
863 if (tbase->aux->task_rt_dump.cur_trace) {
864 trace_one_rx_pkt(tbase, mbuf);
866 int ret = rte_ring_enqueue(ring, mbuf);
867 if (unlikely(ret != 0)) {
868 plogx_dbg("\tFail to send command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
869 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
870 rte_pktmbuf_free(mbuf);
874 void ctrl_ring_set_command(struct rte_mbuf *mbuf, uint64_t udata64)
876 mbuf->udata64 = udata64;
879 uint64_t ctrl_ring_get_command(struct rte_mbuf *mbuf)
881 return mbuf->udata64;
884 void ctrl_ring_set_ip(struct rte_mbuf *mbuf, uint32_t udata32)
886 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
887 prox_headroom->ip = udata32;
890 uint32_t ctrl_ring_get_ip(struct rte_mbuf *mbuf)
892 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
893 return prox_headroom->ip;
896 void ctrl_ring_set_gateway_ip(struct rte_mbuf *mbuf, uint32_t udata32)
898 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
899 prox_headroom->gateway_ip = udata32;
902 uint32_t ctrl_ring_get_gateway_ip(struct rte_mbuf *mbuf)
904 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
905 return prox_headroom->gateway_ip;
908 void ctrl_ring_set_prefix(struct rte_mbuf *mbuf, uint32_t udata32)
910 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
911 prox_headroom->prefix = udata32;
914 uint32_t ctrl_ring_get_prefix(struct rte_mbuf *mbuf)
916 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
917 return prox_headroom->prefix;