2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
17 #include <rte_ethdev.h>
18 #include <rte_version.h>
22 #include "task_base.h"
25 #include "prox_assert.h"
27 #include "mbuf_utils.h"
28 #include "handle_master.h"
30 static void buf_pkt_single(struct task_base *tbase, struct rte_mbuf *mbuf, const uint8_t out)
32 const uint16_t prod = tbase->ws_mbuf->idx[out].prod++;
33 tbase->ws_mbuf->mbuf[out][prod & WS_MBUF_MASK] = mbuf;
36 static inline void buf_pkt_all(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
38 for (uint16_t j = 0; j < n_pkts; ++j) {
39 if (unlikely(out[j] >= OUT_HANDLED)) {
40 rte_pktmbuf_free(mbufs[j]);
41 if (out[j] == OUT_HANDLED)
42 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, 1);
44 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
47 buf_pkt_single(tbase, mbufs[j], out[j]);
53 int tx_pkt_l3(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
56 int first = 0, ret, ok = 0, rc;
57 const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
58 struct rte_mbuf *arp_mbuf = NULL; // used when one need to send both an ARP and a mbuf
61 for (int j = 0; j < n_pkts; j++) {
62 if ((out) && (out[j] >= OUT_HANDLED))
64 if (unlikely((rc = write_dst_mac(tbase, mbufs[j], &ip_dst, &time)) != SEND_MBUF)) {
66 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, j - first, out);
72 // We re-use the mbuf - no need to create a arp_mbuf and delete the existing mbuf
73 mbufs[j]->port = tbase->l3.reachable_port_id;
74 if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, mbufs[j], tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
75 update_arp_update_time(&tbase->l3, time, 1000);
77 update_arp_update_time(&tbase->l3, time, 100);
79 case SEND_MBUF_AND_ARP:
80 // We send the mbuf and an ARP - we need to allocate another mbuf for ARP
81 ret = rte_mempool_get(tbase->l3.arp_pool, (void **)&arp_mbuf);
82 if (likely(ret == 0)) {
83 arp_mbuf->port = tbase->l3.reachable_port_id;
84 if (tx_ring_cti(tbase, tbase->l3.ctrl_plane_ring, REQ_MAC_TO_CTRL, arp_mbuf, tbase->l3.core_id, tbase->l3.task_id, ip_dst) == 0)
85 update_arp_update_time(&tbase->l3, time, 1000);
87 update_arp_update_time(&tbase->l3, time, 100);
89 plog_err("Failed to get a mbuf from arp mempool\n");
90 // We still send the initial mbuf
92 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + j, 1, out);
96 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
101 if (n_pkts - first) {
102 ret = tbase->aux->tx_pkt_l2(tbase, mbufs + first, n_pkts - first, out);
108 /* The following help functions also report stats. Therefore we need
109 to pass the task_base struct. */
110 static inline int txhw_drop(const struct port_queue *port_queue, struct rte_mbuf **mbufs, uint16_t n_pkts, struct task_base *tbase)
115 /* TX vector mode can't transmit more than 32 packets */
116 if (n_pkts > MAX_PMD_TX) {
117 ntx = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, MAX_PMD_TX);
118 ntx += rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs + ntx, n_pkts - ntx);
120 ntx = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, n_pkts);
122 TASK_STATS_ADD_TX(&tbase->aux->stats, ntx);
126 plog_dbg("Failed to send %d packets from %p\n", ret, mbufs[0]);
127 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts - ntx);
128 if (tbase->tx_pkt == tx_pkt_bw) {
129 uint32_t drop_bytes = 0;
131 drop_bytes += mbuf_wire_size(mbufs[ntx]);
132 rte_pktmbuf_free(mbufs[ntx++]);
133 } while (ntx < n_pkts);
134 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
138 rte_pktmbuf_free(mbufs[ntx++]);
139 } while (ntx < n_pkts);
145 static inline int txhw_no_drop(const struct port_queue *port_queue, struct rte_mbuf **mbufs, uint16_t n_pkts, struct task_base *tbase)
150 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
152 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, n_pkts);
160 static inline int ring_enq_drop(struct rte_ring *ring, struct rte_mbuf *const *mbufs, uint16_t n_pkts, __attribute__((unused)) struct task_base *tbase)
163 /* return 0 on succes, -ENOBUFS on failure */
164 // Rings can be single or multiproducer (ctrl rings are multi producer)
165 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
166 if (unlikely(rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts))) {
168 if (unlikely(rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts, NULL) == 0)) {
171 if (tbase->tx_pkt == tx_pkt_bw) {
172 uint32_t drop_bytes = 0;
173 for (uint16_t i = 0; i < n_pkts; ++i) {
174 drop_bytes += mbuf_wire_size(mbufs[i]);
175 rte_pktmbuf_free(mbufs[i]);
177 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
178 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts);
181 for (uint16_t i = 0; i < n_pkts; ++i)
182 rte_pktmbuf_free(mbufs[i]);
183 TASK_STATS_ADD_DROP_TX_FAIL(&tbase->aux->stats, n_pkts);
187 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
192 static inline int ring_enq_no_drop(struct rte_ring *ring, struct rte_mbuf *const *mbufs, uint16_t n_pkts, __attribute__((unused)) struct task_base *tbase)
195 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
196 while (rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts)) {
198 while (rte_ring_enqueue_bulk(ring, (void *const *)mbufs, n_pkts, NULL) == 0) {
202 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
206 void flush_queues_hw(struct task_base *tbase)
210 for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
211 prod = tbase->ws_mbuf->idx[i].prod;
212 cons = tbase->ws_mbuf->idx[i].cons;
215 tbase->ws_mbuf->idx[i].prod = 0;
216 tbase->ws_mbuf->idx[i].cons = 0;
217 txhw_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
221 tbase->flags &= ~FLAG_TX_FLUSH;
224 void flush_queues_sw(struct task_base *tbase)
228 for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
229 prod = tbase->ws_mbuf->idx[i].prod;
230 cons = tbase->ws_mbuf->idx[i].cons;
233 tbase->ws_mbuf->idx[i].prod = 0;
234 tbase->ws_mbuf->idx[i].cons = 0;
235 ring_enq_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
238 tbase->flags &= ~FLAG_TX_FLUSH;
241 void flush_queues_no_drop_hw(struct task_base *tbase)
245 for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
246 prod = tbase->ws_mbuf->idx[i].prod;
247 cons = tbase->ws_mbuf->idx[i].cons;
250 tbase->ws_mbuf->idx[i].prod = 0;
251 tbase->ws_mbuf->idx[i].cons = 0;
252 txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
256 tbase->flags &= ~FLAG_TX_FLUSH;
259 void flush_queues_no_drop_sw(struct task_base *tbase)
263 for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
264 prod = tbase->ws_mbuf->idx[i].prod;
265 cons = tbase->ws_mbuf->idx[i].cons;
268 tbase->ws_mbuf->idx[i].prod = 0;
269 tbase->ws_mbuf->idx[i].cons = 0;
270 ring_enq_no_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), prod - cons, tbase);
273 tbase->flags &= ~FLAG_TX_FLUSH;
276 /* "try" functions try to send packets to sw/hw w/o failing or blocking;
277 They return if ring/queue is full and are used by aggregators.
278 "try" functions do not have drop/no drop flavors
279 They are only implemented in never_discard mode (as by default they
280 use only one outgoing ring. */
281 uint16_t tx_try_self(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
284 tx_pkt_never_discard_self(tbase, mbufs, n_pkts, NULL);
287 tx_pkt_never_discard_self(tbase, mbufs, 64, NULL);
292 uint16_t tx_try_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
294 const int bulk_size = 64;
295 uint16_t ret = bulk_size, sent = 0, n_bulks;
296 n_bulks = n_pkts >> __builtin_ctz(bulk_size);
298 for (int i = 0; i < n_bulks; i++) {
299 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
300 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, bulk_size);
302 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, bulk_size, NULL);
306 if (ret != bulk_size)
309 if ((ret == bulk_size) && (n_pkts & (bulk_size - 1))) {
310 #if RTE_VERSION < RTE_VERSION_NUM(17,5,0,1)
311 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, (n_pkts & (bulk_size - 1)));
313 ret = rte_ring_enqueue_burst(tbase->tx_params_sw.tx_rings[0], (void *const *)mbufs, (n_pkts & (bulk_size - 1)), NULL);
318 TASK_STATS_ADD_TX(&tbase->aux->stats, sent);
322 uint16_t tx_try_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
324 const int bulk_size = 64;
325 uint16_t ret = bulk_size, n_bulks, sent = 0;
326 n_bulks = n_pkts >> __builtin_ctz(bulk_size);
328 const struct port_queue *port_queue = &tbase->tx_params_hw.tx_port_queue[0];
329 for (int i = 0; i < n_bulks; i++) {
330 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, bulk_size);
333 if (ret != bulk_size)
336 if ((ret == bulk_size) && (n_pkts & (bulk_size - 1))) {
337 ret = rte_eth_tx_burst(port_queue->port, port_queue->queue, mbufs, (n_pkts & (bulk_size - 1)));
341 TASK_STATS_ADD_TX(&tbase->aux->stats, sent);
345 int tx_pkt_no_drop_never_discard_hw1_lat_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
347 return txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
350 int tx_pkt_no_drop_never_discard_hw1_thrpt_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
352 static uint8_t fake_out[MAX_PKT_BURST] = {0};
354 if (n_pkts == MAX_PKT_BURST) {
355 // First xmit what was queued
358 prod = tbase->ws_mbuf->idx[0].prod;
359 cons = tbase->ws_mbuf->idx[0].cons;
361 if ((uint16_t)(prod - cons)){
362 tbase->flags &= ~FLAG_TX_FLUSH;
363 tbase->ws_mbuf->idx[0].prod = 0;
364 tbase->ws_mbuf->idx[0].cons = 0;
365 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], tbase->ws_mbuf->mbuf[0] + (cons & WS_MBUF_MASK), (uint16_t)(prod - cons), tbase);
367 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
369 ret+= tx_pkt_no_drop_hw(tbase, mbufs, n_pkts, fake_out);
374 int tx_pkt_never_discard_hw1_lat_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
376 return txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
379 int tx_pkt_never_discard_hw1_thrpt_opt(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
381 static uint8_t fake_out[MAX_PKT_BURST] = {0};
383 if (n_pkts == MAX_PKT_BURST) {
384 // First xmit what was queued
387 prod = tbase->ws_mbuf->idx[0].prod;
388 cons = tbase->ws_mbuf->idx[0].cons;
390 if ((uint16_t)(prod - cons)){
391 tbase->flags &= ~FLAG_TX_FLUSH;
392 tbase->ws_mbuf->idx[0].prod = 0;
393 tbase->ws_mbuf->idx[0].cons = 0;
394 ret+= txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], tbase->ws_mbuf->mbuf[0] + (cons & WS_MBUF_MASK), (uint16_t)(prod - cons), tbase);
396 ret+= txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
398 ret+= tx_pkt_hw(tbase, mbufs, n_pkts, fake_out);
403 /* Transmit to hw using tx_params_hw_sw structure
404 This function is used to transmit to hw when tx_params_hw_sw should be used
405 i.e. when the task needs to transmit both to hw and sw */
406 int tx_pkt_no_drop_never_discard_hw1_no_pointer(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
408 txhw_no_drop(&tbase->tx_params_hw_sw.tx_port_queue, mbufs, n_pkts, tbase);
412 int tx_pkt_no_drop_never_discard_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
414 return ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
417 int tx_pkt_never_discard_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
419 return ring_enq_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
422 static uint16_t tx_pkt_free_dropped(__attribute__((unused)) struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
426 /* The most probable and most important optimize case is if
427 the no packets should be dropped. */
428 for (i = 0; i + 8 < n_pkts; i += 8) {
429 v |= *((uint64_t*)(&out[i]));
431 for (; i < n_pkts; ++i) {
436 /* At least some packets need to be dropped, so the
437 mbufs array needs to be updated. */
439 uint16_t n_discard = 0;
440 for (uint16_t i = 0; i < n_pkts; ++i) {
441 if (unlikely(out[i] >= OUT_HANDLED)) {
442 rte_pktmbuf_free(mbufs[i]);
443 n_discard += out[i] == OUT_DISCARD;
446 mbufs[n_kept++] = mbufs[i];
448 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, n_discard);
449 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, n_pkts - n_kept - n_discard);
455 int tx_pkt_no_drop_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
457 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
461 ret = txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_kept, tbase);
465 int tx_pkt_no_drop_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
467 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
471 ret = ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_kept, tbase);
475 int tx_pkt_hw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
477 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
480 return txhw_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_kept, tbase);
484 int tx_pkt_sw1(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
486 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
489 return ring_enq_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_kept, tbase);
493 int tx_pkt_self(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, uint8_t *out)
495 const uint16_t n_kept = tx_pkt_free_dropped(tbase, mbufs, n_pkts, out);
497 TASK_STATS_ADD_TX(&tbase->aux->stats, n_kept);
498 tbase->ws_mbuf->idx[0].nb_rx = n_kept;
499 struct rte_mbuf **tx_mbuf = tbase->ws_mbuf->mbuf[0] + (tbase->ws_mbuf->idx[0].prod & WS_MBUF_MASK);
500 for (uint16_t i = 0; i < n_kept; ++i) {
501 tx_mbuf[i] = mbufs[i];
506 int tx_pkt_never_discard_self(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
508 TASK_STATS_ADD_TX(&tbase->aux->stats, n_pkts);
509 tbase->ws_mbuf->idx[0].nb_rx = n_pkts;
510 struct rte_mbuf **tx_mbuf = tbase->ws_mbuf->mbuf[0] + (tbase->ws_mbuf->idx[0].prod & WS_MBUF_MASK);
511 for (uint16_t i = 0; i < n_pkts; ++i) {
512 tx_mbuf[i] = mbufs[i];
517 int tx_pkt_no_drop_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
520 buf_pkt_all(tbase, mbufs, n_pkts, out);
522 const uint8_t nb_bufs = tbase->tx_params_hw.nb_txports;
525 for (uint8_t i = 0; i < nb_bufs; ++i) {
526 prod = tbase->ws_mbuf->idx[i].prod;
527 cons = tbase->ws_mbuf->idx[i].cons;
529 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
530 tbase->flags &= ~FLAG_TX_FLUSH;
531 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
532 ret+= txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
538 int tx_pkt_no_drop_sw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
541 buf_pkt_all(tbase, mbufs, n_pkts, out);
543 const uint8_t nb_bufs = tbase->tx_params_sw.nb_txrings;
546 for (uint8_t i = 0; i < nb_bufs; ++i) {
547 prod = tbase->ws_mbuf->idx[i].prod;
548 cons = tbase->ws_mbuf->idx[i].cons;
550 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
551 tbase->flags &= ~FLAG_TX_FLUSH;
552 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
553 ret += ring_enq_no_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
559 int tx_pkt_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
562 buf_pkt_all(tbase, mbufs, n_pkts, out);
564 const uint8_t nb_bufs = tbase->tx_params_hw.nb_txports;
567 for (uint8_t i = 0; i < nb_bufs; ++i) {
568 prod = tbase->ws_mbuf->idx[i].prod;
569 cons = tbase->ws_mbuf->idx[i].cons;
571 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
572 tbase->flags &= ~FLAG_TX_FLUSH;
573 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
574 ret += txhw_drop(&tbase->tx_params_hw.tx_port_queue[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
580 int tx_pkt_sw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
583 buf_pkt_all(tbase, mbufs, n_pkts, out);
585 const uint8_t nb_bufs = tbase->tx_params_sw.nb_txrings;
587 for (uint8_t i = 0; i < nb_bufs; ++i) {
588 prod = tbase->ws_mbuf->idx[i].prod;
589 cons = tbase->ws_mbuf->idx[i].cons;
591 if (((uint16_t)(prod - cons)) >= MAX_PKT_BURST) {
592 tbase->flags &= ~FLAG_TX_FLUSH;
593 tbase->ws_mbuf->idx[i].cons = cons + MAX_PKT_BURST;
594 ret+= ring_enq_drop(tbase->tx_params_sw.tx_rings[i], tbase->ws_mbuf->mbuf[i] + (cons & WS_MBUF_MASK), MAX_PKT_BURST, tbase);
600 static inline void trace_one_rx_pkt(struct task_base *tbase, struct rte_mbuf *mbuf)
603 /* For each packet being transmitted, find which
604 buffer represent the packet as it was before
607 uint32_t len = sizeof(tbase->aux->task_rt_dump.pkt_mbuf_addr)/sizeof(tbase->aux->task_rt_dump.pkt_mbuf_addr[0]);
608 for (;j < len; ++j) {
609 if (tbase->aux->task_rt_dump.pkt_mbuf_addr[j] == mbuf)
613 #if RTE_VERSION >= RTE_VERSION_NUM(1,8,0,0)
616 rte_pktmbuf_data_len(&tmp) = tbase->aux->task_rt_dump.pkt_cpy_len[j];
617 rte_pktmbuf_pkt_len(&tmp) = tbase->aux->task_rt_dump.pkt_cpy_len[j];
618 tmp.buf_addr = tbase->aux->task_rt_dump.pkt_cpy[j];
619 plogdx_info(&tmp, "Trace RX: ");
623 static inline void trace_one_tx_pkt(struct task_base *tbase, struct rte_mbuf *mbuf, uint8_t *out, uint32_t i)
628 plogdx_info(mbuf, "Handled: ");
631 plogdx_info(mbuf, "Dropped: ");
634 plogdx_info(mbuf, "TX[%d]: ", out[i]);
637 } else if (tbase->aux->tx_pkt_orig == tx_pkt_drop_all) {
638 plogdx_info(mbuf, "Dropped: ");
640 plogdx_info(mbuf, "TX[0]: ");
643 static void unset_trace(struct task_base *tbase)
645 if (0 == tbase->aux->task_rt_dump.n_trace) {
646 if (tbase->tx_pkt == tx_pkt_l3) {
647 tbase->aux->tx_pkt_l2 = tbase->aux->tx_pkt_orig;
648 tbase->aux->tx_pkt_orig = NULL;
650 tbase->tx_pkt = tbase->aux->tx_pkt_orig;
651 tbase->aux->tx_pkt_orig = NULL;
653 tbase->aux->task_rt_dump.cur_trace = 0;
654 task_base_del_rx_pkt_function(tbase, rx_pkt_trace);
658 int tx_pkt_trace(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
661 if (tbase->aux->task_rt_dump.cur_trace == 0) {
662 // No packet received since dumping...
663 tbase->aux->task_rt_dump.n_print_tx = tbase->aux->task_rt_dump.n_trace;
664 if (tbase->aux->task_rt_dump.n_trace < n_pkts) {
665 tbase->aux->task_rt_dump.n_trace = 0;
666 tbase->aux->task_rt_dump.cur_trace = 0;
667 task_base_del_rx_pkt_function(tbase, rx_pkt_trace);
669 tbase->aux->task_rt_dump.n_trace -= n_pkts;
671 ret = tx_pkt_dump(tbase, mbufs, n_pkts, out);
672 tbase->aux->task_rt_dump.n_print_tx = 0;
675 plog_info("Tracing %d pkts\n", tbase->aux->task_rt_dump.cur_trace);
676 uint32_t cur_trace = (n_pkts < tbase->aux->task_rt_dump.cur_trace) ? n_pkts: tbase->aux->task_rt_dump.cur_trace;
677 for (uint32_t i = 0; i < cur_trace; ++i) {
678 trace_one_rx_pkt(tbase, mbufs[i]);
679 trace_one_tx_pkt(tbase, mbufs[i], out, i);
682 ret = tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
688 int tx_pkt_dump(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
690 uint32_t n_dump = tbase->aux->task_rt_dump.n_print_tx;
693 n_dump = n_pkts < n_dump? n_pkts : n_dump;
694 for (uint32_t i = 0; i < n_dump; ++i) {
698 plogdx_info(mbufs[i], "Handled: ");
701 plogdx_info(mbufs[i], "Dropped: ");
704 plogdx_info(mbufs[i], "TX[%d]: ", out[i]);
708 plogdx_info(mbufs[i], "TX: ");
710 tbase->aux->task_rt_dump.n_print_tx -= n_dump;
712 ret = tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
714 if (0 == tbase->aux->task_rt_dump.n_print_tx) {
715 if (tbase->tx_pkt == tx_pkt_l3) {
716 tbase->aux->tx_pkt_l2 = tbase->aux->tx_pkt_orig;
717 tbase->aux->tx_pkt_orig = NULL;
719 tbase->tx_pkt = tbase->aux->tx_pkt_orig;
720 tbase->aux->tx_pkt_orig = NULL;
726 /* Gather the distribution of the number of packets that have been
727 xmitted from one TX call. Since the value is only modified by the
728 task that xmits the packet, no atomic operation is needed. */
729 int tx_pkt_distr(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
731 if (likely(n_pkts < TX_BUCKET_SIZE))
732 tbase->aux->tx_bucket[n_pkts]++;
734 tbase->aux->tx_bucket[TX_BUCKET_SIZE - 1]++;
735 return tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
738 int tx_pkt_bw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
740 uint32_t tx_bytes = 0;
741 uint32_t drop_bytes = 0;
743 for (uint16_t i = 0; i < n_pkts; ++i) {
744 if (!out || out[i] < OUT_HANDLED)
745 tx_bytes += mbuf_wire_size(mbufs[i]);
747 drop_bytes += mbuf_wire_size(mbufs[i]);
750 TASK_STATS_ADD_TX_BYTES(&tbase->aux->stats, tx_bytes);
751 TASK_STATS_ADD_DROP_BYTES(&tbase->aux->stats, drop_bytes);
752 return tbase->aux->tx_pkt_orig(tbase, mbufs, n_pkts, out);
755 int tx_pkt_drop_all(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, uint8_t *out)
757 for (uint16_t j = 0; j < n_pkts; ++j) {
758 rte_pktmbuf_free(mbufs[j]);
761 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, n_pkts);
763 for (uint16_t j = 0; j < n_pkts; ++j) {
764 if (out[j] == OUT_HANDLED)
765 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, 1);
767 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
772 static inline void dump_pkts(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
774 uint32_t n_dump = tbase->aux->task_rt_dump.n_print_tx;
775 uint32_t n_trace = tbase->aux->task_rt_dump.n_trace;
777 if (unlikely(n_dump)) {
778 n_dump = n_pkts < n_dump? n_pkts : n_dump;
779 for (uint32_t i = 0; i < n_dump; ++i) {
780 plogdx_info(mbufs[i], "TX: ");
782 tbase->aux->task_rt_dump.n_print_tx -= n_dump;
783 } else if (unlikely(n_trace)) {
784 n_trace = n_pkts < n_trace? n_pkts : n_trace;
785 for (uint32_t i = 0; i < n_trace; ++i) {
786 plogdx_info(mbufs[i], "TX: ");
788 tbase->aux->task_rt_dump.n_trace -= n_trace;
792 // ctrlplane packets are slow path, hence cost of checking if dump ortrace is needed in not too important
793 // easier to have this implementation than an implementation similar to dataplane tx
794 int tx_ctrlplane_hw(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
796 dump_pkts(tbase, mbufs, n_pkts);
797 return txhw_no_drop(&tbase->tx_params_hw.tx_port_queue[0], mbufs, n_pkts, tbase);
800 int tx_ctrlplane_sw(struct task_base *tbase, struct rte_mbuf **mbufs, const uint16_t n_pkts, __attribute__((unused)) uint8_t *out)
802 dump_pkts(tbase, mbufs, n_pkts);
803 return ring_enq_no_drop(tbase->tx_params_sw.tx_rings[0], mbufs, n_pkts, tbase);
806 static inline int tx_ring_all(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
808 if (tbase->aux->task_rt_dump.cur_trace) {
809 trace_one_rx_pkt(tbase, mbuf);
811 mbuf->udata64 = ((uint64_t)ip << 32) | (core_id << 16) | (task_id << 8) | command;
812 return rte_ring_enqueue(ring, mbuf);
815 int tx_ring_cti(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint8_t core_id, uint8_t task_id, uint32_t ip)
817 plogx_dbg("\tSending command %s with ip %d.%d.%d.%d to ring %p using mbuf %p, core %d and task %d - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, core_id, task_id, rte_ring_free_count(ring));
818 int ret = tx_ring_all(tbase, ring, command, mbuf, core_id, task_id, ip);
819 if (unlikely(ret != 0)) {
820 plogx_dbg("\tFail to send command %s with ip %d.%d.%d.%d to ring %p using mbuf %p, core %d and task %d - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, core_id, task_id, rte_ring_free_count(ring));
821 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
822 rte_pktmbuf_free(mbuf);
827 void tx_ring_ip(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf, uint32_t ip)
829 plogx_dbg("\tSending command %s with ip %d.%d.%d.%d to ring %p using mbuf %p - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, rte_ring_free_count(ring));
830 int ret = tx_ring_all(tbase, ring, command, mbuf, 0, 0, ip);
831 if (unlikely(ret != 0)) {
832 plogx_dbg("\tFail to send command %s with ip %d.%d.%d.%d to ring %p using mbuf %p - ring size now %d\n", actions_string[command], IP4(ip), ring, mbuf, rte_ring_free_count(ring));
833 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
834 rte_pktmbuf_free(mbuf);
838 void tx_ring(struct task_base *tbase, struct rte_ring *ring, uint16_t command, struct rte_mbuf *mbuf)
840 plogx_dbg("\tSending command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
841 int ret = tx_ring_all(tbase, ring, command, mbuf, 0, 0, 0);
842 if (unlikely(ret != 0)) {
843 plogx_dbg("\tFail to send command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
844 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
845 rte_pktmbuf_free(mbuf);
849 void tx_ring_route(struct task_base *tbase, struct rte_ring *ring, int add, struct rte_mbuf *mbuf, uint32_t ip, uint32_t gateway_ip, uint32_t prefix)
853 command = ROUTE_ADD_FROM_CTRL;
855 command = ROUTE_DEL_FROM_CTRL;
857 plogx_dbg("\tSending command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
858 ctrl_ring_set_command(mbuf, command);
859 ctrl_ring_set_ip(mbuf, ip);
860 ctrl_ring_set_gateway_ip(mbuf, gateway_ip);
861 ctrl_ring_set_prefix(mbuf, prefix);
862 if (tbase->aux->task_rt_dump.cur_trace) {
863 trace_one_rx_pkt(tbase, mbuf);
865 int ret = rte_ring_enqueue(ring, mbuf);
866 if (unlikely(ret != 0)) {
867 plogx_dbg("\tFail to send command %s to ring %p using mbuf %p - ring size now %d\n", actions_string[command], ring, mbuf, rte_ring_free_count(ring));
868 TASK_STATS_ADD_DROP_DISCARD(&tbase->aux->stats, 1);
869 rte_pktmbuf_free(mbuf);
873 void ctrl_ring_set_command(struct rte_mbuf *mbuf, uint64_t udata64)
875 mbuf->udata64 = udata64;
878 uint64_t ctrl_ring_get_command(struct rte_mbuf *mbuf)
880 return mbuf->udata64;
883 void ctrl_ring_set_ip(struct rte_mbuf *mbuf, uint32_t udata32)
885 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
886 prox_headroom->ip = udata32;
889 uint32_t ctrl_ring_get_ip(struct rte_mbuf *mbuf)
891 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
892 return prox_headroom->ip;
895 void ctrl_ring_set_gateway_ip(struct rte_mbuf *mbuf, uint32_t udata32)
897 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
898 prox_headroom->gateway_ip = udata32;
901 uint32_t ctrl_ring_get_gateway_ip(struct rte_mbuf *mbuf)
903 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
904 return prox_headroom->gateway_ip;
907 void ctrl_ring_set_prefix(struct rte_mbuf *mbuf, uint32_t udata32)
909 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
910 prox_headroom->prefix = udata32;
913 uint32_t ctrl_ring_get_prefix(struct rte_mbuf *mbuf)
915 struct prox_headroom *prox_headroom = (struct prox_headroom *)(rte_pktmbuf_mtod(mbuf, uint8_t*) - sizeof(struct prox_headroom));
916 return prox_headroom->prefix;