2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
21 #include <rte_cycles.h>
22 #include <rte_version.h>
23 #include <rte_byteorder.h>
24 #include <rte_ether.h>
25 #include <rte_hash_crc.h>
27 #include "prox_shared.h"
29 #include "prox_malloc.h"
30 #include "handle_gen.h"
31 #include "handle_lat.h"
32 #include "task_init.h"
33 #include "task_base.h"
34 #include "prox_port_cfg.h"
39 #include "mbuf_utils.h"
41 #include "prox_cksum.h"
43 #include "prox_assert.h"
45 #include "token_time.h"
46 #include "local_mbuf.h"
49 #include "handle_master.h"
58 #define MAX_TEMPLATE_INDEX 65536
59 #define TEMPLATE_INDEX_MASK (MAX_TEMPLATE_INDEX - 1)
60 #define MBUF_ARP MAX_TEMPLATE_INDEX
62 #define IP4(x) x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, x >> 24
64 static void pkt_template_init_mbuf(struct pkt_template *pkt_template, struct rte_mbuf *mbuf, uint8_t *pkt)
66 const uint32_t pkt_size = pkt_template->len;
68 rte_pktmbuf_pkt_len(mbuf) = pkt_size;
69 rte_pktmbuf_data_len(mbuf) = pkt_size;
71 rte_memcpy(pkt, pkt_template->buf, pkt_template->len);
74 struct task_gen_pcap {
75 struct task_base base;
77 struct local_mbuf local_mbuf;
79 struct pkt_template *proto;
87 struct task_base base;
90 struct token_time token_time;
91 struct local_mbuf local_mbuf;
92 struct pkt_template *pkt_template; /* packet templates used at runtime */
93 uint64_t write_duration_estimate; /* how long it took previously to write the time stamps in the packets */
94 uint64_t earliest_tsc_next_pkt;
95 uint64_t new_rate_bps;
96 uint64_t pkt_queue_index;
97 uint32_t n_pkts; /* number of packets in pcap */
98 uint32_t pkt_idx; /* current packet from pcap */
99 uint32_t pkt_count; /* how many pakets to generate */
100 uint32_t max_frame_size;
101 uint32_t runtime_flags;
103 uint16_t packet_id_pos;
107 uint8_t generator_id;
108 uint8_t n_rands; /* number of randoms */
109 uint8_t min_bulk_size;
110 uint8_t max_bulk_size;
112 uint8_t runtime_checksum_needed;
115 uint32_t rand_mask; /* since the random vals are uniform, masks don't introduce bias */
116 uint32_t fixed_bits; /* length of each random (max len = 4) */
117 uint16_t rand_offset; /* each random has an offset*/
118 uint8_t rand_len; /* # bytes to take from random (no bias introduced) */
121 uint64_t pkt_tsc_offset[64];
122 struct pkt_template *pkt_template_orig; /* packet templates (from inline or from pcap) */
123 struct ether_addr src_mac;
125 uint8_t cksum_offload;
126 struct prox_port_cfg *port;
127 } __rte_cache_aligned;
129 static inline uint8_t ipv4_get_hdr_len(struct ipv4_hdr *ip)
131 /* Optimize for common case of IPv4 header without options. */
132 if (ip->version_ihl == 0x45)
133 return sizeof(struct ipv4_hdr);
134 if (unlikely(ip->version_ihl >> 4 != 4)) {
135 plog_warn("IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
138 return (ip->version_ihl & 0xF) * 4;
141 static void parse_l2_l3_len(uint8_t *pkt, uint16_t *l2_len, uint16_t *l3_len, uint16_t len)
143 *l2_len = sizeof(struct ether_hdr);
145 struct vlan_hdr *vlan_hdr;
146 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
148 uint16_t ether_type = eth_hdr->ether_type;
151 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (*l2_len + sizeof(struct vlan_hdr) < len)) {
152 vlan_hdr = (struct vlan_hdr *)(pkt + *l2_len);
154 ether_type = vlan_hdr->eth_proto;
157 // No L3 cksum offload for IPv6, but TODO L4 offload
158 // ETYPE_EoGRE CRC not implemented yet
160 switch (ether_type) {
174 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
179 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + *l2_len);
180 *l3_len = ipv4_get_hdr_len(ip);
184 static void checksum_packet(uint8_t *hdr, struct rte_mbuf *mbuf, struct pkt_template *pkt_template, int cksum_offload)
186 uint16_t l2_len = pkt_template->l2_len;
187 uint16_t l3_len = pkt_template->l3_len;
190 struct ipv4_hdr *ip = (struct ipv4_hdr*)(hdr + l2_len);
191 prox_ip_udp_cksum(mbuf, ip, l2_len, l3_len, cksum_offload);
195 static void task_gen_reset_token_time(struct task_gen *task)
197 token_time_set_bpp(&task->token_time, task->new_rate_bps);
198 token_time_reset(&task->token_time, rte_rdtsc(), 0);
201 static void task_gen_take_count(struct task_gen *task, uint32_t send_bulk)
203 if (task->pkt_count == (uint32_t)-1)
206 if (task->pkt_count >= send_bulk)
207 task->pkt_count -= send_bulk;
213 static int handle_gen_pcap_bulk(struct task_base *tbase, struct rte_mbuf **mbuf, uint16_t n_pkts)
215 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
216 uint64_t now = rte_rdtsc();
217 uint64_t send_bulk = 0;
218 uint32_t pkt_idx_tmp = task->pkt_idx;
220 if (pkt_idx_tmp == task->n_pkts) {
221 PROX_ASSERT(task->loop);
225 for (uint16_t j = 0; j < 64; ++j) {
226 uint64_t tsc = task->proto_tsc[pkt_idx_tmp];
227 if (task->last_tsc + tsc <= now) {
228 task->last_tsc += tsc;
231 if (pkt_idx_tmp == task->n_pkts) {
242 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
243 if (new_pkts == NULL)
246 for (uint16_t j = 0; j < send_bulk; ++j) {
247 struct rte_mbuf *next_pkt = new_pkts[j];
248 struct pkt_template *pkt_template = &task->proto[task->pkt_idx];
249 uint8_t *hdr = rte_pktmbuf_mtod(next_pkt, uint8_t *);
251 pkt_template_init_mbuf(pkt_template, next_pkt, hdr);
254 if (task->pkt_idx == task->n_pkts) {
262 return task->base.tx_pkt(&task->base, new_pkts, send_bulk, NULL);
265 static uint64_t bytes_to_tsc(struct task_gen *task, uint32_t bytes)
267 const uint64_t hz = task->hz;
268 const uint64_t bytes_per_hz = task->link_speed;
270 if (bytes_per_hz == UINT64_MAX)
273 return hz * bytes / bytes_per_hz;
276 static uint32_t task_gen_next_pkt_idx(const struct task_gen *task, uint32_t pkt_idx)
278 return pkt_idx + 1 == task->n_pkts? 0 : pkt_idx + 1;
281 static uint32_t task_gen_offset_pkt_idx(const struct task_gen *task, uint32_t offset)
283 return (task->pkt_idx + offset) % task->n_pkts;
286 static uint32_t task_gen_calc_send_bulk(const struct task_gen *task, uint32_t *total_bytes)
288 /* The biggest bulk we allow to send is task->max_bulk_size
289 packets. The max bulk size can also be limited by the
290 pkt_count field. At the same time, we are rate limiting
291 based on the specified speed (in bytes per second) so token
292 bucket based rate limiting must also be applied. The
293 minimum bulk size is also constrained. If the calculated
294 bulk size is less then the minimum, then don't send
297 const uint32_t min_bulk = task->min_bulk_size;
298 uint32_t max_bulk = task->max_bulk_size;
300 if (task->pkt_count != (uint32_t)-1 && task->pkt_count < max_bulk) {
301 max_bulk = task->pkt_count;
304 uint32_t send_bulk = 0;
305 uint32_t pkt_idx_tmp = task->pkt_idx;
306 uint32_t would_send_bytes = 0;
310 * TODO - this must be improved to take into account the fact that, after applying randoms
311 * The packet can be replaced by an ARP
313 for (uint16_t j = 0; j < max_bulk; ++j) {
314 struct pkt_template *pktpl = &task->pkt_template[pkt_idx_tmp];
315 pkt_size = pktpl->len;
316 uint32_t pkt_len = pkt_len_to_wire_size(pkt_size);
317 if (pkt_len + would_send_bytes > task->token_time.bytes_now)
320 pkt_idx_tmp = task_gen_next_pkt_idx(task, pkt_idx_tmp);
323 would_send_bytes += pkt_len;
326 if (send_bulk < min_bulk)
328 *total_bytes = would_send_bytes;
332 static void task_gen_apply_random_fields(struct task_gen *task, uint8_t *hdr)
334 uint32_t ret, ret_tmp;
336 for (uint16_t i = 0; i < task->n_rands; ++i) {
337 ret = random_next(&task->rand[i].state);
338 ret_tmp = (ret & task->rand[i].rand_mask) | task->rand[i].fixed_bits;
340 ret_tmp = rte_bswap32(ret_tmp);
341 /* At this point, the lower order bytes (BE) contain
342 the generated value. The address where the values
343 of interest starts is at ret_tmp + 4 - rand_len. */
344 uint8_t *pret_tmp = (uint8_t*)&ret_tmp;
345 rte_memcpy(hdr + task->rand[i].rand_offset, pret_tmp + 4 - task->rand[i].rand_len, task->rand[i].rand_len);
349 static void task_gen_apply_all_random_fields(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
354 for (uint16_t i = 0; i < count; ++i)
355 task_gen_apply_random_fields(task, pkt_hdr[i]);
358 static void task_gen_apply_accur_pos(struct task_gen *task, uint8_t *pkt_hdr, uint32_t accuracy)
360 *(uint32_t *)(pkt_hdr + task->accur_pos) = accuracy;
363 static void task_gen_apply_sig(struct task_gen *task, uint8_t *pkt_hdr)
365 *(uint32_t *)(pkt_hdr + task->sig_pos) = task->sig;
368 static void task_gen_apply_all_accur_pos(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
370 if (!task->accur_pos)
373 /* The accuracy of task->pkt_queue_index - 64 is stored in
374 packet task->pkt_queue_index. The ID modulo 64 is the
376 for (uint16_t j = 0; j < count; ++j) {
377 if ((mbufs[j]->udata64 & MBUF_ARP) == 0) {
378 uint32_t accuracy = task->accur[(task->pkt_queue_index + j) & 63];
379 task_gen_apply_accur_pos(task, pkt_hdr[j], accuracy);
384 static void task_gen_apply_all_sig(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
389 for (uint16_t j = 0; j < count; ++j) {
390 if ((mbufs[j]->udata64 & MBUF_ARP) == 0) {
391 task_gen_apply_sig(task, pkt_hdr[j]);
396 static void task_gen_apply_unique_id(struct task_gen *task, uint8_t *pkt_hdr, const struct unique_id *id)
398 struct unique_id *dst = (struct unique_id *)(pkt_hdr + task->packet_id_pos);
403 static void task_gen_apply_all_unique_id(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
405 if (!task->packet_id_pos)
408 for (uint16_t i = 0; i < count; ++i) {
409 if ((mbufs[i]->udata64 & MBUF_ARP) == 0) {
411 unique_id_init(&id, task->generator_id, task->pkt_queue_index++);
412 task_gen_apply_unique_id(task, pkt_hdr[i], &id);
417 static void task_gen_checksum_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
419 if (!(task->runtime_flags & TASK_TX_CRC))
422 if (!task->runtime_checksum_needed)
425 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - count);
426 for (uint16_t i = 0; i < count; ++i) {
427 if ((mbufs[i]->udata64 & MBUF_ARP) == 0) {
428 struct pkt_template *pkt_template = &task->pkt_template[pkt_idx];
429 checksum_packet(pkt_hdr[i], mbufs[i], pkt_template, task->cksum_offload);
430 pkt_idx = task_gen_next_pkt_idx(task, pkt_idx);
435 static void task_gen_consume_tokens(struct task_gen *task, uint32_t tokens, uint32_t send_count)
437 /* If max burst has been sent, we can't keep up so just assume
438 that we can (leaving a "gap" in the packet stream on the
440 task->token_time.bytes_now -= tokens;
441 if (send_count == task->max_bulk_size && task->token_time.bytes_now > tokens) {
442 task->token_time.bytes_now = tokens;
446 static uint64_t task_gen_calc_bulk_duration(struct task_gen *task, uint32_t count)
448 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - 1);
449 struct pkt_template *last_pkt_template = &task->pkt_template[pkt_idx];
450 uint32_t last_pkt_len = pkt_len_to_wire_size(last_pkt_template->len);
451 uint64_t last_pkt_duration = bytes_to_tsc(task, last_pkt_len);
452 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1] + last_pkt_duration;
454 return bulk_duration;
457 static uint64_t task_gen_write_latency(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
459 if (!task->lat_enabled)
462 uint64_t tx_tsc, delta_t;
463 uint64_t tsc_before_tx = 0;
465 /* Just before sending the packets, apply the time stamp
466 relative to when the first packet will be sent. The first
467 packet will be sent now. The time is read for each packet
468 to reduce the error towards the actual time the packet will
470 uint64_t write_tsc_after, write_tsc_before;
472 write_tsc_before = rte_rdtsc();
474 /* The time it took previously to write the time stamps in the
475 packets is used as an estimate for how long it will take to
476 write the time stamps now. The estimated time at which the
477 packets will actually be sent will be at tx_tsc. */
478 tx_tsc = write_tsc_before + task->write_duration_estimate;
480 /* The offset delta_t tracks the difference between the actual
481 time and the time written in the packets. Adding the offset
482 to the actual time insures that the time written in the
483 packets is monotonically increasing. At the same time,
484 simply sleeping until delta_t is zero would leave a period
485 of silence on the line. The error has been introduced
486 earlier, but the packets have already been sent. */
487 if (tx_tsc < task->earliest_tsc_next_pkt)
488 delta_t = task->earliest_tsc_next_pkt - tx_tsc;
492 for (uint16_t i = 0; i < count; ++i) {
493 uint32_t *pos = (uint32_t *)(pkt_hdr[i] + task->lat_pos);
494 const uint64_t pkt_tsc = tx_tsc + delta_t + task->pkt_tsc_offset[i];
496 *pos = pkt_tsc >> LATENCY_ACCURACY;
499 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, count);
501 task->earliest_tsc_next_pkt = tx_tsc + delta_t + bulk_duration;
502 write_tsc_after = rte_rdtsc();
503 task->write_duration_estimate = write_tsc_after - write_tsc_before;
505 /* Make sure that the time stamps that were written
506 are valid. The offset must be taken into account */
508 tsc_before_tx = rte_rdtsc();
509 } while (tsc_before_tx < tx_tsc);
510 return tsc_before_tx;
513 static void task_gen_store_accuracy(struct task_gen *task, uint32_t count, uint64_t tsc_before_tx)
515 if (!task->accur_pos)
518 uint64_t accur = rte_rdtsc() - tsc_before_tx;
519 uint64_t first_accuracy_idx = task->pkt_queue_index - count;
521 for (uint32_t i = 0; i < count; ++i) {
522 uint32_t accuracy_idx = (first_accuracy_idx + i) & 63;
524 task->accur[accuracy_idx] = accur;
528 static void task_gen_load_and_prefetch(struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
530 for (uint16_t i = 0; i < count; ++i)
531 rte_prefetch0(mbufs[i]);
532 for (uint16_t i = 0; i < count; ++i)
533 pkt_hdr[i] = rte_pktmbuf_mtod(mbufs[i], uint8_t *);
534 for (uint16_t i = 0; i < count; ++i)
535 rte_prefetch0(pkt_hdr[i]);
538 static void task_gen_build_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
540 uint64_t will_send_bytes = 0;
542 for (uint16_t i = 0; i < count; ++i) {
543 struct pkt_template *pktpl = &task->pkt_template[task->pkt_idx];
544 struct pkt_template *pkt_template = &task->pkt_template[task->pkt_idx];
545 pkt_template_init_mbuf(pkt_template, mbufs[i], pkt_hdr[i]);
546 mbufs[i]->udata64 = task->pkt_idx & TEMPLATE_INDEX_MASK;
547 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
548 if (task->lat_enabled) {
549 task->pkt_tsc_offset[i] = bytes_to_tsc(task, will_send_bytes);
550 will_send_bytes += pkt_len_to_wire_size(pkt_template->len);
552 task->pkt_idx = task_gen_next_pkt_idx(task, task->pkt_idx);
556 static void task_gen_update_config(struct task_gen *task)
558 if (task->token_time.cfg.bpp != task->new_rate_bps)
559 task_gen_reset_token_time(task);
562 static inline void build_value(struct task_gen *task, uint32_t mask, int bit_pos, uint32_t val, uint32_t fixed_bits)
564 struct task_base *tbase = (struct task_base *)task;
566 build_value(task, mask >> 1, bit_pos + 1, val, fixed_bits);
568 build_value(task, mask >> 1, bit_pos + 1, val | (1 << bit_pos), fixed_bits);
571 register_ip_to_ctrl_plane(tbase->l3.tmaster, rte_cpu_to_be_32(val | fixed_bits), tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
574 static inline void register_all_ip_to_ctrl_plane(struct task_gen *task)
576 struct task_base *tbase = (struct task_base *)task;
581 for (uint32_t i = 0; i < task->n_pkts; ++i) {
582 struct pkt_template *pktpl = &task->pkt_template[i];
583 unsigned int ip_src_pos = 0;
585 unsigned int l2_len = sizeof(struct ether_hdr);
587 uint8_t *pkt = pktpl->buf;
588 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
589 uint16_t ether_type = eth_hdr->ether_type;
590 struct vlan_hdr *vlan_hdr;
593 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(struct vlan_hdr) < pktpl->len)) {
594 vlan_hdr = (struct vlan_hdr *)(pkt + l2_len);
596 ether_type = vlan_hdr->eth_proto;
598 if ((ether_type == ETYPE_MPLSU) || (ether_type == ETYPE_MPLSM)) {
602 if ((ether_type != ETYPE_IPv4) && !maybe_ipv4)
605 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + l2_len);
606 PROX_PANIC(ip->version_ihl >> 4 != 4, "IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
608 // Even if IPv4 header contains options, options are after ip src and dst
609 ip_src_pos = l2_len + sizeof(struct ipv4_hdr) - 2 * sizeof(uint32_t);
610 uint32_t *ip_src = ((uint32_t *)(pktpl->buf + ip_src_pos));
611 plog_info("\tip_src_pos = %d, ip_src = %x\n", ip_src_pos, *ip_src);
612 register_ip_to_ctrl_plane(tbase->l3.tmaster, *ip_src, tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
614 for (int j = 0; j < task->n_rands; j++) {
615 offset = task->rand[j].rand_offset;
616 len = task->rand[j].rand_len;
617 mask = task->rand[j].rand_mask;
618 fixed = task->rand[j].fixed_bits;
619 plog_info("offset = %d, len = %d, mask = %x, fixed = %x\n", offset, len, mask, fixed);
620 if ((offset < ip_src_pos + 4) && (offset + len >= ip_src_pos)) {
621 if (offset >= ip_src_pos) {
622 int32_t ip_src_mask = (1 << (4 + ip_src_pos - offset) * 8) - 1;
623 mask = mask & ip_src_mask;
624 fixed = (fixed & ip_src_mask) | (rte_be_to_cpu_32(*ip_src) & ~ip_src_mask);
625 build_value(task, mask, 0, 0, fixed);
627 int32_t bits = ((ip_src_pos + 4 - offset - len) * 8);
629 fixed = (fixed << bits) | (rte_be_to_cpu_32(*ip_src) & ((1 << bits) - 1));
630 build_value(task, mask, 0, 0, fixed);
637 static int handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
639 struct task_gen *task = (struct task_gen *)tbase;
640 uint8_t out[MAX_PKT_BURST] = {0};
645 #if RTE_VERSION < RTE_VERSION_NUM(16,4,0,0)
646 // On more recent DPDK, we use the speed_capa of the port, and not the negotiated speed
647 // If link is down, link_speed is 0
648 if (unlikely(task->link_speed == 0)) {
649 if (task->port && task->port->link_speed != 0) {
650 task->link_speed = task->port->link_speed * 125000L;
651 plog_info("\tPort %u: link speed is %ld Mbps\n",
652 (uint8_t)(task->port - prox_port_cfg), 8 * task->link_speed / 1000000);
658 task_gen_update_config(task);
660 if (task->pkt_count == 0) {
661 task_gen_reset_token_time(task);
664 if (!task->token_time.cfg.bpp)
667 token_time_update(&task->token_time, rte_rdtsc());
669 uint32_t would_send_bytes;
670 uint32_t send_bulk = task_gen_calc_send_bulk(task, &would_send_bytes);
674 task_gen_take_count(task, send_bulk);
675 task_gen_consume_tokens(task, would_send_bytes, send_bulk);
677 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
678 if (new_pkts == NULL)
680 uint8_t *pkt_hdr[MAX_RING_BURST];
682 task_gen_load_and_prefetch(new_pkts, pkt_hdr, send_bulk);
683 task_gen_build_packets(task, new_pkts, pkt_hdr, send_bulk);
684 task_gen_apply_all_random_fields(task, pkt_hdr, send_bulk);
685 task_gen_apply_all_accur_pos(task, new_pkts, pkt_hdr, send_bulk);
686 task_gen_apply_all_sig(task, new_pkts, pkt_hdr, send_bulk);
687 task_gen_apply_all_unique_id(task, new_pkts, pkt_hdr, send_bulk);
689 uint64_t tsc_before_tx;
691 tsc_before_tx = task_gen_write_latency(task, pkt_hdr, send_bulk);
692 task_gen_checksum_packets(task, new_pkts, pkt_hdr, send_bulk);
693 ret = task->base.tx_pkt(&task->base, new_pkts, send_bulk, out);
694 task_gen_store_accuracy(task, send_bulk, tsc_before_tx);
698 static void init_task_gen_seeds(struct task_gen *task)
700 for (size_t i = 0; i < sizeof(task->rand)/sizeof(task->rand[0]); ++i)
701 random_init_seed(&task->rand[i].state);
704 static uint32_t pcap_count_pkts(pcap_t *handle, uint32_t *max_frame_size)
706 struct pcap_pkthdr header;
710 long pkt1_fpos = ftell(pcap_file(handle));
712 while ((buf = pcap_next(handle, &header))) {
713 if (header.len > *max_frame_size)
714 *max_frame_size = header.len;
717 int ret2 = fseek(pcap_file(handle), pkt1_fpos, SEEK_SET);
718 PROX_PANIC(ret2 != 0, "Failed to reset reading pcap file\n");
722 static uint64_t avg_time_stamp(uint64_t *time_stamp, uint32_t n)
724 uint64_t tot_inter_pkt = 0;
726 for (uint32_t i = 0; i < n; ++i)
727 tot_inter_pkt += time_stamp[i];
728 return (tot_inter_pkt + n / 2)/n;
731 static int pcap_read_pkts(pcap_t *handle, const char *file_name, uint32_t n_pkts, struct pkt_template *proto, uint64_t *time_stamp)
733 struct pcap_pkthdr header;
737 for (uint32_t i = 0; i < n_pkts; ++i) {
738 buf = pcap_next(handle, &header);
740 PROX_PANIC(buf == NULL, "Failed to read packet %d from pcap %s\n", i, file_name);
741 proto[i].len = header.len;
742 len = RTE_MIN(header.len, sizeof(proto[i].buf));
743 if (header.len > len)
744 plogx_warn("Packet truncated from %u to %zu bytes\n", header.len, len);
747 static struct timeval beg;
753 tv = tv_diff(&beg, &header.ts);
754 tv_to_tsc(&tv, time_stamp + i);
756 rte_memcpy(proto[i].buf, buf, len);
759 if (time_stamp && n_pkts) {
760 for (uint32_t i = n_pkts - 1; i > 0; --i)
761 time_stamp[i] -= time_stamp[i - 1];
762 /* Since the handle function will loop the packets,
763 there is one time-stamp that is not provided by the
764 pcap file. This is the time between the last and
765 the first packet. This implementation takes the
766 average of the inter-packet times here. */
768 time_stamp[0] = avg_time_stamp(time_stamp + 1, n_pkts - 1);
774 static int check_pkt_size(struct task_gen *task, uint32_t pkt_size, int do_panic)
776 const uint16_t min_len = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr);
777 const uint16_t max_len = task->max_frame_size;
780 PROX_PANIC(pkt_size == 0, "Invalid packet size length (no packet defined?)\n");
781 PROX_PANIC(pkt_size > max_len, "pkt_size out of range (must be <= %u)\n", max_len);
782 PROX_PANIC(pkt_size < min_len, "pkt_size out of range (must be >= %u)\n", min_len);
786 plog_err("Invalid packet size length (no packet defined?)\n");
789 if (pkt_size > max_len) {
790 plog_err("pkt_size out of range (must be <= %u)\n", max_len);
793 if (pkt_size < min_len) {
794 plog_err("pkt_size out of range (must be >= %u)\n", min_len);
801 static int check_all_pkt_size(struct task_gen *task, int do_panic)
804 for (uint32_t i = 0; i < task->n_pkts;++i) {
805 if ((rc = check_pkt_size(task, task->pkt_template[i].len, do_panic)) != 0)
811 static int check_fields_in_bounds(struct task_gen *task, uint32_t pkt_size, int do_panic)
813 if (task->lat_enabled) {
814 uint32_t pos_beg = task->lat_pos;
815 uint32_t pos_end = task->lat_pos + 3U;
818 PROX_PANIC(pkt_size <= pos_end, "Writing latency at %u-%u, but packet size is %u bytes\n",
819 pos_beg, pos_end, pkt_size);
820 else if (pkt_size <= pos_end) {
821 plog_err("Writing latency at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
825 if (task->packet_id_pos) {
826 uint32_t pos_beg = task->packet_id_pos;
827 uint32_t pos_end = task->packet_id_pos + 4U;
830 PROX_PANIC(pkt_size <= pos_end, "Writing packet at %u-%u, but packet size is %u bytes\n",
831 pos_beg, pos_end, pkt_size);
832 else if (pkt_size <= pos_end) {
833 plog_err("Writing packet at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
837 if (task->accur_pos) {
838 uint32_t pos_beg = task->accur_pos;
839 uint32_t pos_end = task->accur_pos + 3U;
842 PROX_PANIC(pkt_size <= pos_end, "Writing accuracy at %u%-u, but packet size is %u bytes\n",
843 pos_beg, pos_end, pkt_size);
844 else if (pkt_size <= pos_end) {
845 plog_err("Writing accuracy at %u%-u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
852 static void task_gen_pkt_template_recalc_metadata(struct task_gen *task)
854 struct pkt_template *template;
856 for (size_t i = 0; i < task->n_pkts; ++i) {
857 template = &task->pkt_template[i];
858 parse_l2_l3_len(template->buf, &template->l2_len, &template->l3_len, template->len);
862 static void task_gen_pkt_template_recalc_checksum(struct task_gen *task)
864 struct pkt_template *template;
867 task->runtime_checksum_needed = 0;
868 for (size_t i = 0; i < task->n_pkts; ++i) {
869 template = &task->pkt_template[i];
870 if (template->l2_len == 0)
872 ip = (struct ipv4_hdr *)(template->buf + template->l2_len);
874 ip->hdr_checksum = 0;
875 prox_ip_cksum_sw(ip);
876 uint32_t l4_len = rte_bswap16(ip->total_length) - template->l3_len;
878 if (ip->next_proto_id == IPPROTO_UDP) {
879 struct udp_hdr *udp = (struct udp_hdr *)(((uint8_t *)ip) + template->l3_len);
880 prox_udp_cksum_sw(udp, l4_len, ip->src_addr, ip->dst_addr);
881 } else if (ip->next_proto_id == IPPROTO_TCP) {
882 struct tcp_hdr *tcp = (struct tcp_hdr *)(((uint8_t *)ip) + template->l3_len);
883 prox_tcp_cksum_sw(tcp, l4_len, ip->src_addr, ip->dst_addr);
886 /* The current implementation avoids checksum
887 calculation by determining that at packet
888 construction time, no fields are applied that would
889 require a recalculation of the checksum. */
890 if (task->lat_enabled && task->lat_pos > template->l2_len)
891 task->runtime_checksum_needed = 1;
892 if (task->accur_pos > template->l2_len)
893 task->runtime_checksum_needed = 1;
894 if (task->packet_id_pos > template->l2_len)
895 task->runtime_checksum_needed = 1;
899 static void task_gen_pkt_template_recalc_all(struct task_gen *task)
901 task_gen_pkt_template_recalc_metadata(task);
902 task_gen_pkt_template_recalc_checksum(task);
905 static void task_gen_reset_pkt_templates_len(struct task_gen *task)
907 struct pkt_template *src, *dst;
909 for (size_t i = 0; i < task->n_pkts; ++i) {
910 src = &task->pkt_template_orig[i];
911 dst = &task->pkt_template[i];
916 static void task_gen_reset_pkt_templates_content(struct task_gen *task)
918 struct pkt_template *src, *dst;
920 for (size_t i = 0; i < task->n_pkts; ++i) {
921 src = &task->pkt_template_orig[i];
922 dst = &task->pkt_template[i];
923 memcpy(dst->buf, src->buf, dst->len);
927 static void task_gen_reset_pkt_templates(struct task_gen *task)
929 task_gen_reset_pkt_templates_len(task);
930 task_gen_reset_pkt_templates_content(task);
931 task_gen_pkt_template_recalc_all(task);
934 static void task_init_gen_load_pkt_inline(struct task_gen *task, struct task_args *targ)
936 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
940 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
941 task->pkt_template = prox_zmalloc(mem_size, socket_id);
942 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
944 PROX_PANIC(task->pkt_template == NULL ||
945 task->pkt_template_orig == NULL,
946 "Failed to allocate %lu bytes (in huge pages) for packet template\n", mem_size);
948 task->pkt_template->buf = prox_zmalloc(task->max_frame_size, socket_id);
949 task->pkt_template_orig->buf = prox_zmalloc(task->max_frame_size, socket_id);
950 PROX_PANIC(task->pkt_template->buf == NULL ||
951 task->pkt_template_orig->buf == NULL,
952 "Failed to allocate %u bytes (in huge pages) for packet\n", task->max_frame_size);
954 PROX_PANIC(targ->pkt_size > task->max_frame_size,
955 targ->pkt_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE - 4 ?
956 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
958 rte_memcpy(task->pkt_template_orig[0].buf, targ->pkt_inline, targ->pkt_size);
959 task->pkt_template_orig[0].len = targ->pkt_size;
960 task_gen_reset_pkt_templates(task);
961 check_all_pkt_size(task, 1);
962 check_fields_in_bounds(task, task->pkt_template[0].len, 1);
965 static void task_init_gen_load_pcap(struct task_gen *task, struct task_args *targ)
967 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
968 char err[PCAP_ERRBUF_SIZE];
969 uint32_t max_frame_size;
970 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
971 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
973 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
974 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
975 PROX_PANIC(max_frame_size > task->max_frame_size,
976 max_frame_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE -4 ?
977 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
980 task->n_pkts = RTE_MIN(task->n_pkts, targ->n_pkts);
981 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
982 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
983 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
984 task->pkt_template = prox_zmalloc(mem_size, socket_id);
985 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
986 PROX_PANIC(task->pkt_template == NULL ||
987 task->pkt_template_orig == NULL,
988 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
990 for (uint i = 0; i < task->n_pkts; i++) {
991 task->pkt_template[i].buf = prox_zmalloc(max_frame_size, socket_id);
992 task->pkt_template_orig[i].buf = prox_zmalloc(max_frame_size, socket_id);
994 PROX_PANIC(task->pkt_template->buf == NULL ||
995 task->pkt_template_orig->buf == NULL,
996 "Failed to allocate %u bytes (in huge pages) for pcap file\n", task->max_frame_size);
999 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->pkt_template_orig, NULL);
1001 task_gen_reset_pkt_templates(task);
1004 static struct rte_mempool *task_gen_create_mempool(struct task_args *targ, uint16_t max_frame_size)
1006 static char name[] = "gen_pool";
1007 struct rte_mempool *ret;
1008 const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
1011 uint32_t mbuf_size = TX_MBUF_SIZE;
1012 if (max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM > mbuf_size)
1013 mbuf_size = max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
1014 plog_info("\t\tCreating mempool with name '%s'\n", name);
1015 ret = rte_mempool_create(name, targ->nb_mbuf - 1, mbuf_size,
1016 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
1017 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
1019 PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
1020 sock_id, targ->nb_mbuf - 1);
1022 plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", ret,
1023 targ->nb_mbuf - 1, mbuf_size, targ->nb_cache_mbuf, sock_id);
1028 void task_gen_set_pkt_count(struct task_base *tbase, uint32_t count)
1030 struct task_gen *task = (struct task_gen *)tbase;
1032 task->pkt_count = count;
1035 int task_gen_set_pkt_size(struct task_base *tbase, uint32_t pkt_size)
1037 struct task_gen *task = (struct task_gen *)tbase;
1040 if ((rc = check_pkt_size(task, pkt_size, 0)) != 0)
1042 if ((rc = check_fields_in_bounds(task, pkt_size, 0)) != 0)
1044 task->pkt_template[0].len = pkt_size;
1048 void task_gen_set_rate(struct task_base *tbase, uint64_t bps)
1050 struct task_gen *task = (struct task_gen *)tbase;
1052 task->new_rate_bps = bps;
1055 void task_gen_reset_randoms(struct task_base *tbase)
1057 struct task_gen *task = (struct task_gen *)tbase;
1059 for (uint32_t i = 0; i < task->n_rands; ++i) {
1060 task->rand[i].rand_mask = 0;
1061 task->rand[i].fixed_bits = 0;
1062 task->rand[i].rand_offset = 0;
1067 int task_gen_set_value(struct task_base *tbase, uint32_t value, uint32_t offset, uint32_t len)
1069 struct task_gen *task = (struct task_gen *)tbase;
1071 for (size_t i = 0; i < task->n_pkts; ++i) {
1072 uint32_t to_write = rte_cpu_to_be_32(value) >> ((4 - len) * 8);
1073 uint8_t *dst = task->pkt_template[i].buf;
1075 rte_memcpy(dst + offset, &to_write, len);
1078 task_gen_pkt_template_recalc_all(task);
1083 void task_gen_reset_values(struct task_base *tbase)
1085 struct task_gen *task = (struct task_gen *)tbase;
1087 task_gen_reset_pkt_templates_content(task);
1090 uint32_t task_gen_get_n_randoms(struct task_base *tbase)
1092 struct task_gen *task = (struct task_gen *)tbase;
1094 return task->n_rands;
1097 static void init_task_gen_pcap(struct task_base *tbase, struct task_args *targ)
1099 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1100 const uint32_t sockid = rte_lcore_to_socket_id(targ->lconf->id);
1101 uint32_t max_frame_size;
1103 task->loop = targ->loop;
1105 task->hz = rte_get_tsc_hz();
1107 char err[PCAP_ERRBUF_SIZE];
1108 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1109 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1111 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
1112 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1114 task->local_mbuf.mempool = task_gen_create_mempool(targ, max_frame_size);
1116 PROX_PANIC(!strcmp(targ->pcap_file, ""), "No pcap file defined\n");
1119 plogx_info("Configured to load %u packets\n", targ->n_pkts);
1120 if (task->n_pkts > targ->n_pkts)
1121 task->n_pkts = targ->n_pkts;
1123 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
1125 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1127 size_t mem_size = task->n_pkts * (sizeof(*task->proto) + sizeof(*task->proto_tsc));
1128 uint8_t *mem = prox_zmalloc(mem_size, sockid);
1130 PROX_PANIC(mem == NULL, "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1131 task->proto = (struct pkt_template *) mem;
1132 task->proto_tsc = (uint64_t *)(mem + task->n_pkts * sizeof(*task->proto));
1134 for (uint i = 0; i < targ->n_pkts; i++) {
1135 task->proto[i].buf = prox_zmalloc(max_frame_size, sockid);
1136 PROX_PANIC(task->proto[i].buf == NULL, "Failed to allocate %u bytes (in huge pages) for pcap file\n", max_frame_size);
1139 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->proto, task->proto_tsc);
1143 static int task_gen_find_random_with_offset(struct task_gen *task, uint32_t offset)
1145 for (uint32_t i = 0; i < task->n_rands; ++i) {
1146 if (task->rand[i].rand_offset == offset) {
1154 int task_gen_add_rand(struct task_base *tbase, const char *rand_str, uint32_t offset, uint32_t rand_id)
1156 struct task_gen *task = (struct task_gen *)tbase;
1157 uint32_t existing_rand;
1159 if (rand_id == UINT32_MAX && task->n_rands == 64) {
1160 plog_err("Too many randoms\n");
1163 uint32_t mask, fixed, len;
1165 if (parse_random_str(&mask, &fixed, &len, rand_str)) {
1166 plog_err("%s\n", get_parse_err());
1169 task->runtime_checksum_needed = 1;
1171 existing_rand = task_gen_find_random_with_offset(task, offset);
1172 if (existing_rand != UINT32_MAX) {
1173 plog_warn("Random at offset %d already set => overwriting len = %d %s\n", offset, len, rand_str);
1174 rand_id = existing_rand;
1175 task->rand[rand_id].rand_len = len;
1176 task->rand[rand_id].rand_offset = offset;
1177 task->rand[rand_id].rand_mask = mask;
1178 task->rand[rand_id].fixed_bits = fixed;
1182 task->rand[task->n_rands].rand_len = len;
1183 task->rand[task->n_rands].rand_offset = offset;
1184 task->rand[task->n_rands].rand_mask = mask;
1185 task->rand[task->n_rands].fixed_bits = fixed;
1191 static void start(struct task_base *tbase)
1193 struct task_gen *task = (struct task_gen *)tbase;
1194 task->pkt_queue_index = 0;
1196 task_gen_reset_token_time(task);
1197 if (tbase->l3.tmaster) {
1198 register_all_ip_to_ctrl_plane(task);
1201 // task->port->link_speed reports the link speed in Mbps e.g. 40k for a 40 Gbps NIC.
1202 // task->link_speed reports link speed in Bytes per sec.
1203 #if RTE_VERSION < RTE_VERSION_NUM(16,4,0,0)
1204 // It can be 0 if link is down, and must hence be updated in fast path.
1205 task->link_speed = task->port->link_speed * 125000L;
1206 if (task->link_speed)
1207 plog_info("\tPort %u: link speed is %ld Mbps\n",
1208 (uint8_t)(task->port - prox_port_cfg), 8 * task->link_speed / 1000000);
1210 plog_info("\tPort %u: link speed is %ld Mbps - link might be down\n",
1211 (uint8_t)(task->port - prox_port_cfg), 8 * task->link_speed / 1000000);
1213 if (task->port->link_speed == UINT32_MAX)
1214 task->link_speed = UINT64_MAX;
1216 task->link_speed = task->port->link_speed * 125000L;
1217 plog_info("\tPort %u: link max speed is %ld Mbps\n",
1218 (uint8_t)(task->port - prox_port_cfg), 8 * task->link_speed / 1000000);
1223 Handle the case when two tasks transmit to the same port
1224 and one of them is stopped. In that case ARP (requests or replies)
1225 might not be sent. Master will have to keep a list of rings.
1226 stop will have to de-register IP from ctrl plane.
1227 un-registration will remove the ring. when having more than
1228 one active rings, master can always use the first one
1232 static void start_pcap(struct task_base *tbase)
1234 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1235 /* When we start, the first packet is sent immediately. */
1236 task->last_tsc = rte_rdtsc() - task->proto_tsc[0];
1240 static void init_task_gen_early(struct task_args *targ)
1242 uint8_t *generator_count = prox_sh_find_system("generator_count");
1244 if (generator_count == NULL) {
1245 generator_count = prox_zmalloc(sizeof(*generator_count), rte_lcore_to_socket_id(targ->lconf->id));
1246 PROX_PANIC(generator_count == NULL, "Failed to allocate generator count\n");
1247 prox_sh_add_system("generator_count", generator_count);
1249 targ->generator_id = *generator_count;
1250 (*generator_count)++;
1253 static void init_task_gen(struct task_base *tbase, struct task_args *targ)
1255 struct task_gen *task = (struct task_gen *)tbase;
1257 task->packet_id_pos = targ->packet_id_pos;
1259 struct prox_port_cfg *port = find_reachable_port(targ);
1260 // TODO: check that all reachable ports have the same mtu...
1262 task->cksum_offload = port->requested_tx_offload & (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM);
1264 task->max_frame_size = port->mtu + ETHER_HDR_LEN + 2 * PROX_VLAN_TAG_SIZE;
1266 // Not generating to any port...
1267 task->max_frame_size = ETHER_MAX_LEN;
1269 task->local_mbuf.mempool = task_gen_create_mempool(targ, task->max_frame_size);
1270 PROX_PANIC(task->local_mbuf.mempool == NULL, "Failed to create mempool\n");
1272 task->hz = rte_get_tsc_hz();
1273 task->lat_pos = targ->lat_pos;
1274 task->accur_pos = targ->accur_pos;
1275 task->sig_pos = targ->sig_pos;
1276 task->sig = targ->sig;
1277 task->new_rate_bps = targ->rate_bps;
1280 * For tokens, use 10 Gbps as base rate
1281 * Scripts can then use speed command, with speed=100 as 10 Gbps and speed=400 as 40 Gbps
1282 * Script can query prox "port info" command to find out the port link speed to know
1283 * at which rate to start. Note that virtio running on OVS returns 10 Gbps, so a script has
1284 * probably also to check the driver (as returned by the same "port info" command.
1286 struct token_time_cfg tt_cfg = token_time_cfg_create(1250000000, rte_get_tsc_hz(), -1);
1287 token_time_init(&task->token_time, &tt_cfg);
1289 init_task_gen_seeds(task);
1291 task->min_bulk_size = targ->min_bulk_size;
1292 task->max_bulk_size = targ->max_bulk_size;
1293 if (task->min_bulk_size < 1)
1294 task->min_bulk_size = 1;
1295 if (task->max_bulk_size < 1)
1296 task->max_bulk_size = 64;
1297 PROX_PANIC(task->max_bulk_size > 64, "max_bulk_size higher than 64\n");
1298 PROX_PANIC(task->max_bulk_size < task->min_bulk_size, "max_bulk_size must be > than min_bulk_size\n");
1300 task->pkt_count = -1;
1301 task->lat_enabled = targ->lat_enabled;
1302 task->runtime_flags = targ->runtime_flags;
1303 PROX_PANIC((task->lat_pos || task->accur_pos) && !task->lat_enabled, "lat not enabled by lat pos or accur pos configured\n");
1305 task->generator_id = targ->generator_id;
1306 plog_info("\tGenerator id = %d\n", task->generator_id);
1307 task->link_speed = UINT64_MAX;
1309 if (!strcmp(targ->pcap_file, "")) {
1310 plog_info("\tUsing inline definition of a packet\n");
1311 task_init_gen_load_pkt_inline(task, targ);
1313 plog_info("Loading from pcap %s\n", targ->pcap_file);
1314 task_init_gen_load_pcap(task, targ);
1317 PROX_PANIC(((targ->nb_txrings == 0) && (targ->nb_txports == 0)), "Gen mode requires a tx ring or a tx port");
1318 if ((targ->flags & DSF_KEEP_SRC_MAC) == 0) {
1319 uint8_t *src_addr = prox_port_cfg[tbase->tx_params_hw.tx_port_queue->port].eth_addr.addr_bytes;
1320 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1321 rte_memcpy(&task->pkt_template[i].buf[6], src_addr, 6);
1324 memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(struct ether_addr));
1325 for (uint32_t i = 0; i < targ->n_rand_str; ++i) {
1326 PROX_PANIC(task_gen_add_rand(tbase, targ->rand_str[i], targ->rand_offset[i], UINT32_MAX),
1327 "Failed to add random\n");
1331 static struct task_init task_init_gen = {
1333 .init = init_task_gen,
1334 .handle = handle_gen_bulk,
1336 .early_init = init_task_gen_early,
1338 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1339 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1340 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1342 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1344 .size = sizeof(struct task_gen)
1347 static struct task_init task_init_gen_l3 = {
1349 .sub_mode_str = "l3",
1350 .init = init_task_gen,
1351 .handle = handle_gen_bulk,
1353 .early_init = init_task_gen_early,
1355 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1356 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1357 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1359 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1361 .size = sizeof(struct task_gen)
1364 static struct task_init task_init_gen_pcap = {
1366 .sub_mode_str = "pcap",
1367 .init = init_task_gen_pcap,
1368 .handle = handle_gen_pcap_bulk,
1369 .start = start_pcap,
1370 .early_init = init_task_gen_early,
1372 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1374 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1376 .size = sizeof(struct task_gen_pcap)
1379 __attribute__((constructor)) static void reg_task_gen(void)
1381 reg_task(&task_init_gen);
1382 reg_task(&task_init_gen_l3);
1383 reg_task(&task_init_gen_pcap);