2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
20 #include <rte_cycles.h>
21 #include <rte_version.h>
22 #include <rte_byteorder.h>
23 #include <rte_ether.h>
24 #include <rte_hash_crc.h>
26 #include "prox_shared.h"
28 #include "prox_malloc.h"
29 #include "handle_gen.h"
30 #include "handle_lat.h"
31 #include "task_init.h"
32 #include "task_base.h"
33 #include "prox_port_cfg.h"
38 #include "mbuf_utils.h"
40 #include "prox_cksum.h"
42 #include "prox_assert.h"
44 #include "token_time.h"
45 #include "local_mbuf.h"
48 #include "handle_master.h"
57 #define MAX_TEMPLATE_INDEX 65536
58 #define TEMPLATE_INDEX_MASK (MAX_TEMPLATE_INDEX - 1)
60 #define IP4(x) x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, x >> 24
62 static void pkt_template_init_mbuf(struct pkt_template *pkt_template, struct rte_mbuf *mbuf, uint8_t *pkt)
64 const uint32_t pkt_size = pkt_template->len;
66 rte_pktmbuf_pkt_len(mbuf) = pkt_size;
67 rte_pktmbuf_data_len(mbuf) = pkt_size;
69 rte_memcpy(pkt, pkt_template->buf, pkt_template->len);
72 struct task_gen_pcap {
73 struct task_base base;
75 struct local_mbuf local_mbuf;
77 struct pkt_template *proto;
85 struct task_base base;
87 struct token_time token_time;
88 struct local_mbuf local_mbuf;
89 struct pkt_template *pkt_template; /* packet templates used at runtime */
90 uint64_t write_duration_estimate; /* how long it took previously to write the time stamps in the packets */
91 uint64_t earliest_tsc_next_pkt;
92 uint64_t new_rate_bps;
93 uint64_t pkt_queue_index;
94 uint32_t n_pkts; /* number of packets in pcap */
95 uint32_t pkt_idx; /* current packet from pcap */
96 uint32_t pkt_count; /* how many pakets to generate */
97 uint32_t max_frame_size;
98 uint32_t runtime_flags;
100 uint16_t packet_id_pos;
104 uint8_t generator_id;
105 uint8_t n_rands; /* number of randoms */
106 uint8_t min_bulk_size;
107 uint8_t max_bulk_size;
109 uint8_t runtime_checksum_needed;
112 uint32_t rand_mask; /* since the random vals are uniform, masks don't introduce bias */
113 uint32_t fixed_bits; /* length of each random (max len = 4) */
114 uint16_t rand_offset; /* each random has an offset*/
115 uint8_t rand_len; /* # bytes to take from random (no bias introduced) */
118 uint64_t pkt_tsc_offset[64];
119 struct pkt_template *pkt_template_orig; /* packet templates (from inline or from pcap) */
120 struct ether_addr src_mac;
122 uint8_t cksum_offload;
123 struct prox_port_cfg *port;
124 uint64_t *bytes_to_tsc;
125 } __rte_cache_aligned;
127 static inline uint8_t ipv4_get_hdr_len(struct ipv4_hdr *ip)
129 /* Optimize for common case of IPv4 header without options. */
130 if (ip->version_ihl == 0x45)
131 return sizeof(struct ipv4_hdr);
132 if (unlikely(ip->version_ihl >> 4 != 4)) {
133 plog_warn("IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
136 return (ip->version_ihl & 0xF) * 4;
139 static void parse_l2_l3_len(uint8_t *pkt, uint16_t *l2_len, uint16_t *l3_len, uint16_t len)
141 *l2_len = sizeof(struct ether_hdr);
143 struct vlan_hdr *vlan_hdr;
144 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
146 uint16_t ether_type = eth_hdr->ether_type;
149 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (*l2_len + sizeof(struct vlan_hdr) < len)) {
150 vlan_hdr = (struct vlan_hdr *)(pkt + *l2_len);
152 ether_type = vlan_hdr->eth_proto;
155 // No L3 cksum offload for IPv6, but TODO L4 offload
156 // ETYPE_EoGRE CRC not implemented yet
158 switch (ether_type) {
172 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
177 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + *l2_len);
178 *l3_len = ipv4_get_hdr_len(ip);
182 static void checksum_packet(uint8_t *hdr, struct rte_mbuf *mbuf, struct pkt_template *pkt_template, int cksum_offload)
184 uint16_t l2_len = pkt_template->l2_len;
185 uint16_t l3_len = pkt_template->l3_len;
188 struct ipv4_hdr *ip = (struct ipv4_hdr*)(hdr + l2_len);
189 prox_ip_udp_cksum(mbuf, ip, l2_len, l3_len, cksum_offload);
193 static void task_gen_reset_token_time(struct task_gen *task)
195 token_time_set_bpp(&task->token_time, task->new_rate_bps);
196 token_time_reset(&task->token_time, rte_rdtsc(), 0);
199 static void task_gen_take_count(struct task_gen *task, uint32_t send_bulk)
201 if (task->pkt_count == (uint32_t)-1)
204 if (task->pkt_count >= send_bulk)
205 task->pkt_count -= send_bulk;
211 static int handle_gen_pcap_bulk(struct task_base *tbase, struct rte_mbuf **mbuf, uint16_t n_pkts)
213 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
214 uint64_t now = rte_rdtsc();
215 uint64_t send_bulk = 0;
216 uint32_t pkt_idx_tmp = task->pkt_idx;
218 if (pkt_idx_tmp == task->n_pkts) {
219 PROX_ASSERT(task->loop);
223 for (uint16_t j = 0; j < 64; ++j) {
224 uint64_t tsc = task->proto_tsc[pkt_idx_tmp];
225 if (task->last_tsc + tsc <= now) {
226 task->last_tsc += tsc;
229 if (pkt_idx_tmp == task->n_pkts) {
240 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
241 if (new_pkts == NULL)
244 for (uint16_t j = 0; j < send_bulk; ++j) {
245 struct rte_mbuf *next_pkt = new_pkts[j];
246 struct pkt_template *pkt_template = &task->proto[task->pkt_idx];
247 uint8_t *hdr = rte_pktmbuf_mtod(next_pkt, uint8_t *);
249 pkt_template_init_mbuf(pkt_template, next_pkt, hdr);
252 if (task->pkt_idx == task->n_pkts) {
260 return task->base.tx_pkt(&task->base, new_pkts, send_bulk, NULL);
263 static inline uint64_t bytes_to_tsc(struct task_gen *task, uint32_t bytes)
265 return task->bytes_to_tsc[bytes];
268 static uint32_t task_gen_next_pkt_idx(const struct task_gen *task, uint32_t pkt_idx)
270 return pkt_idx + 1 == task->n_pkts? 0 : pkt_idx + 1;
273 static uint32_t task_gen_offset_pkt_idx(const struct task_gen *task, uint32_t offset)
275 return (task->pkt_idx + offset) % task->n_pkts;
278 static uint32_t task_gen_calc_send_bulk(const struct task_gen *task, uint32_t *total_bytes)
280 /* The biggest bulk we allow to send is task->max_bulk_size
281 packets. The max bulk size can also be limited by the
282 pkt_count field. At the same time, we are rate limiting
283 based on the specified speed (in bytes per second) so token
284 bucket based rate limiting must also be applied. The
285 minimum bulk size is also constrained. If the calculated
286 bulk size is less then the minimum, then don't send
289 const uint32_t min_bulk = task->min_bulk_size;
290 uint32_t max_bulk = task->max_bulk_size;
292 if (task->pkt_count != (uint32_t)-1 && task->pkt_count < max_bulk) {
293 max_bulk = task->pkt_count;
296 uint32_t send_bulk = 0;
297 uint32_t pkt_idx_tmp = task->pkt_idx;
298 uint32_t would_send_bytes = 0;
302 * TODO - this must be improved to take into account the fact that, after applying randoms
303 * The packet can be replaced by an ARP
305 for (uint16_t j = 0; j < max_bulk; ++j) {
306 struct pkt_template *pktpl = &task->pkt_template[pkt_idx_tmp];
307 pkt_size = pktpl->len;
308 uint32_t pkt_len = pkt_len_to_wire_size(pkt_size);
309 if (pkt_len + would_send_bytes > task->token_time.bytes_now)
312 pkt_idx_tmp = task_gen_next_pkt_idx(task, pkt_idx_tmp);
315 would_send_bytes += pkt_len;
318 if (send_bulk < min_bulk)
320 *total_bytes = would_send_bytes;
324 static void task_gen_apply_random_fields(struct task_gen *task, uint8_t *hdr)
326 uint32_t ret, ret_tmp;
328 for (uint16_t i = 0; i < task->n_rands; ++i) {
329 ret = random_next(&task->rand[i].state);
330 ret_tmp = (ret & task->rand[i].rand_mask) | task->rand[i].fixed_bits;
332 ret_tmp = rte_bswap32(ret_tmp);
333 /* At this point, the lower order bytes (BE) contain
334 the generated value. The address where the values
335 of interest starts is at ret_tmp + 4 - rand_len. */
336 uint8_t *pret_tmp = (uint8_t*)&ret_tmp;
337 rte_memcpy(hdr + task->rand[i].rand_offset, pret_tmp + 4 - task->rand[i].rand_len, task->rand[i].rand_len);
341 static void task_gen_apply_all_random_fields(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
346 for (uint16_t i = 0; i < count; ++i)
347 task_gen_apply_random_fields(task, pkt_hdr[i]);
350 static void task_gen_apply_accur_pos(struct task_gen *task, uint8_t *pkt_hdr, uint32_t accuracy)
352 *(uint32_t *)(pkt_hdr + task->accur_pos) = accuracy;
355 static void task_gen_apply_sig(struct task_gen *task, struct pkt_template *dst)
358 *(uint32_t *)(dst->buf + task->sig_pos) = task->sig;
361 static void task_gen_apply_all_accur_pos(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
363 if (!task->accur_pos)
366 /* The accuracy of task->pkt_queue_index - 64 is stored in
367 packet task->pkt_queue_index. The ID modulo 64 is the
369 for (uint16_t j = 0; j < count; ++j) {
370 uint32_t accuracy = task->accur[(task->pkt_queue_index + j) & 63];
371 task_gen_apply_accur_pos(task, pkt_hdr[j], accuracy);
375 static void task_gen_apply_unique_id(struct task_gen *task, uint8_t *pkt_hdr, const struct unique_id *id)
377 struct unique_id *dst = (struct unique_id *)(pkt_hdr + task->packet_id_pos);
382 static void task_gen_apply_all_unique_id(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
384 if (!task->packet_id_pos)
387 for (uint16_t i = 0; i < count; ++i) {
389 unique_id_init(&id, task->generator_id, task->pkt_queue_index++);
390 task_gen_apply_unique_id(task, pkt_hdr[i], &id);
394 static void task_gen_checksum_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
396 if (!(task->runtime_flags & TASK_TX_CRC))
399 if (!task->runtime_checksum_needed)
402 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - count);
403 for (uint16_t i = 0; i < count; ++i) {
404 struct pkt_template *pkt_template = &task->pkt_template[pkt_idx];
405 checksum_packet(pkt_hdr[i], mbufs[i], pkt_template, task->cksum_offload);
406 pkt_idx = task_gen_next_pkt_idx(task, pkt_idx);
410 static void task_gen_consume_tokens(struct task_gen *task, uint32_t tokens, uint32_t send_count)
412 /* If max burst has been sent, we can't keep up so just assume
413 that we can (leaving a "gap" in the packet stream on the
415 task->token_time.bytes_now -= tokens;
416 if (send_count == task->max_bulk_size && task->token_time.bytes_now > tokens) {
417 task->token_time.bytes_now = tokens;
421 static uint64_t task_gen_calc_bulk_duration(struct task_gen *task, uint32_t count)
423 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - 1);
424 struct pkt_template *last_pkt_template = &task->pkt_template[pkt_idx];
425 uint32_t last_pkt_len = pkt_len_to_wire_size(last_pkt_template->len);
426 #ifdef NO_EXTRAPOLATION
427 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1];
429 uint64_t last_pkt_duration = bytes_to_tsc(task, last_pkt_len);
430 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1] + last_pkt_duration;
433 return bulk_duration;
436 static uint64_t task_gen_write_latency(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
438 if (!task->lat_enabled)
441 uint64_t tx_tsc, delta_t;
442 uint64_t tsc_before_tx = 0;
444 /* Just before sending the packets, apply the time stamp
445 relative to when the first packet will be sent. The first
446 packet will be sent now. The time is read for each packet
447 to reduce the error towards the actual time the packet will
449 uint64_t write_tsc_after, write_tsc_before;
451 write_tsc_before = rte_rdtsc();
453 /* The time it took previously to write the time stamps in the
454 packets is used as an estimate for how long it will take to
455 write the time stamps now. The estimated time at which the
456 packets will actually be sent will be at tx_tsc. */
457 tx_tsc = write_tsc_before + task->write_duration_estimate;
459 /* The offset delta_t tracks the difference between the actual
460 time and the time written in the packets. Adding the offset
461 to the actual time insures that the time written in the
462 packets is monotonically increasing. At the same time,
463 simply sleeping until delta_t is zero would leave a period
464 of silence on the line. The error has been introduced
465 earlier, but the packets have already been sent. */
467 /* This happens typically if previous bulk was delayed
468 by an interrupt e.g. (with Time in nsec)
469 Time x: sleep 4 microsec
470 Time x+4000: send 64 packets (64 packets as 4000 nsec, w/ 10Gbps 64 bytes)
471 Time x+5000: send 16 packets (16 packets as 1000 nsec)
472 When we send the 16 packets, the 64 ealier packets are not yet
474 if (tx_tsc < task->earliest_tsc_next_pkt)
475 delta_t = task->earliest_tsc_next_pkt - tx_tsc;
479 for (uint16_t i = 0; i < count; ++i) {
480 uint32_t *pos = (uint32_t *)(pkt_hdr[i] + task->lat_pos);
481 const uint64_t pkt_tsc = tx_tsc + delta_t + task->pkt_tsc_offset[i];
482 *pos = pkt_tsc >> LATENCY_ACCURACY;
485 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, count);
486 task->earliest_tsc_next_pkt = tx_tsc + delta_t + bulk_duration;
487 write_tsc_after = rte_rdtsc();
488 task->write_duration_estimate = write_tsc_after - write_tsc_before;
490 /* Make sure that the time stamps that were written
491 are valid. The offset must be taken into account */
493 tsc_before_tx = rte_rdtsc();
494 } while (tsc_before_tx < tx_tsc);
496 return tsc_before_tx;
499 static void task_gen_store_accuracy(struct task_gen *task, uint32_t count, uint64_t tsc_before_tx)
501 if (!task->accur_pos)
504 uint64_t accur = rte_rdtsc() - tsc_before_tx;
505 uint64_t first_accuracy_idx = task->pkt_queue_index - count;
507 for (uint32_t i = 0; i < count; ++i) {
508 uint32_t accuracy_idx = (first_accuracy_idx + i) & 63;
510 task->accur[accuracy_idx] = accur;
514 static void task_gen_load_and_prefetch(struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
516 for (uint16_t i = 0; i < count; ++i)
517 rte_prefetch0(mbufs[i]);
518 for (uint16_t i = 0; i < count; ++i)
519 pkt_hdr[i] = rte_pktmbuf_mtod(mbufs[i], uint8_t *);
520 for (uint16_t i = 0; i < count; ++i)
521 rte_prefetch0(pkt_hdr[i]);
524 static void task_gen_build_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
526 uint64_t will_send_bytes = 0;
528 for (uint16_t i = 0; i < count; ++i) {
529 struct pkt_template *pktpl = &task->pkt_template[task->pkt_idx];
530 struct pkt_template *pkt_template = &task->pkt_template[task->pkt_idx];
531 pkt_template_init_mbuf(pkt_template, mbufs[i], pkt_hdr[i]);
532 mbufs[i]->udata64 = task->pkt_idx & TEMPLATE_INDEX_MASK;
533 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
534 if (task->lat_enabled) {
535 #ifdef NO_EXTRAPOLATION
536 task->pkt_tsc_offset[i] = 0;
538 task->pkt_tsc_offset[i] = bytes_to_tsc(task, will_send_bytes);
540 will_send_bytes += pkt_len_to_wire_size(pkt_template->len);
542 task->pkt_idx = task_gen_next_pkt_idx(task, task->pkt_idx);
546 static void task_gen_update_config(struct task_gen *task)
548 if (task->token_time.cfg.bpp != task->new_rate_bps)
549 task_gen_reset_token_time(task);
552 static inline void build_value(struct task_gen *task, uint32_t mask, int bit_pos, uint32_t val, uint32_t fixed_bits)
554 struct task_base *tbase = (struct task_base *)task;
556 build_value(task, mask >> 1, bit_pos + 1, val, fixed_bits);
558 build_value(task, mask >> 1, bit_pos + 1, val | (1 << bit_pos), fixed_bits);
561 register_ip_to_ctrl_plane(tbase->l3.tmaster, rte_cpu_to_be_32(val | fixed_bits), tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
564 static inline void register_all_ip_to_ctrl_plane(struct task_gen *task)
566 struct task_base *tbase = (struct task_base *)task;
571 for (uint32_t i = 0; i < task->n_pkts; ++i) {
572 struct pkt_template *pktpl = &task->pkt_template[i];
573 unsigned int ip_src_pos = 0;
575 unsigned int l2_len = sizeof(struct ether_hdr);
577 uint8_t *pkt = pktpl->buf;
578 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
579 uint16_t ether_type = eth_hdr->ether_type;
580 struct vlan_hdr *vlan_hdr;
583 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(struct vlan_hdr) < pktpl->len)) {
584 vlan_hdr = (struct vlan_hdr *)(pkt + l2_len);
586 ether_type = vlan_hdr->eth_proto;
588 if ((ether_type == ETYPE_MPLSU) || (ether_type == ETYPE_MPLSM)) {
592 if ((ether_type != ETYPE_IPv4) && !maybe_ipv4)
595 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + l2_len);
596 PROX_PANIC(ip->version_ihl >> 4 != 4, "IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
598 // Even if IPv4 header contains options, options are after ip src and dst
599 ip_src_pos = l2_len + sizeof(struct ipv4_hdr) - 2 * sizeof(uint32_t);
600 uint32_t *ip_src = ((uint32_t *)(pktpl->buf + ip_src_pos));
601 plog_info("\tip_src_pos = %d, ip_src = %x\n", ip_src_pos, *ip_src);
602 register_ip_to_ctrl_plane(tbase->l3.tmaster, *ip_src, tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
604 for (int j = 0; j < task->n_rands; j++) {
605 offset = task->rand[j].rand_offset;
606 len = task->rand[j].rand_len;
607 mask = task->rand[j].rand_mask;
608 fixed = task->rand[j].fixed_bits;
609 plog_info("offset = %d, len = %d, mask = %x, fixed = %x\n", offset, len, mask, fixed);
610 if ((offset < ip_src_pos + 4) && (offset + len >= ip_src_pos)) {
611 if (offset >= ip_src_pos) {
612 int32_t ip_src_mask = (1 << (4 + ip_src_pos - offset) * 8) - 1;
613 mask = mask & ip_src_mask;
614 fixed = (fixed & ip_src_mask) | (rte_be_to_cpu_32(*ip_src) & ~ip_src_mask);
615 build_value(task, mask, 0, 0, fixed);
617 int32_t bits = ((ip_src_pos + 4 - offset - len) * 8);
619 fixed = (fixed << bits) | (rte_be_to_cpu_32(*ip_src) & ((1 << bits) - 1));
620 build_value(task, mask, 0, 0, fixed);
627 static int handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
629 struct task_gen *task = (struct task_gen *)tbase;
630 uint8_t out[MAX_PKT_BURST] = {0};
635 task_gen_update_config(task);
637 if (task->pkt_count == 0) {
638 task_gen_reset_token_time(task);
641 if (!task->token_time.cfg.bpp)
644 token_time_update(&task->token_time, rte_rdtsc());
646 uint32_t would_send_bytes;
647 uint32_t send_bulk = task_gen_calc_send_bulk(task, &would_send_bytes);
651 task_gen_take_count(task, send_bulk);
652 task_gen_consume_tokens(task, would_send_bytes, send_bulk);
654 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
655 if (new_pkts == NULL)
657 uint8_t *pkt_hdr[MAX_RING_BURST];
659 task_gen_load_and_prefetch(new_pkts, pkt_hdr, send_bulk);
660 task_gen_build_packets(task, new_pkts, pkt_hdr, send_bulk);
661 task_gen_apply_all_random_fields(task, pkt_hdr, send_bulk);
662 task_gen_apply_all_accur_pos(task, new_pkts, pkt_hdr, send_bulk);
663 task_gen_apply_all_unique_id(task, new_pkts, pkt_hdr, send_bulk);
665 uint64_t tsc_before_tx;
667 tsc_before_tx = task_gen_write_latency(task, pkt_hdr, send_bulk);
668 task_gen_checksum_packets(task, new_pkts, pkt_hdr, send_bulk);
669 ret = task->base.tx_pkt(&task->base, new_pkts, send_bulk, out);
670 task_gen_store_accuracy(task, send_bulk, tsc_before_tx);
672 // If we failed to send some packets, we need to do some clean-up:
675 // We need re-use the packets indexes not being sent
676 // Hence non-sent packets will not be considered as lost by the receiver when it looks at
677 // packet ids. This should also increase the percentage of packets used for latency measurements
678 task->pkt_queue_index -= ret;
680 // In case of failures, the estimate about when we can send next packet (earliest_tsc_next_pkt) is wrong
681 // This would result in under-estimated latency (up to 0 or negative)
682 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, ret);
683 task->earliest_tsc_next_pkt -= bulk_duration;
688 static void init_task_gen_seeds(struct task_gen *task)
690 for (size_t i = 0; i < sizeof(task->rand)/sizeof(task->rand[0]); ++i)
691 random_init_seed(&task->rand[i].state);
694 static uint32_t pcap_count_pkts(pcap_t *handle, uint32_t *max_frame_size)
696 struct pcap_pkthdr header;
700 long pkt1_fpos = ftell(pcap_file(handle));
702 while ((buf = pcap_next(handle, &header))) {
703 if (header.len > *max_frame_size)
704 *max_frame_size = header.len;
707 int ret2 = fseek(pcap_file(handle), pkt1_fpos, SEEK_SET);
708 PROX_PANIC(ret2 != 0, "Failed to reset reading pcap file\n");
712 static uint64_t avg_time_stamp(uint64_t *time_stamp, uint32_t n)
714 uint64_t tot_inter_pkt = 0;
716 for (uint32_t i = 0; i < n; ++i)
717 tot_inter_pkt += time_stamp[i];
718 return (tot_inter_pkt + n / 2)/n;
721 static int pcap_read_pkts(pcap_t *handle, const char *file_name, uint32_t n_pkts, struct pkt_template *proto, uint64_t *time_stamp)
723 struct pcap_pkthdr header;
727 for (uint32_t i = 0; i < n_pkts; ++i) {
728 buf = pcap_next(handle, &header);
730 PROX_PANIC(buf == NULL, "Failed to read packet %d from pcap %s\n", i, file_name);
731 proto[i].len = header.len;
732 len = RTE_MIN(header.len, sizeof(proto[i].buf));
733 if (header.len > len)
734 plogx_warn("Packet truncated from %u to %zu bytes\n", header.len, len);
737 static struct timeval beg;
743 tv = tv_diff(&beg, &header.ts);
744 tv_to_tsc(&tv, time_stamp + i);
746 rte_memcpy(proto[i].buf, buf, len);
749 if (time_stamp && n_pkts) {
750 for (uint32_t i = n_pkts - 1; i > 0; --i)
751 time_stamp[i] -= time_stamp[i - 1];
752 /* Since the handle function will loop the packets,
753 there is one time-stamp that is not provided by the
754 pcap file. This is the time between the last and
755 the first packet. This implementation takes the
756 average of the inter-packet times here. */
758 time_stamp[0] = avg_time_stamp(time_stamp + 1, n_pkts - 1);
764 static int check_pkt_size(struct task_gen *task, uint32_t pkt_size, int do_panic)
766 const uint16_t min_len = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr);
767 const uint16_t max_len = task->max_frame_size;
770 PROX_PANIC(pkt_size == 0, "Invalid packet size length (no packet defined?)\n");
771 PROX_PANIC(pkt_size > max_len, "pkt_size out of range (must be <= %u)\n", max_len);
772 PROX_PANIC(pkt_size < min_len, "pkt_size out of range (must be >= %u)\n", min_len);
776 plog_err("Invalid packet size length (no packet defined?)\n");
779 if (pkt_size > max_len) {
780 plog_err("pkt_size out of range (must be <= %u)\n", max_len);
783 if (pkt_size < min_len) {
784 plog_err("pkt_size out of range (must be >= %u)\n", min_len);
791 static int check_all_pkt_size(struct task_gen *task, int do_panic)
794 for (uint32_t i = 0; i < task->n_pkts;++i) {
795 if ((rc = check_pkt_size(task, task->pkt_template[i].len, do_panic)) != 0)
801 static int check_fields_in_bounds(struct task_gen *task, uint32_t pkt_size, int do_panic)
803 if (task->lat_enabled) {
804 uint32_t pos_beg = task->lat_pos;
805 uint32_t pos_end = task->lat_pos + 3U;
808 PROX_PANIC(pkt_size <= pos_end, "Writing latency at %u-%u, but packet size is %u bytes\n",
809 pos_beg, pos_end, pkt_size);
810 else if (pkt_size <= pos_end) {
811 plog_err("Writing latency at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
815 if (task->packet_id_pos) {
816 uint32_t pos_beg = task->packet_id_pos;
817 uint32_t pos_end = task->packet_id_pos + 4U;
820 PROX_PANIC(pkt_size <= pos_end, "Writing packet at %u-%u, but packet size is %u bytes\n",
821 pos_beg, pos_end, pkt_size);
822 else if (pkt_size <= pos_end) {
823 plog_err("Writing packet at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
827 if (task->accur_pos) {
828 uint32_t pos_beg = task->accur_pos;
829 uint32_t pos_end = task->accur_pos + 3U;
832 PROX_PANIC(pkt_size <= pos_end, "Writing accuracy at %u%-u, but packet size is %u bytes\n",
833 pos_beg, pos_end, pkt_size);
834 else if (pkt_size <= pos_end) {
835 plog_err("Writing accuracy at %u%-u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
842 static void task_gen_pkt_template_recalc_metadata(struct task_gen *task)
844 struct pkt_template *template;
846 for (size_t i = 0; i < task->n_pkts; ++i) {
847 template = &task->pkt_template[i];
848 parse_l2_l3_len(template->buf, &template->l2_len, &template->l3_len, template->len);
852 static void task_gen_pkt_template_recalc_checksum(struct task_gen *task)
854 struct pkt_template *template;
857 task->runtime_checksum_needed = 0;
858 for (size_t i = 0; i < task->n_pkts; ++i) {
859 template = &task->pkt_template[i];
860 if (template->l2_len == 0)
862 ip = (struct ipv4_hdr *)(template->buf + template->l2_len);
864 ip->hdr_checksum = 0;
865 prox_ip_cksum_sw(ip);
866 uint32_t l4_len = rte_bswap16(ip->total_length) - template->l3_len;
868 if (ip->next_proto_id == IPPROTO_UDP) {
869 struct udp_hdr *udp = (struct udp_hdr *)(((uint8_t *)ip) + template->l3_len);
870 prox_udp_cksum_sw(udp, l4_len, ip->src_addr, ip->dst_addr);
871 } else if (ip->next_proto_id == IPPROTO_TCP) {
872 struct tcp_hdr *tcp = (struct tcp_hdr *)(((uint8_t *)ip) + template->l3_len);
873 prox_tcp_cksum_sw(tcp, l4_len, ip->src_addr, ip->dst_addr);
876 /* The current implementation avoids checksum
877 calculation by determining that at packet
878 construction time, no fields are applied that would
879 require a recalculation of the checksum. */
880 if (task->lat_enabled && task->lat_pos > template->l2_len)
881 task->runtime_checksum_needed = 1;
882 if (task->accur_pos > template->l2_len)
883 task->runtime_checksum_needed = 1;
884 if (task->packet_id_pos > template->l2_len)
885 task->runtime_checksum_needed = 1;
889 static void task_gen_pkt_template_recalc_all(struct task_gen *task)
891 task_gen_pkt_template_recalc_metadata(task);
892 task_gen_pkt_template_recalc_checksum(task);
895 static void task_gen_reset_pkt_templates_len(struct task_gen *task)
897 struct pkt_template *src, *dst;
899 for (size_t i = 0; i < task->n_pkts; ++i) {
900 src = &task->pkt_template_orig[i];
901 dst = &task->pkt_template[i];
906 static void task_gen_reset_pkt_templates_content(struct task_gen *task)
908 struct pkt_template *src, *dst;
910 for (size_t i = 0; i < task->n_pkts; ++i) {
911 src = &task->pkt_template_orig[i];
912 dst = &task->pkt_template[i];
913 memcpy(dst->buf, src->buf, dst->len);
914 task_gen_apply_sig(task, dst);
918 static void task_gen_reset_pkt_templates(struct task_gen *task)
920 task_gen_reset_pkt_templates_len(task);
921 task_gen_reset_pkt_templates_content(task);
922 task_gen_pkt_template_recalc_all(task);
925 static void task_init_gen_load_pkt_inline(struct task_gen *task, struct task_args *targ)
927 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
931 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
932 task->pkt_template = prox_zmalloc(mem_size, socket_id);
933 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
935 PROX_PANIC(task->pkt_template == NULL ||
936 task->pkt_template_orig == NULL,
937 "Failed to allocate %lu bytes (in huge pages) for packet template\n", mem_size);
939 task->pkt_template->buf = prox_zmalloc(task->max_frame_size, socket_id);
940 task->pkt_template_orig->buf = prox_zmalloc(task->max_frame_size, socket_id);
941 PROX_PANIC(task->pkt_template->buf == NULL ||
942 task->pkt_template_orig->buf == NULL,
943 "Failed to allocate %u bytes (in huge pages) for packet\n", task->max_frame_size);
945 PROX_PANIC(targ->pkt_size > task->max_frame_size,
946 targ->pkt_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE - 4 ?
947 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
949 rte_memcpy(task->pkt_template_orig[0].buf, targ->pkt_inline, targ->pkt_size);
950 task->pkt_template_orig[0].len = targ->pkt_size;
951 task_gen_reset_pkt_templates(task);
952 check_all_pkt_size(task, 1);
953 check_fields_in_bounds(task, task->pkt_template[0].len, 1);
956 static void task_init_gen_load_pcap(struct task_gen *task, struct task_args *targ)
958 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
959 char err[PCAP_ERRBUF_SIZE];
960 uint32_t max_frame_size;
961 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
962 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
964 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
965 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
966 PROX_PANIC(max_frame_size > task->max_frame_size,
967 max_frame_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE -4 ?
968 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
971 task->n_pkts = RTE_MIN(task->n_pkts, targ->n_pkts);
972 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
973 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
974 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
975 task->pkt_template = prox_zmalloc(mem_size, socket_id);
976 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
977 PROX_PANIC(task->pkt_template == NULL ||
978 task->pkt_template_orig == NULL,
979 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
981 for (uint i = 0; i < task->n_pkts; i++) {
982 task->pkt_template[i].buf = prox_zmalloc(max_frame_size, socket_id);
983 task->pkt_template_orig[i].buf = prox_zmalloc(max_frame_size, socket_id);
985 PROX_PANIC(task->pkt_template->buf == NULL ||
986 task->pkt_template_orig->buf == NULL,
987 "Failed to allocate %u bytes (in huge pages) for pcap file\n", task->max_frame_size);
990 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->pkt_template_orig, NULL);
992 task_gen_reset_pkt_templates(task);
995 static struct rte_mempool *task_gen_create_mempool(struct task_args *targ, uint16_t max_frame_size)
997 static char name[] = "gen_pool";
998 struct rte_mempool *ret;
999 const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
1002 uint32_t mbuf_size = TX_MBUF_SIZE;
1003 if (max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM > mbuf_size)
1004 mbuf_size = max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
1005 plog_info("\t\tCreating mempool with name '%s'\n", name);
1006 ret = rte_mempool_create(name, targ->nb_mbuf - 1, mbuf_size,
1007 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
1008 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
1010 PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
1011 sock_id, targ->nb_mbuf - 1);
1013 plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", ret,
1014 targ->nb_mbuf - 1, mbuf_size, targ->nb_cache_mbuf, sock_id);
1019 void task_gen_set_pkt_count(struct task_base *tbase, uint32_t count)
1021 struct task_gen *task = (struct task_gen *)tbase;
1023 task->pkt_count = count;
1026 int task_gen_set_pkt_size(struct task_base *tbase, uint32_t pkt_size)
1028 struct task_gen *task = (struct task_gen *)tbase;
1031 if ((rc = check_pkt_size(task, pkt_size, 0)) != 0)
1033 if ((rc = check_fields_in_bounds(task, pkt_size, 0)) != 0)
1035 task->pkt_template[0].len = pkt_size;
1039 void task_gen_set_rate(struct task_base *tbase, uint64_t bps)
1041 struct task_gen *task = (struct task_gen *)tbase;
1043 task->new_rate_bps = bps;
1046 void task_gen_reset_randoms(struct task_base *tbase)
1048 struct task_gen *task = (struct task_gen *)tbase;
1050 for (uint32_t i = 0; i < task->n_rands; ++i) {
1051 task->rand[i].rand_mask = 0;
1052 task->rand[i].fixed_bits = 0;
1053 task->rand[i].rand_offset = 0;
1058 int task_gen_set_value(struct task_base *tbase, uint32_t value, uint32_t offset, uint32_t len)
1060 struct task_gen *task = (struct task_gen *)tbase;
1062 for (size_t i = 0; i < task->n_pkts; ++i) {
1063 uint32_t to_write = rte_cpu_to_be_32(value) >> ((4 - len) * 8);
1064 uint8_t *dst = task->pkt_template[i].buf;
1066 rte_memcpy(dst + offset, &to_write, len);
1069 task_gen_pkt_template_recalc_all(task);
1074 void task_gen_reset_values(struct task_base *tbase)
1076 struct task_gen *task = (struct task_gen *)tbase;
1078 task_gen_reset_pkt_templates_content(task);
1081 uint32_t task_gen_get_n_randoms(struct task_base *tbase)
1083 struct task_gen *task = (struct task_gen *)tbase;
1085 return task->n_rands;
1088 static void init_task_gen_pcap(struct task_base *tbase, struct task_args *targ)
1090 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1091 const uint32_t sockid = rte_lcore_to_socket_id(targ->lconf->id);
1092 uint32_t max_frame_size;
1094 task->loop = targ->loop;
1096 task->hz = rte_get_tsc_hz();
1098 char err[PCAP_ERRBUF_SIZE];
1099 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1100 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1102 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
1103 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1105 task->local_mbuf.mempool = task_gen_create_mempool(targ, max_frame_size);
1107 PROX_PANIC(!strcmp(targ->pcap_file, ""), "No pcap file defined\n");
1110 plogx_info("Configured to load %u packets\n", targ->n_pkts);
1111 if (task->n_pkts > targ->n_pkts)
1112 task->n_pkts = targ->n_pkts;
1114 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
1116 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1118 size_t mem_size = task->n_pkts * (sizeof(*task->proto) + sizeof(*task->proto_tsc));
1119 uint8_t *mem = prox_zmalloc(mem_size, sockid);
1121 PROX_PANIC(mem == NULL, "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1122 task->proto = (struct pkt_template *) mem;
1123 task->proto_tsc = (uint64_t *)(mem + task->n_pkts * sizeof(*task->proto));
1125 for (uint i = 0; i < targ->n_pkts; i++) {
1126 task->proto[i].buf = prox_zmalloc(max_frame_size, sockid);
1127 PROX_PANIC(task->proto[i].buf == NULL, "Failed to allocate %u bytes (in huge pages) for pcap file\n", max_frame_size);
1130 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->proto, task->proto_tsc);
1134 static int task_gen_find_random_with_offset(struct task_gen *task, uint32_t offset)
1136 for (uint32_t i = 0; i < task->n_rands; ++i) {
1137 if (task->rand[i].rand_offset == offset) {
1145 int task_gen_add_rand(struct task_base *tbase, const char *rand_str, uint32_t offset, uint32_t rand_id)
1147 struct task_gen *task = (struct task_gen *)tbase;
1148 uint32_t existing_rand;
1150 if (rand_id == UINT32_MAX && task->n_rands == 64) {
1151 plog_err("Too many randoms\n");
1154 uint32_t mask, fixed, len;
1156 if (parse_random_str(&mask, &fixed, &len, rand_str)) {
1157 plog_err("%s\n", get_parse_err());
1160 task->runtime_checksum_needed = 1;
1162 existing_rand = task_gen_find_random_with_offset(task, offset);
1163 if (existing_rand != UINT32_MAX) {
1164 plog_warn("Random at offset %d already set => overwriting len = %d %s\n", offset, len, rand_str);
1165 rand_id = existing_rand;
1166 task->rand[rand_id].rand_len = len;
1167 task->rand[rand_id].rand_offset = offset;
1168 task->rand[rand_id].rand_mask = mask;
1169 task->rand[rand_id].fixed_bits = fixed;
1173 task->rand[task->n_rands].rand_len = len;
1174 task->rand[task->n_rands].rand_offset = offset;
1175 task->rand[task->n_rands].rand_mask = mask;
1176 task->rand[task->n_rands].fixed_bits = fixed;
1182 static void start(struct task_base *tbase)
1184 struct task_gen *task = (struct task_gen *)tbase;
1185 task->pkt_queue_index = 0;
1187 task_gen_reset_token_time(task);
1188 if (tbase->l3.tmaster) {
1189 register_all_ip_to_ctrl_plane(task);
1193 Handle the case when two tasks transmit to the same port
1194 and one of them is stopped. In that case ARP (requests or replies)
1195 might not be sent. Master will have to keep a list of rings.
1196 stop will have to de-register IP from ctrl plane.
1197 un-registration will remove the ring. when having more than
1198 one active rings, master can always use the first one
1202 static void start_pcap(struct task_base *tbase)
1204 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1205 /* When we start, the first packet is sent immediately. */
1206 task->last_tsc = rte_rdtsc() - task->proto_tsc[0];
1210 static void init_task_gen_early(struct task_args *targ)
1212 uint8_t *generator_count = prox_sh_find_system("generator_count");
1214 if (generator_count == NULL) {
1215 generator_count = prox_zmalloc(sizeof(*generator_count), rte_lcore_to_socket_id(targ->lconf->id));
1216 PROX_PANIC(generator_count == NULL, "Failed to allocate generator count\n");
1217 prox_sh_add_system("generator_count", generator_count);
1219 targ->generator_id = *generator_count;
1220 (*generator_count)++;
1223 static void init_task_gen(struct task_base *tbase, struct task_args *targ)
1225 struct task_gen *task = (struct task_gen *)tbase;
1227 task->packet_id_pos = targ->packet_id_pos;
1229 struct prox_port_cfg *port = find_reachable_port(targ);
1230 // TODO: check that all reachable ports have the same mtu...
1232 task->cksum_offload = port->requested_tx_offload & (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM);
1234 task->max_frame_size = port->mtu + ETHER_HDR_LEN + 2 * PROX_VLAN_TAG_SIZE;
1236 // Not generating to any port...
1237 task->max_frame_size = ETHER_MAX_LEN;
1239 task->local_mbuf.mempool = task_gen_create_mempool(targ, task->max_frame_size);
1240 PROX_PANIC(task->local_mbuf.mempool == NULL, "Failed to create mempool\n");
1242 task->hz = rte_get_tsc_hz();
1243 task->lat_pos = targ->lat_pos;
1244 task->accur_pos = targ->accur_pos;
1245 task->sig_pos = targ->sig_pos;
1246 task->sig = targ->sig;
1247 task->new_rate_bps = targ->rate_bps;
1250 * For tokens, use 10 Gbps as base rate
1251 * Scripts can then use speed command, with speed=100 as 10 Gbps and speed=400 as 40 Gbps
1252 * Script can query prox "port info" command to find out the port link speed to know
1253 * at which rate to start. Note that virtio running on OVS returns 10 Gbps, so a script has
1254 * probably also to check the driver (as returned by the same "port info" command.
1256 struct token_time_cfg tt_cfg = token_time_cfg_create(1250000000, rte_get_tsc_hz(), -1);
1257 token_time_init(&task->token_time, &tt_cfg);
1259 init_task_gen_seeds(task);
1261 task->min_bulk_size = targ->min_bulk_size;
1262 task->max_bulk_size = targ->max_bulk_size;
1263 if (task->min_bulk_size < 1)
1264 task->min_bulk_size = 1;
1265 if (task->max_bulk_size < 1)
1266 task->max_bulk_size = 64;
1267 PROX_PANIC(task->max_bulk_size > 64, "max_bulk_size higher than 64\n");
1268 PROX_PANIC(task->max_bulk_size < task->min_bulk_size, "max_bulk_size must be > than min_bulk_size\n");
1270 task->pkt_count = -1;
1271 task->lat_enabled = targ->lat_enabled;
1272 task->runtime_flags = targ->runtime_flags;
1273 PROX_PANIC((task->lat_pos || task->accur_pos) && !task->lat_enabled, "lat not enabled by lat pos or accur pos configured\n");
1275 task->generator_id = targ->generator_id;
1276 plog_info("\tGenerator id = %d\n", task->generator_id);
1278 // Allocate array holding bytes to tsc for supported frame sizes
1279 task->bytes_to_tsc = prox_zmalloc(task->max_frame_size * MAX_PKT_BURST * sizeof(task->bytes_to_tsc[0]), rte_lcore_to_socket_id(targ->lconf->id));
1280 PROX_PANIC(task->bytes_to_tsc == NULL,
1281 "Failed to allocate %u bytes (in huge pages) for bytes_to_tsc\n", task->max_frame_size);
1283 // task->port->max_link_speed reports the maximum, non negotiated ink speed in Mbps e.g. 40k for a 40 Gbps NIC.
1284 // It can be UINT32_MAX (virtual devices or not supported by DPDK < 16.04)
1285 uint64_t bytes_per_hz = UINT64_MAX;
1286 if ((task->port) && (task->port->max_link_speed != UINT32_MAX)) {
1287 bytes_per_hz = task->port->max_link_speed * 125000L;
1288 plog_info("\tPort %u: max link speed is %ld Mbps\n",
1289 (uint8_t)(task->port - prox_port_cfg), 8 * bytes_per_hz / 1000000);
1291 // There are cases where hz estimate might be slighly over-estimated
1292 // This results in too much extrapolation
1293 // Only account for 99% of extrapolation to handle cases with up to 1% error clocks
1294 for (unsigned int i = 0; i < task->max_frame_size * MAX_PKT_BURST ; i++) {
1295 if (bytes_per_hz == UINT64_MAX)
1296 task->bytes_to_tsc[i] = 0;
1298 task->bytes_to_tsc[i] = (task->hz * i * 0.99) / bytes_per_hz;
1301 if (!strcmp(targ->pcap_file, "")) {
1302 plog_info("\tUsing inline definition of a packet\n");
1303 task_init_gen_load_pkt_inline(task, targ);
1305 plog_info("Loading from pcap %s\n", targ->pcap_file);
1306 task_init_gen_load_pcap(task, targ);
1309 PROX_PANIC(((targ->nb_txrings == 0) && (targ->nb_txports == 0)), "Gen mode requires a tx ring or a tx port");
1310 if ((targ->flags & DSF_KEEP_SRC_MAC) == 0) {
1311 uint8_t *src_addr = prox_port_cfg[tbase->tx_params_hw.tx_port_queue->port].eth_addr.addr_bytes;
1312 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1313 rte_memcpy(&task->pkt_template[i].buf[6], src_addr, 6);
1316 memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(struct ether_addr));
1317 for (uint32_t i = 0; i < targ->n_rand_str; ++i) {
1318 PROX_PANIC(task_gen_add_rand(tbase, targ->rand_str[i], targ->rand_offset[i], UINT32_MAX),
1319 "Failed to add random\n");
1323 static struct task_init task_init_gen = {
1325 .init = init_task_gen,
1326 .handle = handle_gen_bulk,
1328 .early_init = init_task_gen_early,
1330 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1331 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1332 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1334 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1336 .size = sizeof(struct task_gen)
1339 static struct task_init task_init_gen_l3 = {
1341 .sub_mode_str = "l3",
1342 .init = init_task_gen,
1343 .handle = handle_gen_bulk,
1345 .early_init = init_task_gen_early,
1347 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1348 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1349 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1351 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1353 .size = sizeof(struct task_gen)
1356 static struct task_init task_init_gen_pcap = {
1358 .sub_mode_str = "pcap",
1359 .init = init_task_gen_pcap,
1360 .handle = handle_gen_pcap_bulk,
1361 .start = start_pcap,
1362 .early_init = init_task_gen_early,
1364 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1366 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1368 .size = sizeof(struct task_gen_pcap)
1371 __attribute__((constructor)) static void reg_task_gen(void)
1373 reg_task(&task_init_gen);
1374 reg_task(&task_init_gen_l3);
1375 reg_task(&task_init_gen_pcap);