2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
20 #include <rte_cycles.h>
21 #include <rte_version.h>
22 #include <rte_byteorder.h>
23 #include <rte_ether.h>
24 #include <rte_hash_crc.h>
26 #include "prox_shared.h"
28 #include "prox_malloc.h"
29 #include "handle_gen.h"
30 #include "handle_lat.h"
31 #include "task_init.h"
32 #include "task_base.h"
33 #include "prox_port_cfg.h"
38 #include "mbuf_utils.h"
40 #include "prox_cksum.h"
42 #include "prox_assert.h"
44 #include "token_time.h"
45 #include "local_mbuf.h"
48 #include "handle_master.h"
57 #define IP4(x) x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, x >> 24
59 #define TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC 1
61 static void pkt_template_init_mbuf(struct pkt_template *pkt_template, struct rte_mbuf *mbuf, uint8_t *pkt)
63 const uint32_t pkt_size = pkt_template->len;
65 rte_pktmbuf_pkt_len(mbuf) = pkt_size;
66 rte_pktmbuf_data_len(mbuf) = pkt_size;
68 rte_memcpy(pkt, pkt_template->buf, pkt_template->len);
71 struct task_gen_pcap {
72 struct task_base base;
74 struct local_mbuf local_mbuf;
76 struct pkt_template *proto;
84 struct task_base base;
86 struct token_time token_time;
87 struct local_mbuf local_mbuf;
88 struct pkt_template *pkt_template; /* packet templates used at runtime */
89 uint64_t write_duration_estimate; /* how long it took previously to write the time stamps in the packets */
90 uint64_t earliest_tsc_next_pkt;
91 uint64_t new_rate_bps;
92 uint64_t pkt_queue_index;
93 uint32_t n_pkts; /* number of packets in pcap */
94 uint32_t pkt_idx; /* current packet from pcap */
95 uint32_t pkt_count; /* how many pakets to generate */
96 uint32_t max_frame_size;
97 uint32_t runtime_flags;
99 uint16_t packet_id_pos;
103 uint8_t generator_id;
104 uint8_t n_rands; /* number of randoms */
105 uint8_t min_bulk_size;
106 uint8_t max_bulk_size;
108 uint8_t runtime_checksum_needed;
111 uint32_t rand_mask; /* since the random vals are uniform, masks don't introduce bias */
112 uint32_t fixed_bits; /* length of each random (max len = 4) */
113 uint16_t rand_offset; /* each random has an offset*/
114 uint8_t rand_len; /* # bytes to take from random (no bias introduced) */
116 uint64_t accur[ACCURACY_WINDOW];
117 uint64_t pkt_tsc_offset[64];
118 struct pkt_template *pkt_template_orig; /* packet templates (from inline or from pcap) */
119 struct ether_addr src_mac;
121 uint8_t cksum_offload;
122 struct prox_port_cfg *port;
123 uint64_t *bytes_to_tsc;
124 } __rte_cache_aligned;
126 static inline uint8_t ipv4_get_hdr_len(struct ipv4_hdr *ip)
128 /* Optimize for common case of IPv4 header without options. */
129 if (ip->version_ihl == 0x45)
130 return sizeof(struct ipv4_hdr);
131 if (unlikely(ip->version_ihl >> 4 != 4)) {
132 plog_warn("IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
135 return (ip->version_ihl & 0xF) * 4;
138 static void parse_l2_l3_len(uint8_t *pkt, uint16_t *l2_len, uint16_t *l3_len, uint16_t len)
140 *l2_len = sizeof(struct ether_hdr);
142 struct vlan_hdr *vlan_hdr;
143 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
145 uint16_t ether_type = eth_hdr->ether_type;
148 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (*l2_len + sizeof(struct vlan_hdr) < len)) {
149 vlan_hdr = (struct vlan_hdr *)(pkt + *l2_len);
151 ether_type = vlan_hdr->eth_proto;
154 // No L3 cksum offload for IPv6, but TODO L4 offload
155 // ETYPE_EoGRE CRC not implemented yet
157 switch (ether_type) {
171 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
176 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + *l2_len);
177 *l3_len = ipv4_get_hdr_len(ip);
181 static void checksum_packet(uint8_t *hdr, struct rte_mbuf *mbuf, struct pkt_template *pkt_template, int cksum_offload)
183 uint16_t l2_len = pkt_template->l2_len;
184 uint16_t l3_len = pkt_template->l3_len;
187 struct ipv4_hdr *ip = (struct ipv4_hdr*)(hdr + l2_len);
188 prox_ip_udp_cksum(mbuf, ip, l2_len, l3_len, cksum_offload);
192 static void task_gen_reset_token_time(struct task_gen *task)
194 token_time_set_bpp(&task->token_time, task->new_rate_bps);
195 token_time_reset(&task->token_time, rte_rdtsc(), 0);
198 static void task_gen_take_count(struct task_gen *task, uint32_t send_bulk)
200 if (task->pkt_count == (uint32_t)-1)
203 if (task->pkt_count >= send_bulk)
204 task->pkt_count -= send_bulk;
210 static int handle_gen_pcap_bulk(struct task_base *tbase, struct rte_mbuf **mbuf, uint16_t n_pkts)
212 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
213 uint64_t now = rte_rdtsc();
214 uint64_t send_bulk = 0;
215 uint32_t pkt_idx_tmp = task->pkt_idx;
217 if (pkt_idx_tmp == task->n_pkts) {
218 PROX_ASSERT(task->loop);
222 for (uint16_t j = 0; j < 64; ++j) {
223 uint64_t tsc = task->proto_tsc[pkt_idx_tmp];
224 if (task->last_tsc + tsc <= now) {
225 task->last_tsc += tsc;
228 if (pkt_idx_tmp == task->n_pkts) {
239 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
240 if (new_pkts == NULL)
243 for (uint16_t j = 0; j < send_bulk; ++j) {
244 struct rte_mbuf *next_pkt = new_pkts[j];
245 struct pkt_template *pkt_template = &task->proto[task->pkt_idx];
246 uint8_t *hdr = rte_pktmbuf_mtod(next_pkt, uint8_t *);
248 pkt_template_init_mbuf(pkt_template, next_pkt, hdr);
251 if (task->pkt_idx == task->n_pkts) {
259 return task->base.tx_pkt(&task->base, new_pkts, send_bulk, NULL);
262 static inline uint64_t bytes_to_tsc(struct task_gen *task, uint32_t bytes)
264 return task->bytes_to_tsc[bytes];
267 static uint32_t task_gen_next_pkt_idx(const struct task_gen *task, uint32_t pkt_idx)
269 return pkt_idx + 1 == task->n_pkts? 0 : pkt_idx + 1;
272 static uint32_t task_gen_offset_pkt_idx(const struct task_gen *task, uint32_t offset)
274 return (task->pkt_idx + offset) % task->n_pkts;
277 static uint32_t task_gen_calc_send_bulk(const struct task_gen *task, uint32_t *total_bytes)
279 /* The biggest bulk we allow to send is task->max_bulk_size
280 packets. The max bulk size can also be limited by the
281 pkt_count field. At the same time, we are rate limiting
282 based on the specified speed (in bytes per second) so token
283 bucket based rate limiting must also be applied. The
284 minimum bulk size is also constrained. If the calculated
285 bulk size is less then the minimum, then don't send
288 const uint32_t min_bulk = task->min_bulk_size;
289 uint32_t max_bulk = task->max_bulk_size;
291 if (task->pkt_count != (uint32_t)-1 && task->pkt_count < max_bulk) {
292 max_bulk = task->pkt_count;
295 uint32_t send_bulk = 0;
296 uint32_t pkt_idx_tmp = task->pkt_idx;
297 uint32_t would_send_bytes = 0;
301 * TODO - this must be improved to take into account the fact that, after applying randoms
302 * The packet can be replaced by an ARP
304 for (uint16_t j = 0; j < max_bulk; ++j) {
305 struct pkt_template *pktpl = &task->pkt_template[pkt_idx_tmp];
306 pkt_size = pktpl->len;
307 uint32_t pkt_len = pkt_len_to_wire_size(pkt_size);
308 if (pkt_len + would_send_bytes > task->token_time.bytes_now)
311 pkt_idx_tmp = task_gen_next_pkt_idx(task, pkt_idx_tmp);
314 would_send_bytes += pkt_len;
317 if (send_bulk < min_bulk)
319 *total_bytes = would_send_bytes;
323 static void task_gen_apply_random_fields(struct task_gen *task, uint8_t *hdr)
325 uint32_t ret, ret_tmp;
327 for (uint16_t i = 0; i < task->n_rands; ++i) {
328 ret = random_next(&task->rand[i].state);
329 ret_tmp = (ret & task->rand[i].rand_mask) | task->rand[i].fixed_bits;
331 ret_tmp = rte_bswap32(ret_tmp);
332 /* At this point, the lower order bytes (BE) contain
333 the generated value. The address where the values
334 of interest starts is at ret_tmp + 4 - rand_len. */
335 uint8_t *pret_tmp = (uint8_t*)&ret_tmp;
336 rte_memcpy(hdr + task->rand[i].rand_offset, pret_tmp + 4 - task->rand[i].rand_len, task->rand[i].rand_len);
340 static void task_gen_apply_all_random_fields(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
345 for (uint16_t i = 0; i < count; ++i)
346 task_gen_apply_random_fields(task, pkt_hdr[i]);
349 static void task_gen_apply_accur_pos(struct task_gen *task, uint8_t *pkt_hdr, uint32_t accuracy)
351 *(uint32_t *)(pkt_hdr + task->accur_pos) = accuracy;
354 static void task_gen_apply_sig(struct task_gen *task, struct pkt_template *dst)
357 *(uint32_t *)(dst->buf + task->sig_pos) = task->sig;
360 static void task_gen_apply_all_accur_pos(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
362 if (!task->accur_pos)
365 /* The accuracy of task->pkt_queue_index - ACCURACY_WINDOW is stored in
366 packet task->pkt_queue_index. The ID modulo ACCURACY_WINDOW is the
368 for (uint16_t j = 0; j < count; ++j) {
369 uint32_t accuracy = task->accur[(task->pkt_queue_index + j) & (ACCURACY_WINDOW - 1)];
370 task_gen_apply_accur_pos(task, pkt_hdr[j], accuracy);
374 static void task_gen_apply_unique_id(struct task_gen *task, uint8_t *pkt_hdr, const struct unique_id *id)
376 struct unique_id *dst = (struct unique_id *)(pkt_hdr + task->packet_id_pos);
381 static void task_gen_apply_all_unique_id(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
383 if (!task->packet_id_pos)
386 for (uint16_t i = 0; i < count; ++i) {
388 unique_id_init(&id, task->generator_id, task->pkt_queue_index++);
389 task_gen_apply_unique_id(task, pkt_hdr[i], &id);
393 static void task_gen_checksum_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
395 if (!(task->runtime_flags & TASK_TX_CRC))
398 if (!task->runtime_checksum_needed)
401 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - count);
402 for (uint16_t i = 0; i < count; ++i) {
403 struct pkt_template *pkt_template = &task->pkt_template[pkt_idx];
404 checksum_packet(pkt_hdr[i], mbufs[i], pkt_template, task->cksum_offload);
405 pkt_idx = task_gen_next_pkt_idx(task, pkt_idx);
409 static void task_gen_consume_tokens(struct task_gen *task, uint32_t tokens, uint32_t send_count)
411 /* If max burst has been sent, we can't keep up so just assume
412 that we can (leaving a "gap" in the packet stream on the
414 task->token_time.bytes_now -= tokens;
415 if (send_count == task->max_bulk_size && task->token_time.bytes_now > tokens) {
416 task->token_time.bytes_now = tokens;
420 static uint64_t task_gen_calc_bulk_duration(struct task_gen *task, uint32_t count)
422 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - 1);
423 struct pkt_template *last_pkt_template = &task->pkt_template[pkt_idx];
424 uint32_t last_pkt_len = pkt_len_to_wire_size(last_pkt_template->len);
425 #ifdef NO_EXTRAPOLATION
426 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1];
428 uint64_t last_pkt_duration = bytes_to_tsc(task, last_pkt_len);
429 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1] + last_pkt_duration;
432 return bulk_duration;
435 static uint64_t task_gen_write_latency(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
437 if (!task->lat_enabled)
440 uint64_t tx_tsc, delta_t;
441 uint64_t tsc_before_tx = 0;
443 /* Just before sending the packets, apply the time stamp
444 relative to when the first packet will be sent. The first
445 packet will be sent now. The time is read for each packet
446 to reduce the error towards the actual time the packet will
448 uint64_t write_tsc_after, write_tsc_before;
450 write_tsc_before = rte_rdtsc();
452 /* The time it took previously to write the time stamps in the
453 packets is used as an estimate for how long it will take to
454 write the time stamps now. The estimated time at which the
455 packets will actually be sent will be at tx_tsc. */
456 tx_tsc = write_tsc_before + task->write_duration_estimate;
458 /* The offset delta_t tracks the difference between the actual
459 time and the time written in the packets. Adding the offset
460 to the actual time insures that the time written in the
461 packets is monotonically increasing. At the same time,
462 simply sleeping until delta_t is zero would leave a period
463 of silence on the line. The error has been introduced
464 earlier, but the packets have already been sent. */
466 /* This happens typically if previous bulk was delayed
467 by an interrupt e.g. (with Time in nsec)
468 Time x: sleep 4 microsec
469 Time x+4000: send 64 packets (64 packets as 4000 nsec, w/ 10Gbps 64 bytes)
470 Time x+5000: send 16 packets (16 packets as 1000 nsec)
471 When we send the 16 packets, the 64 ealier packets are not yet
473 if (tx_tsc < task->earliest_tsc_next_pkt)
474 delta_t = task->earliest_tsc_next_pkt - tx_tsc;
478 for (uint16_t i = 0; i < count; ++i) {
479 uint32_t *pos = (uint32_t *)(pkt_hdr[i] + task->lat_pos);
480 const uint64_t pkt_tsc = tx_tsc + delta_t + task->pkt_tsc_offset[i];
481 *pos = pkt_tsc >> LATENCY_ACCURACY;
484 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, count);
485 task->earliest_tsc_next_pkt = tx_tsc + delta_t + bulk_duration;
486 write_tsc_after = rte_rdtsc();
487 task->write_duration_estimate = write_tsc_after - write_tsc_before;
489 /* Make sure that the time stamps that were written
490 are valid. The offset must be taken into account */
492 tsc_before_tx = rte_rdtsc();
493 } while (tsc_before_tx < tx_tsc);
495 return tsc_before_tx;
498 static void task_gen_store_accuracy(struct task_gen *task, uint32_t count, uint64_t tsc_before_tx)
500 if (!task->accur_pos)
503 uint64_t accur = rte_rdtsc() - tsc_before_tx;
504 uint64_t first_accuracy_idx = task->pkt_queue_index - count;
506 for (uint32_t i = 0; i < count; ++i) {
507 uint32_t accuracy_idx = (first_accuracy_idx + i) & (ACCURACY_WINDOW - 1);
509 task->accur[accuracy_idx] = accur;
513 static void task_gen_load_and_prefetch(struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
515 for (uint16_t i = 0; i < count; ++i)
516 rte_prefetch0(mbufs[i]);
517 for (uint16_t i = 0; i < count; ++i)
518 pkt_hdr[i] = rte_pktmbuf_mtod(mbufs[i], uint8_t *);
519 for (uint16_t i = 0; i < count; ++i)
520 rte_prefetch0(pkt_hdr[i]);
523 static void task_gen_build_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
525 uint64_t will_send_bytes = 0;
527 for (uint16_t i = 0; i < count; ++i) {
528 struct pkt_template *pktpl = &task->pkt_template[task->pkt_idx];
529 struct pkt_template *pkt_template = &task->pkt_template[task->pkt_idx];
530 pkt_template_init_mbuf(pkt_template, mbufs[i], pkt_hdr[i]);
531 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
532 if (task->lat_enabled) {
533 #ifdef NO_EXTRAPOLATION
534 task->pkt_tsc_offset[i] = 0;
536 task->pkt_tsc_offset[i] = bytes_to_tsc(task, will_send_bytes);
538 will_send_bytes += pkt_len_to_wire_size(pkt_template->len);
540 task->pkt_idx = task_gen_next_pkt_idx(task, task->pkt_idx);
544 static void task_gen_update_config(struct task_gen *task)
546 if (task->token_time.cfg.bpp != task->new_rate_bps)
547 task_gen_reset_token_time(task);
550 static inline void build_value(struct task_gen *task, uint32_t mask, int bit_pos, uint32_t val, uint32_t fixed_bits)
552 struct task_base *tbase = (struct task_base *)task;
554 build_value(task, mask >> 1, bit_pos + 1, val, fixed_bits);
556 build_value(task, mask >> 1, bit_pos + 1, val | (1 << bit_pos), fixed_bits);
559 register_ip_to_ctrl_plane(tbase->l3.tmaster, rte_cpu_to_be_32(val | fixed_bits), tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
562 static inline void register_all_ip_to_ctrl_plane(struct task_gen *task)
564 struct task_base *tbase = (struct task_base *)task;
569 for (uint32_t i = 0; i < task->n_pkts; ++i) {
570 struct pkt_template *pktpl = &task->pkt_template[i];
571 unsigned int ip_src_pos = 0;
573 unsigned int l2_len = sizeof(struct ether_hdr);
575 uint8_t *pkt = pktpl->buf;
576 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
577 uint16_t ether_type = eth_hdr->ether_type;
578 struct vlan_hdr *vlan_hdr;
581 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(struct vlan_hdr) < pktpl->len)) {
582 vlan_hdr = (struct vlan_hdr *)(pkt + l2_len);
584 ether_type = vlan_hdr->eth_proto;
586 if ((ether_type == ETYPE_MPLSU) || (ether_type == ETYPE_MPLSM)) {
590 if ((ether_type != ETYPE_IPv4) && !maybe_ipv4)
593 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + l2_len);
594 PROX_PANIC(ip->version_ihl >> 4 != 4, "IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
596 // Even if IPv4 header contains options, options are after ip src and dst
597 ip_src_pos = l2_len + sizeof(struct ipv4_hdr) - 2 * sizeof(uint32_t);
598 uint32_t *ip_src = ((uint32_t *)(pktpl->buf + ip_src_pos));
599 plog_info("\tip_src_pos = %d, ip_src = %x\n", ip_src_pos, *ip_src);
600 register_ip_to_ctrl_plane(tbase->l3.tmaster, *ip_src, tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
602 for (int j = 0; j < task->n_rands; j++) {
603 offset = task->rand[j].rand_offset;
604 len = task->rand[j].rand_len;
605 mask = task->rand[j].rand_mask;
606 fixed = task->rand[j].fixed_bits;
607 plog_info("offset = %d, len = %d, mask = %x, fixed = %x\n", offset, len, mask, fixed);
608 if ((offset < ip_src_pos + 4) && (offset + len >= ip_src_pos)) {
609 if (offset >= ip_src_pos) {
610 int32_t ip_src_mask = (1 << (4 + ip_src_pos - offset) * 8) - 1;
611 mask = mask & ip_src_mask;
612 fixed = (fixed & ip_src_mask) | (rte_be_to_cpu_32(*ip_src) & ~ip_src_mask);
613 build_value(task, mask, 0, 0, fixed);
615 int32_t bits = ((ip_src_pos + 4 - offset - len) * 8);
617 fixed = (fixed << bits) | (rte_be_to_cpu_32(*ip_src) & ((1 << bits) - 1));
618 build_value(task, mask, 0, 0, fixed);
625 static int handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
627 struct task_gen *task = (struct task_gen *)tbase;
628 uint8_t out[MAX_PKT_BURST] = {0};
633 task_gen_update_config(task);
635 if (task->pkt_count == 0) {
636 task_gen_reset_token_time(task);
639 if (!task->token_time.cfg.bpp)
642 token_time_update(&task->token_time, rte_rdtsc());
644 uint32_t would_send_bytes;
645 uint32_t send_bulk = task_gen_calc_send_bulk(task, &would_send_bytes);
649 task_gen_take_count(task, send_bulk);
650 task_gen_consume_tokens(task, would_send_bytes, send_bulk);
652 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
653 if (new_pkts == NULL)
655 uint8_t *pkt_hdr[MAX_RING_BURST];
657 task_gen_load_and_prefetch(new_pkts, pkt_hdr, send_bulk);
658 task_gen_build_packets(task, new_pkts, pkt_hdr, send_bulk);
659 task_gen_apply_all_random_fields(task, pkt_hdr, send_bulk);
660 task_gen_apply_all_accur_pos(task, new_pkts, pkt_hdr, send_bulk);
661 task_gen_apply_all_unique_id(task, new_pkts, pkt_hdr, send_bulk);
663 uint64_t tsc_before_tx;
665 tsc_before_tx = task_gen_write_latency(task, pkt_hdr, send_bulk);
666 task_gen_checksum_packets(task, new_pkts, pkt_hdr, send_bulk);
667 ret = task->base.tx_pkt(&task->base, new_pkts, send_bulk, out);
668 task_gen_store_accuracy(task, send_bulk, tsc_before_tx);
670 // If we failed to send some packets, we need to do some clean-up:
673 // We need re-use the packets indexes not being sent
674 // Hence non-sent packets will not be considered as lost by the receiver when it looks at
675 // packet ids. This should also increase the percentage of packets used for latency measurements
676 task->pkt_queue_index -= ret;
678 // In case of failures, the estimate about when we can send next packet (earliest_tsc_next_pkt) is wrong
679 // This would result in under-estimated latency (up to 0 or negative)
680 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, ret);
681 task->earliest_tsc_next_pkt -= bulk_duration;
686 static void init_task_gen_seeds(struct task_gen *task)
688 for (size_t i = 0; i < sizeof(task->rand)/sizeof(task->rand[0]); ++i)
689 random_init_seed(&task->rand[i].state);
692 static uint32_t pcap_count_pkts(pcap_t *handle, uint32_t *max_frame_size)
694 struct pcap_pkthdr header;
698 long pkt1_fpos = ftell(pcap_file(handle));
700 while ((buf = pcap_next(handle, &header))) {
701 if (header.len > *max_frame_size)
702 *max_frame_size = header.len;
705 int ret2 = fseek(pcap_file(handle), pkt1_fpos, SEEK_SET);
706 PROX_PANIC(ret2 != 0, "Failed to reset reading pcap file\n");
710 static uint64_t avg_time_stamp(uint64_t *time_stamp, uint32_t n)
712 uint64_t tot_inter_pkt = 0;
714 for (uint32_t i = 0; i < n; ++i)
715 tot_inter_pkt += time_stamp[i];
716 return (tot_inter_pkt + n / 2)/n;
719 static int pcap_read_pkts(pcap_t *handle, const char *file_name, uint32_t n_pkts, struct pkt_template *proto, uint64_t *time_stamp, uint32_t max_frame_size)
721 struct pcap_pkthdr header;
725 for (uint32_t i = 0; i < n_pkts; ++i) {
726 buf = pcap_next(handle, &header);
728 PROX_PANIC(buf == NULL, "Failed to read packet %d from pcap %s\n", i, file_name);
729 proto[i].len = header.len;
730 len = RTE_MIN(header.len, max_frame_size);
731 if (header.len > len)
732 plogx_warn("Packet truncated from %u to %zu bytes\n", header.len, len);
735 static struct timeval beg;
741 tv = tv_diff(&beg, &header.ts);
742 tv_to_tsc(&tv, time_stamp + i);
744 rte_memcpy(proto[i].buf, buf, len);
747 if (time_stamp && n_pkts) {
748 for (uint32_t i = n_pkts - 1; i > 0; --i)
749 time_stamp[i] -= time_stamp[i - 1];
750 /* Since the handle function will loop the packets,
751 there is one time-stamp that is not provided by the
752 pcap file. This is the time between the last and
753 the first packet. This implementation takes the
754 average of the inter-packet times here. */
756 time_stamp[0] = avg_time_stamp(time_stamp + 1, n_pkts - 1);
762 static int check_pkt_size(struct task_gen *task, uint32_t pkt_size, int do_panic)
764 const uint16_t min_len = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr);
765 const uint16_t max_len = task->max_frame_size;
768 PROX_PANIC(pkt_size == 0, "Invalid packet size length (no packet defined?)\n");
769 PROX_PANIC(pkt_size > max_len, "pkt_size out of range (must be <= %u)\n", max_len);
770 PROX_PANIC(pkt_size < min_len, "pkt_size out of range (must be >= %u)\n", min_len);
774 plog_err("Invalid packet size length (no packet defined?)\n");
777 if (pkt_size > max_len) {
778 plog_err("pkt_size out of range (must be <= %u)\n", max_len);
781 if (pkt_size < min_len) {
782 plog_err("pkt_size out of range (must be >= %u)\n", min_len);
789 static int check_all_pkt_size(struct task_gen *task, int do_panic)
792 for (uint32_t i = 0; i < task->n_pkts;++i) {
793 if ((rc = check_pkt_size(task, task->pkt_template[i].len, do_panic)) != 0)
799 static int check_fields_in_bounds(struct task_gen *task, uint32_t pkt_size, int do_panic)
801 if (task->lat_enabled) {
802 uint32_t pos_beg = task->lat_pos;
803 uint32_t pos_end = task->lat_pos + 3U;
806 PROX_PANIC(pkt_size <= pos_end, "Writing latency at %u-%u, but packet size is %u bytes\n",
807 pos_beg, pos_end, pkt_size);
808 else if (pkt_size <= pos_end) {
809 plog_err("Writing latency at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
813 if (task->packet_id_pos) {
814 uint32_t pos_beg = task->packet_id_pos;
815 uint32_t pos_end = task->packet_id_pos + 4U;
818 PROX_PANIC(pkt_size <= pos_end, "Writing packet at %u-%u, but packet size is %u bytes\n",
819 pos_beg, pos_end, pkt_size);
820 else if (pkt_size <= pos_end) {
821 plog_err("Writing packet at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
825 if (task->accur_pos) {
826 uint32_t pos_beg = task->accur_pos;
827 uint32_t pos_end = task->accur_pos + 3U;
830 PROX_PANIC(pkt_size <= pos_end, "Writing accuracy at %u%-u, but packet size is %u bytes\n",
831 pos_beg, pos_end, pkt_size);
832 else if (pkt_size <= pos_end) {
833 plog_err("Writing accuracy at %u%-u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
840 static void task_gen_pkt_template_recalc_metadata(struct task_gen *task)
842 struct pkt_template *template;
844 for (size_t i = 0; i < task->n_pkts; ++i) {
845 template = &task->pkt_template[i];
846 parse_l2_l3_len(template->buf, &template->l2_len, &template->l3_len, template->len);
850 static void task_gen_pkt_template_recalc_checksum(struct task_gen *task)
852 struct pkt_template *template;
855 task->runtime_checksum_needed = 0;
856 for (size_t i = 0; i < task->n_pkts; ++i) {
857 template = &task->pkt_template[i];
858 if (template->l2_len == 0)
860 ip = (struct ipv4_hdr *)(template->buf + template->l2_len);
862 ip->hdr_checksum = 0;
863 prox_ip_cksum_sw(ip);
864 uint32_t l4_len = rte_bswap16(ip->total_length) - template->l3_len;
866 if (ip->next_proto_id == IPPROTO_UDP) {
867 struct udp_hdr *udp = (struct udp_hdr *)(((uint8_t *)ip) + template->l3_len);
868 prox_udp_cksum_sw(udp, l4_len, ip->src_addr, ip->dst_addr);
869 } else if (ip->next_proto_id == IPPROTO_TCP) {
870 struct tcp_hdr *tcp = (struct tcp_hdr *)(((uint8_t *)ip) + template->l3_len);
871 prox_tcp_cksum_sw(tcp, l4_len, ip->src_addr, ip->dst_addr);
874 /* The current implementation avoids checksum
875 calculation by determining that at packet
876 construction time, no fields are applied that would
877 require a recalculation of the checksum. */
878 if (task->lat_enabled && task->lat_pos > template->l2_len)
879 task->runtime_checksum_needed = 1;
880 if (task->accur_pos > template->l2_len)
881 task->runtime_checksum_needed = 1;
882 if (task->packet_id_pos > template->l2_len)
883 task->runtime_checksum_needed = 1;
887 static void task_gen_pkt_template_recalc_all(struct task_gen *task)
889 task_gen_pkt_template_recalc_metadata(task);
890 task_gen_pkt_template_recalc_checksum(task);
893 static void task_gen_reset_pkt_templates_len(struct task_gen *task)
895 struct pkt_template *src, *dst;
897 for (size_t i = 0; i < task->n_pkts; ++i) {
898 src = &task->pkt_template_orig[i];
899 dst = &task->pkt_template[i];
904 static void task_gen_reset_pkt_templates_content(struct task_gen *task)
906 struct pkt_template *src, *dst;
908 for (size_t i = 0; i < task->n_pkts; ++i) {
909 src = &task->pkt_template_orig[i];
910 dst = &task->pkt_template[i];
911 memcpy(dst->buf, src->buf, RTE_MAX(src->len, dst->len));
912 task_gen_apply_sig(task, dst);
916 static void task_gen_reset_pkt_templates(struct task_gen *task)
918 task_gen_reset_pkt_templates_len(task);
919 task_gen_reset_pkt_templates_content(task);
920 task_gen_pkt_template_recalc_all(task);
923 static void task_init_gen_load_pkt_inline(struct task_gen *task, struct task_args *targ)
925 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
929 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
930 task->pkt_template = prox_zmalloc(mem_size, socket_id);
931 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
933 PROX_PANIC(task->pkt_template == NULL ||
934 task->pkt_template_orig == NULL,
935 "Failed to allocate %lu bytes (in huge pages) for packet template\n", mem_size);
937 task->pkt_template->buf = prox_zmalloc(task->max_frame_size, socket_id);
938 task->pkt_template_orig->buf = prox_zmalloc(task->max_frame_size, socket_id);
939 PROX_PANIC(task->pkt_template->buf == NULL ||
940 task->pkt_template_orig->buf == NULL,
941 "Failed to allocate %u bytes (in huge pages) for packet\n", task->max_frame_size);
943 PROX_PANIC(targ->pkt_size > task->max_frame_size,
944 targ->pkt_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE - 4 ?
945 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
947 rte_memcpy(task->pkt_template_orig[0].buf, targ->pkt_inline, targ->pkt_size);
948 task->pkt_template_orig[0].len = targ->pkt_size;
949 task_gen_reset_pkt_templates(task);
950 check_all_pkt_size(task, 1);
951 check_fields_in_bounds(task, task->pkt_template[0].len, 1);
954 static void task_init_gen_load_pcap(struct task_gen *task, struct task_args *targ)
956 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
957 char err[PCAP_ERRBUF_SIZE];
958 uint32_t max_frame_size;
959 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
960 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
962 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
963 plogx_info("%u packets in pcap file '%s'; max frame size=%d\n", task->n_pkts, targ->pcap_file, max_frame_size);
964 PROX_PANIC(max_frame_size > task->max_frame_size,
965 max_frame_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE -4 ?
966 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
969 task->n_pkts = RTE_MIN(task->n_pkts, targ->n_pkts);
970 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
971 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
972 task->pkt_template = prox_zmalloc(mem_size, socket_id);
973 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
974 PROX_PANIC(task->pkt_template == NULL ||
975 task->pkt_template_orig == NULL,
976 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
978 for (uint i = 0; i < task->n_pkts; i++) {
979 task->pkt_template[i].buf = prox_zmalloc(task->max_frame_size, socket_id);
980 task->pkt_template_orig[i].buf = prox_zmalloc(task->max_frame_size, socket_id);
982 PROX_PANIC(task->pkt_template->buf == NULL ||
983 task->pkt_template_orig->buf == NULL,
984 "Failed to allocate %u bytes (in huge pages) for pcap file\n", task->max_frame_size);
987 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->pkt_template_orig, NULL, max_frame_size);
989 task_gen_reset_pkt_templates(task);
992 static struct rte_mempool *task_gen_create_mempool(struct task_args *targ, uint16_t max_frame_size)
994 static char name[] = "gen_pool";
995 struct rte_mempool *ret;
996 const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
999 uint32_t mbuf_size = TX_MBUF_SIZE;
1000 if (max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM > mbuf_size)
1001 mbuf_size = max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
1002 plog_info("\t\tCreating mempool with name '%s'\n", name);
1003 ret = rte_mempool_create(name, targ->nb_mbuf - 1, mbuf_size,
1004 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
1005 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
1007 PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
1008 sock_id, targ->nb_mbuf - 1);
1010 plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", ret,
1011 targ->nb_mbuf - 1, mbuf_size, targ->nb_cache_mbuf, sock_id);
1016 void task_gen_set_pkt_count(struct task_base *tbase, uint32_t count)
1018 struct task_gen *task = (struct task_gen *)tbase;
1020 task->pkt_count = count;
1023 int task_gen_set_pkt_size(struct task_base *tbase, uint32_t pkt_size)
1025 struct task_gen *task = (struct task_gen *)tbase;
1028 for (size_t i = 0; i < task->n_pkts; ++i) {
1029 if ((rc = check_pkt_size(task, pkt_size, 0)) != 0)
1031 if ((rc = check_fields_in_bounds(task, pkt_size, 0)) != 0)
1034 for (size_t i = 0; i < task->n_pkts; ++i) {
1035 task->pkt_template[i].len = pkt_size;
1040 void task_gen_set_rate(struct task_base *tbase, uint64_t bps)
1042 struct task_gen *task = (struct task_gen *)tbase;
1044 task->new_rate_bps = bps;
1047 void task_gen_reset_randoms(struct task_base *tbase)
1049 struct task_gen *task = (struct task_gen *)tbase;
1051 for (uint32_t i = 0; i < task->n_rands; ++i) {
1052 task->rand[i].rand_mask = 0;
1053 task->rand[i].fixed_bits = 0;
1054 task->rand[i].rand_offset = 0;
1059 int task_gen_set_value(struct task_base *tbase, uint32_t value, uint32_t offset, uint32_t len)
1061 struct task_gen *task = (struct task_gen *)tbase;
1063 if (offset + len > task->max_frame_size)
1065 for (size_t i = 0; i < task->n_pkts; ++i) {
1066 uint32_t to_write = rte_cpu_to_be_32(value) >> ((4 - len) * 8);
1067 uint8_t *dst = task->pkt_template[i].buf;
1069 rte_memcpy(dst + offset, &to_write, len);
1072 task_gen_pkt_template_recalc_all(task);
1077 void task_gen_reset_values(struct task_base *tbase)
1079 struct task_gen *task = (struct task_gen *)tbase;
1081 task_gen_reset_pkt_templates_content(task);
1082 if (task->flags & TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC) {
1083 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1084 rte_memcpy(&task->pkt_template[i].buf[sizeof(struct ether_addr)], &task->src_mac, sizeof(struct ether_addr));
1089 uint32_t task_gen_get_n_randoms(struct task_base *tbase)
1091 struct task_gen *task = (struct task_gen *)tbase;
1093 return task->n_rands;
1096 static void init_task_gen_pcap(struct task_base *tbase, struct task_args *targ)
1098 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1099 const uint32_t sockid = rte_lcore_to_socket_id(targ->lconf->id);
1100 uint32_t max_frame_size;
1102 task->loop = targ->loop;
1104 task->hz = rte_get_tsc_hz();
1106 char err[PCAP_ERRBUF_SIZE];
1107 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1108 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1110 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
1111 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1113 task->local_mbuf.mempool = task_gen_create_mempool(targ, max_frame_size);
1115 PROX_PANIC(!strcmp(targ->pcap_file, ""), "No pcap file defined\n");
1118 plogx_info("Configured to load %u packets\n", targ->n_pkts);
1119 if (task->n_pkts > targ->n_pkts)
1120 task->n_pkts = targ->n_pkts;
1122 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1124 size_t mem_size = task->n_pkts * (sizeof(*task->proto) + sizeof(*task->proto_tsc));
1125 uint8_t *mem = prox_zmalloc(mem_size, sockid);
1127 PROX_PANIC(mem == NULL, "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1128 task->proto = (struct pkt_template *) mem;
1129 task->proto_tsc = (uint64_t *)(mem + task->n_pkts * sizeof(*task->proto));
1131 for (uint i = 0; i < targ->n_pkts; i++) {
1132 task->proto[i].buf = prox_zmalloc(max_frame_size, sockid);
1133 PROX_PANIC(task->proto[i].buf == NULL, "Failed to allocate %u bytes (in huge pages) for pcap file\n", max_frame_size);
1136 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->proto, task->proto_tsc, max_frame_size);
1140 static int task_gen_find_random_with_offset(struct task_gen *task, uint32_t offset)
1142 for (uint32_t i = 0; i < task->n_rands; ++i) {
1143 if (task->rand[i].rand_offset == offset) {
1151 int task_gen_add_rand(struct task_base *tbase, const char *rand_str, uint32_t offset, uint32_t rand_id)
1153 struct task_gen *task = (struct task_gen *)tbase;
1154 uint32_t existing_rand;
1156 if (rand_id == UINT32_MAX && task->n_rands == 64) {
1157 plog_err("Too many randoms\n");
1160 uint32_t mask, fixed, len;
1162 if (parse_random_str(&mask, &fixed, &len, rand_str)) {
1163 plog_err("%s\n", get_parse_err());
1166 task->runtime_checksum_needed = 1;
1168 existing_rand = task_gen_find_random_with_offset(task, offset);
1169 if (existing_rand != UINT32_MAX) {
1170 plog_warn("Random at offset %d already set => overwriting len = %d %s\n", offset, len, rand_str);
1171 rand_id = existing_rand;
1172 task->rand[rand_id].rand_len = len;
1173 task->rand[rand_id].rand_offset = offset;
1174 task->rand[rand_id].rand_mask = mask;
1175 task->rand[rand_id].fixed_bits = fixed;
1179 task->rand[task->n_rands].rand_len = len;
1180 task->rand[task->n_rands].rand_offset = offset;
1181 task->rand[task->n_rands].rand_mask = mask;
1182 task->rand[task->n_rands].fixed_bits = fixed;
1188 static void start(struct task_base *tbase)
1190 struct task_gen *task = (struct task_gen *)tbase;
1191 task->pkt_queue_index = 0;
1193 task_gen_reset_token_time(task);
1194 if (tbase->l3.tmaster) {
1195 register_all_ip_to_ctrl_plane(task);
1199 Handle the case when two tasks transmit to the same port
1200 and one of them is stopped. In that case ARP (requests or replies)
1201 might not be sent. Master will have to keep a list of rings.
1202 stop will have to de-register IP from ctrl plane.
1203 un-registration will remove the ring. when having more than
1204 one active rings, master can always use the first one
1208 static void start_pcap(struct task_base *tbase)
1210 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1211 /* When we start, the first packet is sent immediately. */
1212 task->last_tsc = rte_rdtsc() - task->proto_tsc[0];
1216 static void init_task_gen_early(struct task_args *targ)
1218 uint8_t *generator_count = prox_sh_find_system("generator_count");
1220 if (generator_count == NULL) {
1221 generator_count = prox_zmalloc(sizeof(*generator_count), rte_lcore_to_socket_id(targ->lconf->id));
1222 PROX_PANIC(generator_count == NULL, "Failed to allocate generator count\n");
1223 prox_sh_add_system("generator_count", generator_count);
1225 targ->generator_id = *generator_count;
1226 (*generator_count)++;
1229 static void init_task_gen(struct task_base *tbase, struct task_args *targ)
1231 struct task_gen *task = (struct task_gen *)tbase;
1233 task->packet_id_pos = targ->packet_id_pos;
1235 struct prox_port_cfg *port = find_reachable_port(targ);
1236 // TODO: check that all reachable ports have the same mtu...
1238 task->cksum_offload = port->requested_tx_offload & (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM);
1240 task->max_frame_size = port->mtu + ETHER_HDR_LEN + 2 * PROX_VLAN_TAG_SIZE;
1242 // Not generating to any port...
1243 task->max_frame_size = ETHER_MAX_LEN;
1245 task->local_mbuf.mempool = task_gen_create_mempool(targ, task->max_frame_size);
1246 PROX_PANIC(task->local_mbuf.mempool == NULL, "Failed to create mempool\n");
1248 task->hz = rte_get_tsc_hz();
1249 task->lat_pos = targ->lat_pos;
1250 task->accur_pos = targ->accur_pos;
1251 task->sig_pos = targ->sig_pos;
1252 task->sig = targ->sig;
1253 task->new_rate_bps = targ->rate_bps;
1256 * For tokens, use 10 Gbps as base rate
1257 * Scripts can then use speed command, with speed=100 as 10 Gbps and speed=400 as 40 Gbps
1258 * Script can query prox "port info" command to find out the port link speed to know
1259 * at which rate to start. Note that virtio running on OVS returns 10 Gbps, so a script has
1260 * probably also to check the driver (as returned by the same "port info" command.
1262 struct token_time_cfg tt_cfg = token_time_cfg_create(1250000000, rte_get_tsc_hz(), -1);
1263 token_time_init(&task->token_time, &tt_cfg);
1265 init_task_gen_seeds(task);
1267 task->min_bulk_size = targ->min_bulk_size;
1268 task->max_bulk_size = targ->max_bulk_size;
1269 if (task->min_bulk_size < 1)
1270 task->min_bulk_size = 1;
1271 if (task->max_bulk_size < 1)
1272 task->max_bulk_size = 64;
1273 PROX_PANIC(task->max_bulk_size > 64, "max_bulk_size higher than 64\n");
1274 PROX_PANIC(task->max_bulk_size < task->min_bulk_size, "max_bulk_size must be > than min_bulk_size\n");
1276 task->pkt_count = -1;
1277 task->lat_enabled = targ->lat_enabled;
1278 task->runtime_flags = targ->runtime_flags;
1279 PROX_PANIC((task->lat_pos || task->accur_pos) && !task->lat_enabled, "lat not enabled by lat pos or accur pos configured\n");
1281 task->generator_id = targ->generator_id;
1282 plog_info("\tGenerator id = %d\n", task->generator_id);
1284 // Allocate array holding bytes to tsc for supported frame sizes
1285 task->bytes_to_tsc = prox_zmalloc(task->max_frame_size * MAX_PKT_BURST * sizeof(task->bytes_to_tsc[0]), rte_lcore_to_socket_id(targ->lconf->id));
1286 PROX_PANIC(task->bytes_to_tsc == NULL,
1287 "Failed to allocate %u bytes (in huge pages) for bytes_to_tsc\n", task->max_frame_size);
1289 // task->port->max_link_speed reports the maximum, non negotiated ink speed in Mbps e.g. 40k for a 40 Gbps NIC.
1290 // It can be UINT32_MAX (virtual devices or not supported by DPDK < 16.04)
1291 uint64_t bytes_per_hz = UINT64_MAX;
1292 if ((task->port) && (task->port->max_link_speed != UINT32_MAX)) {
1293 bytes_per_hz = task->port->max_link_speed * 125000L;
1294 plog_info("\tPort %u: max link speed is %ld Mbps\n",
1295 (uint8_t)(task->port - prox_port_cfg), 8 * bytes_per_hz / 1000000);
1297 // There are cases where hz estimate might be slighly over-estimated
1298 // This results in too much extrapolation
1299 // Only account for 99% of extrapolation to handle cases with up to 1% error clocks
1300 for (unsigned int i = 0; i < task->max_frame_size * MAX_PKT_BURST ; i++) {
1301 if (bytes_per_hz == UINT64_MAX)
1302 task->bytes_to_tsc[i] = 0;
1304 task->bytes_to_tsc[i] = (task->hz * i * 0.99) / bytes_per_hz;
1307 if (!strcmp(targ->pcap_file, "")) {
1308 plog_info("\tUsing inline definition of a packet\n");
1309 task_init_gen_load_pkt_inline(task, targ);
1311 plog_info("Loading from pcap %s\n", targ->pcap_file);
1312 task_init_gen_load_pcap(task, targ);
1315 PROX_PANIC(((targ->nb_txrings == 0) && (targ->nb_txports == 0)), "Gen mode requires a tx ring or a tx port");
1316 if ((targ->flags & DSF_KEEP_SRC_MAC) == 0) {
1317 task->flags |= TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC;
1318 memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(struct ether_addr));
1319 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1320 rte_memcpy(&task->pkt_template[i].buf[sizeof(struct ether_addr)], &task->src_mac, sizeof(struct ether_addr));
1323 for (uint32_t i = 0; i < targ->n_rand_str; ++i) {
1324 PROX_PANIC(task_gen_add_rand(tbase, targ->rand_str[i], targ->rand_offset[i], UINT32_MAX),
1325 "Failed to add random\n");
1329 static struct task_init task_init_gen = {
1331 .init = init_task_gen,
1332 .handle = handle_gen_bulk,
1334 .early_init = init_task_gen_early,
1336 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1337 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1338 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1340 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1342 .size = sizeof(struct task_gen)
1345 static struct task_init task_init_gen_l3 = {
1347 .sub_mode_str = "l3",
1348 .init = init_task_gen,
1349 .handle = handle_gen_bulk,
1351 .early_init = init_task_gen_early,
1353 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1354 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1355 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1357 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1359 .size = sizeof(struct task_gen)
1362 /* This mode uses time stamps in the pcap file */
1363 static struct task_init task_init_gen_pcap = {
1365 .sub_mode_str = "pcap",
1366 .init = init_task_gen_pcap,
1367 .handle = handle_gen_pcap_bulk,
1368 .start = start_pcap,
1369 .early_init = init_task_gen_early,
1371 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1373 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1375 .size = sizeof(struct task_gen_pcap)
1378 __attribute__((constructor)) static void reg_task_gen(void)
1380 reg_task(&task_init_gen);
1381 reg_task(&task_init_gen_l3);
1382 reg_task(&task_init_gen_pcap);