2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
20 #include <rte_cycles.h>
21 #include <rte_version.h>
22 #include <rte_byteorder.h>
23 #include <rte_ether.h>
24 #include <rte_hash_crc.h>
26 #include "prox_shared.h"
28 #include "prox_malloc.h"
29 #include "handle_gen.h"
30 #include "handle_lat.h"
31 #include "task_init.h"
32 #include "task_base.h"
33 #include "prox_port_cfg.h"
38 #include "mbuf_utils.h"
40 #include "prox_cksum.h"
42 #include "prox_assert.h"
44 #include "token_time.h"
45 #include "local_mbuf.h"
48 #include "handle_master.h"
57 #define MAX_TEMPLATE_INDEX 65536
58 #define TEMPLATE_INDEX_MASK (MAX_TEMPLATE_INDEX - 1)
60 #define IP4(x) x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, x >> 24
62 #define TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC 1
64 static void pkt_template_init_mbuf(struct pkt_template *pkt_template, struct rte_mbuf *mbuf, uint8_t *pkt)
66 const uint32_t pkt_size = pkt_template->len;
68 rte_pktmbuf_pkt_len(mbuf) = pkt_size;
69 rte_pktmbuf_data_len(mbuf) = pkt_size;
71 rte_memcpy(pkt, pkt_template->buf, pkt_template->len);
74 struct task_gen_pcap {
75 struct task_base base;
77 struct local_mbuf local_mbuf;
79 struct pkt_template *proto;
87 struct task_base base;
89 struct token_time token_time;
90 struct local_mbuf local_mbuf;
91 struct pkt_template *pkt_template; /* packet templates used at runtime */
92 uint64_t write_duration_estimate; /* how long it took previously to write the time stamps in the packets */
93 uint64_t earliest_tsc_next_pkt;
94 uint64_t new_rate_bps;
95 uint64_t pkt_queue_index;
96 uint32_t n_pkts; /* number of packets in pcap */
97 uint32_t pkt_idx; /* current packet from pcap */
98 uint32_t pkt_count; /* how many pakets to generate */
99 uint32_t max_frame_size;
100 uint32_t runtime_flags;
102 uint16_t packet_id_pos;
106 uint8_t generator_id;
107 uint8_t n_rands; /* number of randoms */
108 uint8_t min_bulk_size;
109 uint8_t max_bulk_size;
111 uint8_t runtime_checksum_needed;
114 uint32_t rand_mask; /* since the random vals are uniform, masks don't introduce bias */
115 uint32_t fixed_bits; /* length of each random (max len = 4) */
116 uint16_t rand_offset; /* each random has an offset*/
117 uint8_t rand_len; /* # bytes to take from random (no bias introduced) */
119 uint64_t accur[ACCURACY_WINDOW];
120 uint64_t pkt_tsc_offset[64];
121 struct pkt_template *pkt_template_orig; /* packet templates (from inline or from pcap) */
122 struct ether_addr src_mac;
124 uint8_t cksum_offload;
125 struct prox_port_cfg *port;
126 uint64_t *bytes_to_tsc;
127 } __rte_cache_aligned;
129 static inline uint8_t ipv4_get_hdr_len(struct ipv4_hdr *ip)
131 /* Optimize for common case of IPv4 header without options. */
132 if (ip->version_ihl == 0x45)
133 return sizeof(struct ipv4_hdr);
134 if (unlikely(ip->version_ihl >> 4 != 4)) {
135 plog_warn("IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
138 return (ip->version_ihl & 0xF) * 4;
141 static void parse_l2_l3_len(uint8_t *pkt, uint16_t *l2_len, uint16_t *l3_len, uint16_t len)
143 *l2_len = sizeof(struct ether_hdr);
145 struct vlan_hdr *vlan_hdr;
146 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
148 uint16_t ether_type = eth_hdr->ether_type;
151 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (*l2_len + sizeof(struct vlan_hdr) < len)) {
152 vlan_hdr = (struct vlan_hdr *)(pkt + *l2_len);
154 ether_type = vlan_hdr->eth_proto;
157 // No L3 cksum offload for IPv6, but TODO L4 offload
158 // ETYPE_EoGRE CRC not implemented yet
160 switch (ether_type) {
174 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
179 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + *l2_len);
180 *l3_len = ipv4_get_hdr_len(ip);
184 static void checksum_packet(uint8_t *hdr, struct rte_mbuf *mbuf, struct pkt_template *pkt_template, int cksum_offload)
186 uint16_t l2_len = pkt_template->l2_len;
187 uint16_t l3_len = pkt_template->l3_len;
190 struct ipv4_hdr *ip = (struct ipv4_hdr*)(hdr + l2_len);
191 prox_ip_udp_cksum(mbuf, ip, l2_len, l3_len, cksum_offload);
195 static void task_gen_reset_token_time(struct task_gen *task)
197 token_time_set_bpp(&task->token_time, task->new_rate_bps);
198 token_time_reset(&task->token_time, rte_rdtsc(), 0);
201 static void task_gen_take_count(struct task_gen *task, uint32_t send_bulk)
203 if (task->pkt_count == (uint32_t)-1)
206 if (task->pkt_count >= send_bulk)
207 task->pkt_count -= send_bulk;
213 static int handle_gen_pcap_bulk(struct task_base *tbase, struct rte_mbuf **mbuf, uint16_t n_pkts)
215 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
216 uint64_t now = rte_rdtsc();
217 uint64_t send_bulk = 0;
218 uint32_t pkt_idx_tmp = task->pkt_idx;
220 if (pkt_idx_tmp == task->n_pkts) {
221 PROX_ASSERT(task->loop);
225 for (uint16_t j = 0; j < 64; ++j) {
226 uint64_t tsc = task->proto_tsc[pkt_idx_tmp];
227 if (task->last_tsc + tsc <= now) {
228 task->last_tsc += tsc;
231 if (pkt_idx_tmp == task->n_pkts) {
242 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
243 if (new_pkts == NULL)
246 for (uint16_t j = 0; j < send_bulk; ++j) {
247 struct rte_mbuf *next_pkt = new_pkts[j];
248 struct pkt_template *pkt_template = &task->proto[task->pkt_idx];
249 uint8_t *hdr = rte_pktmbuf_mtod(next_pkt, uint8_t *);
251 pkt_template_init_mbuf(pkt_template, next_pkt, hdr);
254 if (task->pkt_idx == task->n_pkts) {
262 return task->base.tx_pkt(&task->base, new_pkts, send_bulk, NULL);
265 static inline uint64_t bytes_to_tsc(struct task_gen *task, uint32_t bytes)
267 return task->bytes_to_tsc[bytes];
270 static uint32_t task_gen_next_pkt_idx(const struct task_gen *task, uint32_t pkt_idx)
272 return pkt_idx + 1 == task->n_pkts? 0 : pkt_idx + 1;
275 static uint32_t task_gen_offset_pkt_idx(const struct task_gen *task, uint32_t offset)
277 return (task->pkt_idx + offset) % task->n_pkts;
280 static uint32_t task_gen_calc_send_bulk(const struct task_gen *task, uint32_t *total_bytes)
282 /* The biggest bulk we allow to send is task->max_bulk_size
283 packets. The max bulk size can also be limited by the
284 pkt_count field. At the same time, we are rate limiting
285 based on the specified speed (in bytes per second) so token
286 bucket based rate limiting must also be applied. The
287 minimum bulk size is also constrained. If the calculated
288 bulk size is less then the minimum, then don't send
291 const uint32_t min_bulk = task->min_bulk_size;
292 uint32_t max_bulk = task->max_bulk_size;
294 if (task->pkt_count != (uint32_t)-1 && task->pkt_count < max_bulk) {
295 max_bulk = task->pkt_count;
298 uint32_t send_bulk = 0;
299 uint32_t pkt_idx_tmp = task->pkt_idx;
300 uint32_t would_send_bytes = 0;
304 * TODO - this must be improved to take into account the fact that, after applying randoms
305 * The packet can be replaced by an ARP
307 for (uint16_t j = 0; j < max_bulk; ++j) {
308 struct pkt_template *pktpl = &task->pkt_template[pkt_idx_tmp];
309 pkt_size = pktpl->len;
310 uint32_t pkt_len = pkt_len_to_wire_size(pkt_size);
311 if (pkt_len + would_send_bytes > task->token_time.bytes_now)
314 pkt_idx_tmp = task_gen_next_pkt_idx(task, pkt_idx_tmp);
317 would_send_bytes += pkt_len;
320 if (send_bulk < min_bulk)
322 *total_bytes = would_send_bytes;
326 static void task_gen_apply_random_fields(struct task_gen *task, uint8_t *hdr)
328 uint32_t ret, ret_tmp;
330 for (uint16_t i = 0; i < task->n_rands; ++i) {
331 ret = random_next(&task->rand[i].state);
332 ret_tmp = (ret & task->rand[i].rand_mask) | task->rand[i].fixed_bits;
334 ret_tmp = rte_bswap32(ret_tmp);
335 /* At this point, the lower order bytes (BE) contain
336 the generated value. The address where the values
337 of interest starts is at ret_tmp + 4 - rand_len. */
338 uint8_t *pret_tmp = (uint8_t*)&ret_tmp;
339 rte_memcpy(hdr + task->rand[i].rand_offset, pret_tmp + 4 - task->rand[i].rand_len, task->rand[i].rand_len);
343 static void task_gen_apply_all_random_fields(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
348 for (uint16_t i = 0; i < count; ++i)
349 task_gen_apply_random_fields(task, pkt_hdr[i]);
352 static void task_gen_apply_accur_pos(struct task_gen *task, uint8_t *pkt_hdr, uint32_t accuracy)
354 *(uint32_t *)(pkt_hdr + task->accur_pos) = accuracy;
357 static void task_gen_apply_sig(struct task_gen *task, struct pkt_template *dst)
360 *(uint32_t *)(dst->buf + task->sig_pos) = task->sig;
363 static void task_gen_apply_all_accur_pos(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
365 if (!task->accur_pos)
368 /* The accuracy of task->pkt_queue_index - ACCURACY_WINDOW is stored in
369 packet task->pkt_queue_index. The ID modulo ACCURACY_WINDOW is the
371 for (uint16_t j = 0; j < count; ++j) {
372 uint32_t accuracy = task->accur[(task->pkt_queue_index + j) & (ACCURACY_WINDOW - 1)];
373 task_gen_apply_accur_pos(task, pkt_hdr[j], accuracy);
377 static void task_gen_apply_unique_id(struct task_gen *task, uint8_t *pkt_hdr, const struct unique_id *id)
379 struct unique_id *dst = (struct unique_id *)(pkt_hdr + task->packet_id_pos);
384 static void task_gen_apply_all_unique_id(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
386 if (!task->packet_id_pos)
389 for (uint16_t i = 0; i < count; ++i) {
391 unique_id_init(&id, task->generator_id, task->pkt_queue_index++);
392 task_gen_apply_unique_id(task, pkt_hdr[i], &id);
396 static void task_gen_checksum_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
398 if (!(task->runtime_flags & TASK_TX_CRC))
401 if (!task->runtime_checksum_needed)
404 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - count);
405 for (uint16_t i = 0; i < count; ++i) {
406 struct pkt_template *pkt_template = &task->pkt_template[pkt_idx];
407 checksum_packet(pkt_hdr[i], mbufs[i], pkt_template, task->cksum_offload);
408 pkt_idx = task_gen_next_pkt_idx(task, pkt_idx);
412 static void task_gen_consume_tokens(struct task_gen *task, uint32_t tokens, uint32_t send_count)
414 /* If max burst has been sent, we can't keep up so just assume
415 that we can (leaving a "gap" in the packet stream on the
417 task->token_time.bytes_now -= tokens;
418 if (send_count == task->max_bulk_size && task->token_time.bytes_now > tokens) {
419 task->token_time.bytes_now = tokens;
423 static uint64_t task_gen_calc_bulk_duration(struct task_gen *task, uint32_t count)
425 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - 1);
426 struct pkt_template *last_pkt_template = &task->pkt_template[pkt_idx];
427 uint32_t last_pkt_len = pkt_len_to_wire_size(last_pkt_template->len);
428 #ifdef NO_EXTRAPOLATION
429 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1];
431 uint64_t last_pkt_duration = bytes_to_tsc(task, last_pkt_len);
432 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1] + last_pkt_duration;
435 return bulk_duration;
438 static uint64_t task_gen_write_latency(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
440 if (!task->lat_enabled)
443 uint64_t tx_tsc, delta_t;
444 uint64_t tsc_before_tx = 0;
446 /* Just before sending the packets, apply the time stamp
447 relative to when the first packet will be sent. The first
448 packet will be sent now. The time is read for each packet
449 to reduce the error towards the actual time the packet will
451 uint64_t write_tsc_after, write_tsc_before;
453 write_tsc_before = rte_rdtsc();
455 /* The time it took previously to write the time stamps in the
456 packets is used as an estimate for how long it will take to
457 write the time stamps now. The estimated time at which the
458 packets will actually be sent will be at tx_tsc. */
459 tx_tsc = write_tsc_before + task->write_duration_estimate;
461 /* The offset delta_t tracks the difference between the actual
462 time and the time written in the packets. Adding the offset
463 to the actual time insures that the time written in the
464 packets is monotonically increasing. At the same time,
465 simply sleeping until delta_t is zero would leave a period
466 of silence on the line. The error has been introduced
467 earlier, but the packets have already been sent. */
469 /* This happens typically if previous bulk was delayed
470 by an interrupt e.g. (with Time in nsec)
471 Time x: sleep 4 microsec
472 Time x+4000: send 64 packets (64 packets as 4000 nsec, w/ 10Gbps 64 bytes)
473 Time x+5000: send 16 packets (16 packets as 1000 nsec)
474 When we send the 16 packets, the 64 ealier packets are not yet
476 if (tx_tsc < task->earliest_tsc_next_pkt)
477 delta_t = task->earliest_tsc_next_pkt - tx_tsc;
481 for (uint16_t i = 0; i < count; ++i) {
482 uint32_t *pos = (uint32_t *)(pkt_hdr[i] + task->lat_pos);
483 const uint64_t pkt_tsc = tx_tsc + delta_t + task->pkt_tsc_offset[i];
484 *pos = pkt_tsc >> LATENCY_ACCURACY;
487 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, count);
488 task->earliest_tsc_next_pkt = tx_tsc + delta_t + bulk_duration;
489 write_tsc_after = rte_rdtsc();
490 task->write_duration_estimate = write_tsc_after - write_tsc_before;
492 /* Make sure that the time stamps that were written
493 are valid. The offset must be taken into account */
495 tsc_before_tx = rte_rdtsc();
496 } while (tsc_before_tx < tx_tsc);
498 return tsc_before_tx;
501 static void task_gen_store_accuracy(struct task_gen *task, uint32_t count, uint64_t tsc_before_tx)
503 if (!task->accur_pos)
506 uint64_t accur = rte_rdtsc() - tsc_before_tx;
507 uint64_t first_accuracy_idx = task->pkt_queue_index - count;
509 for (uint32_t i = 0; i < count; ++i) {
510 uint32_t accuracy_idx = (first_accuracy_idx + i) & (ACCURACY_WINDOW - 1);
512 task->accur[accuracy_idx] = accur;
516 static void task_gen_load_and_prefetch(struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
518 for (uint16_t i = 0; i < count; ++i)
519 rte_prefetch0(mbufs[i]);
520 for (uint16_t i = 0; i < count; ++i)
521 pkt_hdr[i] = rte_pktmbuf_mtod(mbufs[i], uint8_t *);
522 for (uint16_t i = 0; i < count; ++i)
523 rte_prefetch0(pkt_hdr[i]);
526 static void task_gen_build_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
528 uint64_t will_send_bytes = 0;
530 for (uint16_t i = 0; i < count; ++i) {
531 struct pkt_template *pktpl = &task->pkt_template[task->pkt_idx];
532 struct pkt_template *pkt_template = &task->pkt_template[task->pkt_idx];
533 pkt_template_init_mbuf(pkt_template, mbufs[i], pkt_hdr[i]);
534 mbufs[i]->udata64 = task->pkt_idx & TEMPLATE_INDEX_MASK;
535 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
536 if (task->lat_enabled) {
537 #ifdef NO_EXTRAPOLATION
538 task->pkt_tsc_offset[i] = 0;
540 task->pkt_tsc_offset[i] = bytes_to_tsc(task, will_send_bytes);
542 will_send_bytes += pkt_len_to_wire_size(pkt_template->len);
544 task->pkt_idx = task_gen_next_pkt_idx(task, task->pkt_idx);
548 static void task_gen_update_config(struct task_gen *task)
550 if (task->token_time.cfg.bpp != task->new_rate_bps)
551 task_gen_reset_token_time(task);
554 static inline void build_value(struct task_gen *task, uint32_t mask, int bit_pos, uint32_t val, uint32_t fixed_bits)
556 struct task_base *tbase = (struct task_base *)task;
558 build_value(task, mask >> 1, bit_pos + 1, val, fixed_bits);
560 build_value(task, mask >> 1, bit_pos + 1, val | (1 << bit_pos), fixed_bits);
563 register_ip_to_ctrl_plane(tbase->l3.tmaster, rte_cpu_to_be_32(val | fixed_bits), tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
566 static inline void register_all_ip_to_ctrl_plane(struct task_gen *task)
568 struct task_base *tbase = (struct task_base *)task;
573 for (uint32_t i = 0; i < task->n_pkts; ++i) {
574 struct pkt_template *pktpl = &task->pkt_template[i];
575 unsigned int ip_src_pos = 0;
577 unsigned int l2_len = sizeof(struct ether_hdr);
579 uint8_t *pkt = pktpl->buf;
580 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
581 uint16_t ether_type = eth_hdr->ether_type;
582 struct vlan_hdr *vlan_hdr;
585 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(struct vlan_hdr) < pktpl->len)) {
586 vlan_hdr = (struct vlan_hdr *)(pkt + l2_len);
588 ether_type = vlan_hdr->eth_proto;
590 if ((ether_type == ETYPE_MPLSU) || (ether_type == ETYPE_MPLSM)) {
594 if ((ether_type != ETYPE_IPv4) && !maybe_ipv4)
597 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + l2_len);
598 PROX_PANIC(ip->version_ihl >> 4 != 4, "IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
600 // Even if IPv4 header contains options, options are after ip src and dst
601 ip_src_pos = l2_len + sizeof(struct ipv4_hdr) - 2 * sizeof(uint32_t);
602 uint32_t *ip_src = ((uint32_t *)(pktpl->buf + ip_src_pos));
603 plog_info("\tip_src_pos = %d, ip_src = %x\n", ip_src_pos, *ip_src);
604 register_ip_to_ctrl_plane(tbase->l3.tmaster, *ip_src, tbase->l3.reachable_port_id, tbase->l3.core_id, tbase->l3.task_id);
606 for (int j = 0; j < task->n_rands; j++) {
607 offset = task->rand[j].rand_offset;
608 len = task->rand[j].rand_len;
609 mask = task->rand[j].rand_mask;
610 fixed = task->rand[j].fixed_bits;
611 plog_info("offset = %d, len = %d, mask = %x, fixed = %x\n", offset, len, mask, fixed);
612 if ((offset < ip_src_pos + 4) && (offset + len >= ip_src_pos)) {
613 if (offset >= ip_src_pos) {
614 int32_t ip_src_mask = (1 << (4 + ip_src_pos - offset) * 8) - 1;
615 mask = mask & ip_src_mask;
616 fixed = (fixed & ip_src_mask) | (rte_be_to_cpu_32(*ip_src) & ~ip_src_mask);
617 build_value(task, mask, 0, 0, fixed);
619 int32_t bits = ((ip_src_pos + 4 - offset - len) * 8);
621 fixed = (fixed << bits) | (rte_be_to_cpu_32(*ip_src) & ((1 << bits) - 1));
622 build_value(task, mask, 0, 0, fixed);
629 static int handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
631 struct task_gen *task = (struct task_gen *)tbase;
632 uint8_t out[MAX_PKT_BURST] = {0};
637 task_gen_update_config(task);
639 if (task->pkt_count == 0) {
640 task_gen_reset_token_time(task);
643 if (!task->token_time.cfg.bpp)
646 token_time_update(&task->token_time, rte_rdtsc());
648 uint32_t would_send_bytes;
649 uint32_t send_bulk = task_gen_calc_send_bulk(task, &would_send_bytes);
653 task_gen_take_count(task, send_bulk);
654 task_gen_consume_tokens(task, would_send_bytes, send_bulk);
656 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
657 if (new_pkts == NULL)
659 uint8_t *pkt_hdr[MAX_RING_BURST];
661 task_gen_load_and_prefetch(new_pkts, pkt_hdr, send_bulk);
662 task_gen_build_packets(task, new_pkts, pkt_hdr, send_bulk);
663 task_gen_apply_all_random_fields(task, pkt_hdr, send_bulk);
664 task_gen_apply_all_accur_pos(task, new_pkts, pkt_hdr, send_bulk);
665 task_gen_apply_all_unique_id(task, new_pkts, pkt_hdr, send_bulk);
667 uint64_t tsc_before_tx;
669 tsc_before_tx = task_gen_write_latency(task, pkt_hdr, send_bulk);
670 task_gen_checksum_packets(task, new_pkts, pkt_hdr, send_bulk);
671 ret = task->base.tx_pkt(&task->base, new_pkts, send_bulk, out);
672 task_gen_store_accuracy(task, send_bulk, tsc_before_tx);
674 // If we failed to send some packets, we need to do some clean-up:
677 // We need re-use the packets indexes not being sent
678 // Hence non-sent packets will not be considered as lost by the receiver when it looks at
679 // packet ids. This should also increase the percentage of packets used for latency measurements
680 task->pkt_queue_index -= ret;
682 // In case of failures, the estimate about when we can send next packet (earliest_tsc_next_pkt) is wrong
683 // This would result in under-estimated latency (up to 0 or negative)
684 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, ret);
685 task->earliest_tsc_next_pkt -= bulk_duration;
690 static void init_task_gen_seeds(struct task_gen *task)
692 for (size_t i = 0; i < sizeof(task->rand)/sizeof(task->rand[0]); ++i)
693 random_init_seed(&task->rand[i].state);
696 static uint32_t pcap_count_pkts(pcap_t *handle, uint32_t *max_frame_size)
698 struct pcap_pkthdr header;
702 long pkt1_fpos = ftell(pcap_file(handle));
704 while ((buf = pcap_next(handle, &header))) {
705 if (header.len > *max_frame_size)
706 *max_frame_size = header.len;
709 int ret2 = fseek(pcap_file(handle), pkt1_fpos, SEEK_SET);
710 PROX_PANIC(ret2 != 0, "Failed to reset reading pcap file\n");
714 static uint64_t avg_time_stamp(uint64_t *time_stamp, uint32_t n)
716 uint64_t tot_inter_pkt = 0;
718 for (uint32_t i = 0; i < n; ++i)
719 tot_inter_pkt += time_stamp[i];
720 return (tot_inter_pkt + n / 2)/n;
723 static int pcap_read_pkts(pcap_t *handle, const char *file_name, uint32_t n_pkts, struct pkt_template *proto, uint64_t *time_stamp)
725 struct pcap_pkthdr header;
729 for (uint32_t i = 0; i < n_pkts; ++i) {
730 buf = pcap_next(handle, &header);
732 PROX_PANIC(buf == NULL, "Failed to read packet %d from pcap %s\n", i, file_name);
733 proto[i].len = header.len;
734 len = RTE_MIN(header.len, sizeof(proto[i].buf));
735 if (header.len > len)
736 plogx_warn("Packet truncated from %u to %zu bytes\n", header.len, len);
739 static struct timeval beg;
745 tv = tv_diff(&beg, &header.ts);
746 tv_to_tsc(&tv, time_stamp + i);
748 rte_memcpy(proto[i].buf, buf, len);
751 if (time_stamp && n_pkts) {
752 for (uint32_t i = n_pkts - 1; i > 0; --i)
753 time_stamp[i] -= time_stamp[i - 1];
754 /* Since the handle function will loop the packets,
755 there is one time-stamp that is not provided by the
756 pcap file. This is the time between the last and
757 the first packet. This implementation takes the
758 average of the inter-packet times here. */
760 time_stamp[0] = avg_time_stamp(time_stamp + 1, n_pkts - 1);
766 static int check_pkt_size(struct task_gen *task, uint32_t pkt_size, int do_panic)
768 const uint16_t min_len = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr);
769 const uint16_t max_len = task->max_frame_size;
772 PROX_PANIC(pkt_size == 0, "Invalid packet size length (no packet defined?)\n");
773 PROX_PANIC(pkt_size > max_len, "pkt_size out of range (must be <= %u)\n", max_len);
774 PROX_PANIC(pkt_size < min_len, "pkt_size out of range (must be >= %u)\n", min_len);
778 plog_err("Invalid packet size length (no packet defined?)\n");
781 if (pkt_size > max_len) {
782 plog_err("pkt_size out of range (must be <= %u)\n", max_len);
785 if (pkt_size < min_len) {
786 plog_err("pkt_size out of range (must be >= %u)\n", min_len);
793 static int check_all_pkt_size(struct task_gen *task, int do_panic)
796 for (uint32_t i = 0; i < task->n_pkts;++i) {
797 if ((rc = check_pkt_size(task, task->pkt_template[i].len, do_panic)) != 0)
803 static int check_fields_in_bounds(struct task_gen *task, uint32_t pkt_size, int do_panic)
805 if (task->lat_enabled) {
806 uint32_t pos_beg = task->lat_pos;
807 uint32_t pos_end = task->lat_pos + 3U;
810 PROX_PANIC(pkt_size <= pos_end, "Writing latency at %u-%u, but packet size is %u bytes\n",
811 pos_beg, pos_end, pkt_size);
812 else if (pkt_size <= pos_end) {
813 plog_err("Writing latency at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
817 if (task->packet_id_pos) {
818 uint32_t pos_beg = task->packet_id_pos;
819 uint32_t pos_end = task->packet_id_pos + 4U;
822 PROX_PANIC(pkt_size <= pos_end, "Writing packet at %u-%u, but packet size is %u bytes\n",
823 pos_beg, pos_end, pkt_size);
824 else if (pkt_size <= pos_end) {
825 plog_err("Writing packet at %u-%u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
829 if (task->accur_pos) {
830 uint32_t pos_beg = task->accur_pos;
831 uint32_t pos_end = task->accur_pos + 3U;
834 PROX_PANIC(pkt_size <= pos_end, "Writing accuracy at %u%-u, but packet size is %u bytes\n",
835 pos_beg, pos_end, pkt_size);
836 else if (pkt_size <= pos_end) {
837 plog_err("Writing accuracy at %u%-u, but packet size is %u bytes\n", pos_beg, pos_end, pkt_size);
844 static void task_gen_pkt_template_recalc_metadata(struct task_gen *task)
846 struct pkt_template *template;
848 for (size_t i = 0; i < task->n_pkts; ++i) {
849 template = &task->pkt_template[i];
850 parse_l2_l3_len(template->buf, &template->l2_len, &template->l3_len, template->len);
854 static void task_gen_pkt_template_recalc_checksum(struct task_gen *task)
856 struct pkt_template *template;
859 task->runtime_checksum_needed = 0;
860 for (size_t i = 0; i < task->n_pkts; ++i) {
861 template = &task->pkt_template[i];
862 if (template->l2_len == 0)
864 ip = (struct ipv4_hdr *)(template->buf + template->l2_len);
866 ip->hdr_checksum = 0;
867 prox_ip_cksum_sw(ip);
868 uint32_t l4_len = rte_bswap16(ip->total_length) - template->l3_len;
870 if (ip->next_proto_id == IPPROTO_UDP) {
871 struct udp_hdr *udp = (struct udp_hdr *)(((uint8_t *)ip) + template->l3_len);
872 prox_udp_cksum_sw(udp, l4_len, ip->src_addr, ip->dst_addr);
873 } else if (ip->next_proto_id == IPPROTO_TCP) {
874 struct tcp_hdr *tcp = (struct tcp_hdr *)(((uint8_t *)ip) + template->l3_len);
875 prox_tcp_cksum_sw(tcp, l4_len, ip->src_addr, ip->dst_addr);
878 /* The current implementation avoids checksum
879 calculation by determining that at packet
880 construction time, no fields are applied that would
881 require a recalculation of the checksum. */
882 if (task->lat_enabled && task->lat_pos > template->l2_len)
883 task->runtime_checksum_needed = 1;
884 if (task->accur_pos > template->l2_len)
885 task->runtime_checksum_needed = 1;
886 if (task->packet_id_pos > template->l2_len)
887 task->runtime_checksum_needed = 1;
891 static void task_gen_pkt_template_recalc_all(struct task_gen *task)
893 task_gen_pkt_template_recalc_metadata(task);
894 task_gen_pkt_template_recalc_checksum(task);
897 static void task_gen_reset_pkt_templates_len(struct task_gen *task)
899 struct pkt_template *src, *dst;
901 for (size_t i = 0; i < task->n_pkts; ++i) {
902 src = &task->pkt_template_orig[i];
903 dst = &task->pkt_template[i];
908 static void task_gen_reset_pkt_templates_content(struct task_gen *task)
910 struct pkt_template *src, *dst;
912 for (size_t i = 0; i < task->n_pkts; ++i) {
913 src = &task->pkt_template_orig[i];
914 dst = &task->pkt_template[i];
915 memcpy(dst->buf, src->buf, RTE_MAX(src->len, dst->len));
916 task_gen_apply_sig(task, dst);
920 static void task_gen_reset_pkt_templates(struct task_gen *task)
922 task_gen_reset_pkt_templates_len(task);
923 task_gen_reset_pkt_templates_content(task);
924 task_gen_pkt_template_recalc_all(task);
927 static void task_init_gen_load_pkt_inline(struct task_gen *task, struct task_args *targ)
929 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
933 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
934 task->pkt_template = prox_zmalloc(mem_size, socket_id);
935 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
937 PROX_PANIC(task->pkt_template == NULL ||
938 task->pkt_template_orig == NULL,
939 "Failed to allocate %lu bytes (in huge pages) for packet template\n", mem_size);
941 task->pkt_template->buf = prox_zmalloc(task->max_frame_size, socket_id);
942 task->pkt_template_orig->buf = prox_zmalloc(task->max_frame_size, socket_id);
943 PROX_PANIC(task->pkt_template->buf == NULL ||
944 task->pkt_template_orig->buf == NULL,
945 "Failed to allocate %u bytes (in huge pages) for packet\n", task->max_frame_size);
947 PROX_PANIC(targ->pkt_size > task->max_frame_size,
948 targ->pkt_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE - 4 ?
949 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
951 rte_memcpy(task->pkt_template_orig[0].buf, targ->pkt_inline, targ->pkt_size);
952 task->pkt_template_orig[0].len = targ->pkt_size;
953 task_gen_reset_pkt_templates(task);
954 check_all_pkt_size(task, 1);
955 check_fields_in_bounds(task, task->pkt_template[0].len, 1);
958 static void task_init_gen_load_pcap(struct task_gen *task, struct task_args *targ)
960 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
961 char err[PCAP_ERRBUF_SIZE];
962 uint32_t max_frame_size;
963 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
964 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
966 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
967 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
968 PROX_PANIC(max_frame_size > task->max_frame_size,
969 max_frame_size > ETHER_MAX_LEN + 2 * PROX_VLAN_TAG_SIZE -4 ?
970 "pkt_size too high and jumbo frames disabled" : "pkt_size > mtu");
973 task->n_pkts = RTE_MIN(task->n_pkts, targ->n_pkts);
974 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
975 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
976 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
977 task->pkt_template = prox_zmalloc(mem_size, socket_id);
978 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
979 PROX_PANIC(task->pkt_template == NULL ||
980 task->pkt_template_orig == NULL,
981 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
983 for (uint i = 0; i < task->n_pkts; i++) {
984 task->pkt_template[i].buf = prox_zmalloc(max_frame_size, socket_id);
985 task->pkt_template_orig[i].buf = prox_zmalloc(max_frame_size, socket_id);
987 PROX_PANIC(task->pkt_template->buf == NULL ||
988 task->pkt_template_orig->buf == NULL,
989 "Failed to allocate %u bytes (in huge pages) for pcap file\n", task->max_frame_size);
992 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->pkt_template_orig, NULL);
994 task_gen_reset_pkt_templates(task);
997 static struct rte_mempool *task_gen_create_mempool(struct task_args *targ, uint16_t max_frame_size)
999 static char name[] = "gen_pool";
1000 struct rte_mempool *ret;
1001 const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
1004 uint32_t mbuf_size = TX_MBUF_SIZE;
1005 if (max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM > mbuf_size)
1006 mbuf_size = max_frame_size + (unsigned)sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM;
1007 plog_info("\t\tCreating mempool with name '%s'\n", name);
1008 ret = rte_mempool_create(name, targ->nb_mbuf - 1, mbuf_size,
1009 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
1010 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
1012 PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
1013 sock_id, targ->nb_mbuf - 1);
1015 plog_info("\t\tMempool %p size = %u * %u cache %u, socket %d\n", ret,
1016 targ->nb_mbuf - 1, mbuf_size, targ->nb_cache_mbuf, sock_id);
1021 void task_gen_set_pkt_count(struct task_base *tbase, uint32_t count)
1023 struct task_gen *task = (struct task_gen *)tbase;
1025 task->pkt_count = count;
1028 int task_gen_set_pkt_size(struct task_base *tbase, uint32_t pkt_size)
1030 struct task_gen *task = (struct task_gen *)tbase;
1033 if ((rc = check_pkt_size(task, pkt_size, 0)) != 0)
1035 if ((rc = check_fields_in_bounds(task, pkt_size, 0)) != 0)
1037 task->pkt_template[0].len = pkt_size;
1041 void task_gen_set_rate(struct task_base *tbase, uint64_t bps)
1043 struct task_gen *task = (struct task_gen *)tbase;
1045 task->new_rate_bps = bps;
1048 void task_gen_reset_randoms(struct task_base *tbase)
1050 struct task_gen *task = (struct task_gen *)tbase;
1052 for (uint32_t i = 0; i < task->n_rands; ++i) {
1053 task->rand[i].rand_mask = 0;
1054 task->rand[i].fixed_bits = 0;
1055 task->rand[i].rand_offset = 0;
1060 int task_gen_set_value(struct task_base *tbase, uint32_t value, uint32_t offset, uint32_t len)
1062 struct task_gen *task = (struct task_gen *)tbase;
1064 for (size_t i = 0; i < task->n_pkts; ++i) {
1065 uint32_t to_write = rte_cpu_to_be_32(value) >> ((4 - len) * 8);
1066 uint8_t *dst = task->pkt_template[i].buf;
1068 rte_memcpy(dst + offset, &to_write, len);
1071 task_gen_pkt_template_recalc_all(task);
1076 void task_gen_reset_values(struct task_base *tbase)
1078 struct task_gen *task = (struct task_gen *)tbase;
1080 task_gen_reset_pkt_templates_content(task);
1081 if (task->flags & TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC) {
1082 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1083 rte_memcpy(&task->pkt_template[i].buf[sizeof(struct ether_addr)], &task->src_mac, sizeof(struct ether_addr));
1088 uint32_t task_gen_get_n_randoms(struct task_base *tbase)
1090 struct task_gen *task = (struct task_gen *)tbase;
1092 return task->n_rands;
1095 static void init_task_gen_pcap(struct task_base *tbase, struct task_args *targ)
1097 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1098 const uint32_t sockid = rte_lcore_to_socket_id(targ->lconf->id);
1099 uint32_t max_frame_size;
1101 task->loop = targ->loop;
1103 task->hz = rte_get_tsc_hz();
1105 char err[PCAP_ERRBUF_SIZE];
1106 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1107 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1109 task->n_pkts = pcap_count_pkts(handle, &max_frame_size);
1110 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1112 task->local_mbuf.mempool = task_gen_create_mempool(targ, max_frame_size);
1114 PROX_PANIC(!strcmp(targ->pcap_file, ""), "No pcap file defined\n");
1117 plogx_info("Configured to load %u packets\n", targ->n_pkts);
1118 if (task->n_pkts > targ->n_pkts)
1119 task->n_pkts = targ->n_pkts;
1121 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
1123 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1125 size_t mem_size = task->n_pkts * (sizeof(*task->proto) + sizeof(*task->proto_tsc));
1126 uint8_t *mem = prox_zmalloc(mem_size, sockid);
1128 PROX_PANIC(mem == NULL, "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1129 task->proto = (struct pkt_template *) mem;
1130 task->proto_tsc = (uint64_t *)(mem + task->n_pkts * sizeof(*task->proto));
1132 for (uint i = 0; i < targ->n_pkts; i++) {
1133 task->proto[i].buf = prox_zmalloc(max_frame_size, sockid);
1134 PROX_PANIC(task->proto[i].buf == NULL, "Failed to allocate %u bytes (in huge pages) for pcap file\n", max_frame_size);
1137 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->proto, task->proto_tsc);
1141 static int task_gen_find_random_with_offset(struct task_gen *task, uint32_t offset)
1143 for (uint32_t i = 0; i < task->n_rands; ++i) {
1144 if (task->rand[i].rand_offset == offset) {
1152 int task_gen_add_rand(struct task_base *tbase, const char *rand_str, uint32_t offset, uint32_t rand_id)
1154 struct task_gen *task = (struct task_gen *)tbase;
1155 uint32_t existing_rand;
1157 if (rand_id == UINT32_MAX && task->n_rands == 64) {
1158 plog_err("Too many randoms\n");
1161 uint32_t mask, fixed, len;
1163 if (parse_random_str(&mask, &fixed, &len, rand_str)) {
1164 plog_err("%s\n", get_parse_err());
1167 task->runtime_checksum_needed = 1;
1169 existing_rand = task_gen_find_random_with_offset(task, offset);
1170 if (existing_rand != UINT32_MAX) {
1171 plog_warn("Random at offset %d already set => overwriting len = %d %s\n", offset, len, rand_str);
1172 rand_id = existing_rand;
1173 task->rand[rand_id].rand_len = len;
1174 task->rand[rand_id].rand_offset = offset;
1175 task->rand[rand_id].rand_mask = mask;
1176 task->rand[rand_id].fixed_bits = fixed;
1180 task->rand[task->n_rands].rand_len = len;
1181 task->rand[task->n_rands].rand_offset = offset;
1182 task->rand[task->n_rands].rand_mask = mask;
1183 task->rand[task->n_rands].fixed_bits = fixed;
1189 static void start(struct task_base *tbase)
1191 struct task_gen *task = (struct task_gen *)tbase;
1192 task->pkt_queue_index = 0;
1194 task_gen_reset_token_time(task);
1195 if (tbase->l3.tmaster) {
1196 register_all_ip_to_ctrl_plane(task);
1200 Handle the case when two tasks transmit to the same port
1201 and one of them is stopped. In that case ARP (requests or replies)
1202 might not be sent. Master will have to keep a list of rings.
1203 stop will have to de-register IP from ctrl plane.
1204 un-registration will remove the ring. when having more than
1205 one active rings, master can always use the first one
1209 static void start_pcap(struct task_base *tbase)
1211 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1212 /* When we start, the first packet is sent immediately. */
1213 task->last_tsc = rte_rdtsc() - task->proto_tsc[0];
1217 static void init_task_gen_early(struct task_args *targ)
1219 uint8_t *generator_count = prox_sh_find_system("generator_count");
1221 if (generator_count == NULL) {
1222 generator_count = prox_zmalloc(sizeof(*generator_count), rte_lcore_to_socket_id(targ->lconf->id));
1223 PROX_PANIC(generator_count == NULL, "Failed to allocate generator count\n");
1224 prox_sh_add_system("generator_count", generator_count);
1226 targ->generator_id = *generator_count;
1227 (*generator_count)++;
1230 static void init_task_gen(struct task_base *tbase, struct task_args *targ)
1232 struct task_gen *task = (struct task_gen *)tbase;
1234 task->packet_id_pos = targ->packet_id_pos;
1236 struct prox_port_cfg *port = find_reachable_port(targ);
1237 // TODO: check that all reachable ports have the same mtu...
1239 task->cksum_offload = port->requested_tx_offload & (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM);
1241 task->max_frame_size = port->mtu + ETHER_HDR_LEN + 2 * PROX_VLAN_TAG_SIZE;
1243 // Not generating to any port...
1244 task->max_frame_size = ETHER_MAX_LEN;
1246 task->local_mbuf.mempool = task_gen_create_mempool(targ, task->max_frame_size);
1247 PROX_PANIC(task->local_mbuf.mempool == NULL, "Failed to create mempool\n");
1249 task->hz = rte_get_tsc_hz();
1250 task->lat_pos = targ->lat_pos;
1251 task->accur_pos = targ->accur_pos;
1252 task->sig_pos = targ->sig_pos;
1253 task->sig = targ->sig;
1254 task->new_rate_bps = targ->rate_bps;
1257 * For tokens, use 10 Gbps as base rate
1258 * Scripts can then use speed command, with speed=100 as 10 Gbps and speed=400 as 40 Gbps
1259 * Script can query prox "port info" command to find out the port link speed to know
1260 * at which rate to start. Note that virtio running on OVS returns 10 Gbps, so a script has
1261 * probably also to check the driver (as returned by the same "port info" command.
1263 struct token_time_cfg tt_cfg = token_time_cfg_create(1250000000, rte_get_tsc_hz(), -1);
1264 token_time_init(&task->token_time, &tt_cfg);
1266 init_task_gen_seeds(task);
1268 task->min_bulk_size = targ->min_bulk_size;
1269 task->max_bulk_size = targ->max_bulk_size;
1270 if (task->min_bulk_size < 1)
1271 task->min_bulk_size = 1;
1272 if (task->max_bulk_size < 1)
1273 task->max_bulk_size = 64;
1274 PROX_PANIC(task->max_bulk_size > 64, "max_bulk_size higher than 64\n");
1275 PROX_PANIC(task->max_bulk_size < task->min_bulk_size, "max_bulk_size must be > than min_bulk_size\n");
1277 task->pkt_count = -1;
1278 task->lat_enabled = targ->lat_enabled;
1279 task->runtime_flags = targ->runtime_flags;
1280 PROX_PANIC((task->lat_pos || task->accur_pos) && !task->lat_enabled, "lat not enabled by lat pos or accur pos configured\n");
1282 task->generator_id = targ->generator_id;
1283 plog_info("\tGenerator id = %d\n", task->generator_id);
1285 // Allocate array holding bytes to tsc for supported frame sizes
1286 task->bytes_to_tsc = prox_zmalloc(task->max_frame_size * MAX_PKT_BURST * sizeof(task->bytes_to_tsc[0]), rte_lcore_to_socket_id(targ->lconf->id));
1287 PROX_PANIC(task->bytes_to_tsc == NULL,
1288 "Failed to allocate %u bytes (in huge pages) for bytes_to_tsc\n", task->max_frame_size);
1290 // task->port->max_link_speed reports the maximum, non negotiated ink speed in Mbps e.g. 40k for a 40 Gbps NIC.
1291 // It can be UINT32_MAX (virtual devices or not supported by DPDK < 16.04)
1292 uint64_t bytes_per_hz = UINT64_MAX;
1293 if ((task->port) && (task->port->max_link_speed != UINT32_MAX)) {
1294 bytes_per_hz = task->port->max_link_speed * 125000L;
1295 plog_info("\tPort %u: max link speed is %ld Mbps\n",
1296 (uint8_t)(task->port - prox_port_cfg), 8 * bytes_per_hz / 1000000);
1298 // There are cases where hz estimate might be slighly over-estimated
1299 // This results in too much extrapolation
1300 // Only account for 99% of extrapolation to handle cases with up to 1% error clocks
1301 for (unsigned int i = 0; i < task->max_frame_size * MAX_PKT_BURST ; i++) {
1302 if (bytes_per_hz == UINT64_MAX)
1303 task->bytes_to_tsc[i] = 0;
1305 task->bytes_to_tsc[i] = (task->hz * i * 0.99) / bytes_per_hz;
1308 if (!strcmp(targ->pcap_file, "")) {
1309 plog_info("\tUsing inline definition of a packet\n");
1310 task_init_gen_load_pkt_inline(task, targ);
1312 plog_info("Loading from pcap %s\n", targ->pcap_file);
1313 task_init_gen_load_pcap(task, targ);
1316 PROX_PANIC(((targ->nb_txrings == 0) && (targ->nb_txports == 0)), "Gen mode requires a tx ring or a tx port");
1317 if ((targ->flags & DSF_KEEP_SRC_MAC) == 0) {
1318 task->flags |= TASK_OVERWRITE_SRC_MAC_WITH_PORT_MAC;
1319 memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(struct ether_addr));
1320 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1321 rte_memcpy(&task->pkt_template[i].buf[sizeof(struct ether_addr)], &task->src_mac, sizeof(struct ether_addr));
1324 for (uint32_t i = 0; i < targ->n_rand_str; ++i) {
1325 PROX_PANIC(task_gen_add_rand(tbase, targ->rand_str[i], targ->rand_offset[i], UINT32_MAX),
1326 "Failed to add random\n");
1330 static struct task_init task_init_gen = {
1332 .init = init_task_gen,
1333 .handle = handle_gen_bulk,
1335 .early_init = init_task_gen_early,
1337 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1338 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1339 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1341 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1343 .size = sizeof(struct task_gen)
1346 static struct task_init task_init_gen_l3 = {
1348 .sub_mode_str = "l3",
1349 .init = init_task_gen,
1350 .handle = handle_gen_bulk,
1352 .early_init = init_task_gen_early,
1354 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1355 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1356 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1358 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1360 .size = sizeof(struct task_gen)
1363 static struct task_init task_init_gen_pcap = {
1365 .sub_mode_str = "pcap",
1366 .init = init_task_gen_pcap,
1367 .handle = handle_gen_pcap_bulk,
1368 .start = start_pcap,
1369 .early_init = init_task_gen_early,
1371 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS,
1373 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1375 .size = sizeof(struct task_gen_pcap)
1378 __attribute__((constructor)) static void reg_task_gen(void)
1380 reg_task(&task_init_gen);
1381 reg_task(&task_init_gen_l3);
1382 reg_task(&task_init_gen_pcap);