2 // Copyright (c) 2010-2017 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
21 #include <rte_cycles.h>
22 #include <rte_version.h>
23 #include <rte_byteorder.h>
24 #include <rte_ether.h>
26 #include "prox_shared.h"
28 #include "prox_malloc.h"
29 #include "handle_gen.h"
30 #include "handle_lat.h"
31 #include "task_init.h"
32 #include "task_base.h"
33 #include "prox_port_cfg.h"
38 #include "mbuf_utils.h"
40 #include "prox_cksum.h"
42 #include "prox_assert.h"
44 #include "token_time.h"
45 #include "local_mbuf.h"
48 #include <rte_hash_crc.h>
57 uint8_t buf[ETHER_MAX_LEN];
60 #define FLAG_DST_MAC_KNOWN 1
62 #define FLAG_RANDOM_IPS 4
64 #define MAX_TEMPLATE_INDEX 65536
65 #define TEMPLATE_INDEX_MASK (MAX_TEMPLATE_INDEX - 1)
66 #define MBUF_ARP MAX_TEMPLATE_INDEX
68 #define IP4(x) x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, x >> 24
70 static void pkt_template_init_mbuf(struct pkt_template *pkt_template, struct rte_mbuf *mbuf, uint8_t *pkt)
72 const uint32_t pkt_size = pkt_template->len;
74 rte_pktmbuf_pkt_len(mbuf) = pkt_size;
75 rte_pktmbuf_data_len(mbuf) = pkt_size;
77 rte_memcpy(pkt, pkt_template->buf, pkt_template->len);
80 struct task_gen_pcap {
81 struct task_base base;
83 struct local_mbuf local_mbuf;
85 struct pkt_template *proto;
93 struct task_base base;
96 struct token_time token_time;
97 struct local_mbuf local_mbuf;
98 struct pkt_template *pkt_template; /* packet templates used at runtime */
99 uint64_t write_duration_estimate; /* how long it took previously to write the time stamps in the packets */
100 uint64_t earliest_tsc_next_pkt;
101 uint64_t new_rate_bps;
102 uint64_t pkt_queue_index;
103 uint32_t n_pkts; /* number of packets in pcap */
104 uint32_t pkt_idx; /* current packet from pcap */
105 uint32_t pkt_count; /* how many pakets to generate */
106 uint32_t runtime_flags;
108 uint16_t packet_id_pos;
112 uint8_t generator_id;
113 uint8_t n_rands; /* number of randoms */
114 uint8_t min_bulk_size;
115 uint8_t max_bulk_size;
117 uint8_t runtime_checksum_needed;
120 uint32_t rand_mask; /* since the random vals are uniform, masks don't introduce bias */
121 uint32_t fixed_bits; /* length of each random (max len = 4) */
122 uint16_t rand_offset; /* each random has an offset*/
123 uint8_t rand_len; /* # bytes to take from random (no bias introduced) */
126 uint64_t pkt_tsc_offset[64];
127 struct pkt_template *pkt_template_orig; /* packet templates (from inline or from pcap) */
128 struct ether_addr gw_mac;
129 struct ether_addr src_mac;
130 struct rte_hash *mac_hash;
135 uint8_t cksum_offload;
136 } __rte_cache_aligned;
138 static inline uint8_t ipv4_get_hdr_len(struct ipv4_hdr *ip)
140 /* Optimize for common case of IPv4 header without options. */
141 if (ip->version_ihl == 0x45)
142 return sizeof(struct ipv4_hdr);
143 if (unlikely(ip->version_ihl >> 4 != 4)) {
144 plog_warn("IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
147 return (ip->version_ihl & 0xF) * 4;
150 static void parse_l2_l3_len(uint8_t *pkt, uint16_t *l2_len, uint16_t *l3_len, uint16_t len)
152 *l2_len = sizeof(struct ether_hdr);
154 struct vlan_hdr *vlan_hdr;
155 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
157 uint16_t ether_type = eth_hdr->ether_type;
160 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (*l2_len + sizeof(struct vlan_hdr) < len)) {
161 vlan_hdr = (struct vlan_hdr *)(pkt + *l2_len);
163 ether_type = vlan_hdr->eth_proto;
166 // No L3 cksum offload for IPv6, but TODO L4 offload
167 // ETYPE_EoGRE CRC not implemented yet
169 switch (ether_type) {
183 plog_warn("Unsupported packet type %x - CRC might be wrong\n", ether_type);
188 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + *l2_len);
189 *l3_len = ipv4_get_hdr_len(ip);
193 static void checksum_packet(uint8_t *hdr, struct rte_mbuf *mbuf, struct pkt_template *pkt_template, int cksum_offload)
195 uint16_t l2_len = pkt_template->l2_len;
196 uint16_t l3_len = pkt_template->l3_len;
199 struct ipv4_hdr *ip = (struct ipv4_hdr*)(hdr + l2_len);
200 prox_ip_udp_cksum(mbuf, ip, l2_len, l3_len, cksum_offload);
204 static void task_gen_reset_token_time(struct task_gen *task)
206 token_time_set_bpp(&task->token_time, task->new_rate_bps);
207 token_time_reset(&task->token_time, rte_rdtsc(), 0);
210 static void start(struct task_base *tbase)
212 struct task_gen *task = (struct task_gen *)tbase;
213 task->pkt_queue_index = 0;
215 task_gen_reset_token_time(task);
218 static void start_pcap(struct task_base *tbase)
220 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
221 /* When we start, the first packet is sent immediately. */
222 task->last_tsc = rte_rdtsc() - task->proto_tsc[0];
226 static void task_gen_take_count(struct task_gen *task, uint32_t send_bulk)
228 if (task->pkt_count == (uint32_t)-1)
231 if (task->pkt_count >= send_bulk)
232 task->pkt_count -= send_bulk;
238 static int handle_gen_pcap_bulk(struct task_base *tbase, struct rte_mbuf **mbuf, uint16_t n_pkts)
240 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
241 uint64_t now = rte_rdtsc();
242 uint64_t send_bulk = 0;
243 uint32_t pkt_idx_tmp = task->pkt_idx;
245 if (pkt_idx_tmp == task->n_pkts) {
246 PROX_ASSERT(task->loop);
250 for (uint16_t j = 0; j < 64; ++j) {
251 uint64_t tsc = task->proto_tsc[pkt_idx_tmp];
252 if (task->last_tsc + tsc <= now) {
253 task->last_tsc += tsc;
256 if (pkt_idx_tmp == task->n_pkts) {
267 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
268 if (new_pkts == NULL)
271 for (uint16_t j = 0; j < send_bulk; ++j) {
272 struct rte_mbuf *next_pkt = new_pkts[j];
273 struct pkt_template *pkt_template = &task->proto[task->pkt_idx];
274 uint8_t *hdr = rte_pktmbuf_mtod(next_pkt, uint8_t *);
276 pkt_template_init_mbuf(pkt_template, next_pkt, hdr);
279 if (task->pkt_idx == task->n_pkts) {
287 return task->base.tx_pkt(&task->base, new_pkts, send_bulk, NULL);
290 static uint64_t bytes_to_tsc(struct task_gen *task, uint32_t bytes)
292 const uint64_t hz = task->hz;
293 const uint64_t bytes_per_hz = task->link_speed;
295 if (bytes_per_hz == UINT64_MAX)
298 return hz * bytes / bytes_per_hz;
301 static uint32_t task_gen_next_pkt_idx(const struct task_gen *task, uint32_t pkt_idx)
303 return pkt_idx + 1 == task->n_pkts? 0 : pkt_idx + 1;
306 static uint32_t task_gen_offset_pkt_idx(const struct task_gen *task, uint32_t offset)
308 return (task->pkt_idx + offset) % task->n_pkts;
311 static uint32_t task_gen_calc_send_bulk(const struct task_gen *task, uint32_t *total_bytes)
313 /* The biggest bulk we allow to send is task->max_bulk_size
314 packets. The max bulk size can also be limited by the
315 pkt_count field. At the same time, we are rate limiting
316 based on the specified speed (in bytes per second) so token
317 bucket based rate limiting must also be applied. The
318 minimum bulk size is also constrained. If the calculated
319 bulk size is less then the minimum, then don't send
322 const uint32_t min_bulk = task->min_bulk_size;
323 uint32_t max_bulk = task->max_bulk_size;
325 if (task->pkt_count != (uint32_t)-1 && task->pkt_count < max_bulk) {
326 max_bulk = task->pkt_count;
329 uint32_t send_bulk = 0;
330 uint32_t pkt_idx_tmp = task->pkt_idx;
331 uint32_t would_send_bytes = 0;
335 * TODO - this must be improved to take into account the fact that, after applying randoms
336 * The packet can be replaced by an ARP
338 for (uint16_t j = 0; j < max_bulk; ++j) {
339 struct pkt_template *pktpl = &task->pkt_template[pkt_idx_tmp];
340 if (unlikely((task->flags & (FLAG_L3_GEN | FLAG_DST_MAC_KNOWN)) == FLAG_L3_GEN)) {
341 // Generator is supposed to get MAC address - MAC is still unknown for this template
342 // generate ARP Request to gateway instead of the intended packet
345 pkt_size = pktpl->len;
347 uint32_t pkt_len = pkt_len_to_wire_size(pkt_size);
348 if (pkt_len + would_send_bytes > task->token_time.bytes_now)
351 pkt_idx_tmp = task_gen_next_pkt_idx(task, pkt_idx_tmp);
354 would_send_bytes += pkt_len;
357 if (send_bulk < min_bulk)
359 *total_bytes = would_send_bytes;
363 static inline void create_arp(struct rte_mbuf *mbuf, uint8_t *pkt_hdr, uint64_t *src_mac, uint32_t ip_dst, uint32_t ip_src)
365 uint64_t mac_bcast = 0xFFFFFFFFFFFF;
366 rte_pktmbuf_pkt_len(mbuf) = 42;
367 rte_pktmbuf_data_len(mbuf) = 42;
369 struct ether_hdr_arp *hdr_arp = (struct ether_hdr_arp *)pkt_hdr;
371 memcpy(&hdr_arp->ether_hdr.d_addr.addr_bytes, &mac_bcast, 6);
372 memcpy(&hdr_arp->ether_hdr.s_addr.addr_bytes, src_mac, 6);
373 hdr_arp->ether_hdr.ether_type = ETYPE_ARP;
374 hdr_arp->arp.htype = 0x100,
375 hdr_arp->arp.ptype = 0x0008;
376 hdr_arp->arp.hlen = 6;
377 hdr_arp->arp.plen = 4;
378 hdr_arp->arp.oper = 0x100;
379 hdr_arp->arp.data.spa = ip_src;
380 hdr_arp->arp.data.tpa = ip_dst;
381 memset(&hdr_arp->arp.data.tha, 0, sizeof(struct ether_addr));
382 memcpy(&hdr_arp->arp.data.sha, src_mac, sizeof(struct ether_addr));
385 static int task_gen_write_dst_mac(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
387 uint32_t ip_dst_pos, ip_src_pos, ip_dst, ip_src;
391 if (task->flags & FLAG_L3_GEN) {
393 if (unlikely((task->flags & FLAG_DST_MAC_KNOWN) == 0)) {
394 for (i = 0; i < count; ++i) {
395 struct pkt_template *pktpl = &task->pkt_template[mbufs[i]->udata64 & TEMPLATE_INDEX_MASK];
396 create_arp(mbufs[i], pkt_hdr[i], (uint64_t *)&pktpl->buf[6], task->gw_ip, pktpl->ip_src);
397 mbufs[i]->udata64 |= MBUF_ARP;
400 for (i = 0; i < count; ++i) {
401 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
402 memcpy(&hdr->d_addr.addr_bytes, &task->gw_mac, 6);
405 } else if (unlikely((task->flags & FLAG_RANDOM_IPS) != 0) || (task->n_pkts >= 4)){
406 // Find mac in lookup table. Send ARP if not found
407 int32_t positions[MAX_PKT_BURST], idx;
408 void *keys[MAX_PKT_BURST];
409 uint32_t key[MAX_PKT_BURST];
410 for (i = 0; i < count; ++i) {
411 uint8_t *hdr = (uint8_t *)pkt_hdr[i];
412 struct pkt_template *pktpl = &task->pkt_template[mbufs[i]->udata64 & TEMPLATE_INDEX_MASK];
413 ip_dst_pos = pktpl->ip_dst_pos;
414 ip_dst = *(uint32_t *)(hdr + ip_dst_pos);
418 ret = rte_hash_lookup_bulk(task->mac_hash, (const void **)&keys, count, positions);
419 if (unlikely(ret < 0)) {
420 plogx_err("lookup_bulk failed in mac_hash\n");
421 tx_pkt_drop_all((struct task_base *)task, mbufs, count, NULL);
424 for (i = 0; i < count; ++i) {
426 if (unlikely(idx < 0)) {
427 // mac not found for this IP
428 struct pkt_template *pktpl = &task->pkt_template[mbufs[i]->udata64 & TEMPLATE_INDEX_MASK];
429 uint8_t *hdr = (uint8_t *)pkt_hdr[i];
430 ip_src_pos = pktpl->ip_dst_pos - 4;
431 ip_src = *(uint32_t *)(hdr + ip_src_pos);
432 create_arp(mbufs[i], pkt_hdr[i], (uint64_t *)&hdr[6], key[i], ip_src);
433 mbufs[i]->udata64 |= MBUF_ARP;
435 // mac found for this IP
436 struct ether_hdr_arp *hdr_arp = (struct ether_hdr_arp *)pkt_hdr[i];
437 memcpy(&hdr_arp->ether_hdr.d_addr.addr_bytes, &task->dst_mac[idx], 6);
441 for (i = 0; i < count; ++i) {
442 uint8_t *hdr = (uint8_t *)pkt_hdr[i];
443 struct pkt_template *pktpl = &task->pkt_template[mbufs[i]->udata64 & TEMPLATE_INDEX_MASK];
445 // Check if packet template already has the mac
446 if (unlikely(pktpl->dst_mac == 0)) {
447 // no random_ip, can take from from packet template but no mac (yet)
448 uint32_t ip_dst_pos = pktpl->ip_dst_pos;
449 ip_dst = *(uint32_t *)(hdr + ip_dst_pos);
450 create_arp(mbufs[i], pkt_hdr[i], (uint64_t *)&pktpl->buf[6], ip_dst, pktpl->ip_src);
451 mbufs[i]->udata64 |= MBUF_ARP;
453 // no random ip, mac known
454 struct ether_hdr_arp *hdr_arp = (struct ether_hdr_arp *)pkt_hdr[i];
455 memcpy(&hdr_arp->ether_hdr.d_addr.addr_bytes, &pktpl->dst_mac, 6);
463 static void task_gen_apply_random_fields(struct task_gen *task, uint8_t *hdr)
465 uint32_t ret, ret_tmp;
467 for (uint16_t i = 0; i < task->n_rands; ++i) {
468 ret = random_next(&task->rand[i].state);
469 ret_tmp = (ret & task->rand[i].rand_mask) | task->rand[i].fixed_bits;
471 ret_tmp = rte_bswap32(ret_tmp);
472 /* At this point, the lower order bytes (BE) contain
473 the generated value. The address where the values
474 of interest starts is at ret_tmp + 4 - rand_len. */
475 uint8_t *pret_tmp = (uint8_t*)&ret_tmp;
476 rte_memcpy(hdr + task->rand[i].rand_offset, pret_tmp + 4 - task->rand[i].rand_len, task->rand[i].rand_len);
480 static void task_gen_apply_all_random_fields(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
485 for (uint16_t i = 0; i < count; ++i)
486 task_gen_apply_random_fields(task, pkt_hdr[i]);
489 static void task_gen_apply_accur_pos(struct task_gen *task, uint8_t *pkt_hdr, uint32_t accuracy)
491 *(uint32_t *)(pkt_hdr + task->accur_pos) = accuracy;
494 static void task_gen_apply_sig(struct task_gen *task, uint8_t *pkt_hdr)
496 *(uint32_t *)(pkt_hdr + task->sig_pos) = task->sig;
499 static void task_gen_apply_all_accur_pos(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
501 if (!task->accur_pos)
504 /* The accuracy of task->pkt_queue_index - 64 is stored in
505 packet task->pkt_queue_index. The ID modulo 64 is the
507 for (uint16_t j = 0; j < count; ++j) {
508 if ((mbufs[j]->udata64 & MBUF_ARP) == 0) {
509 uint32_t accuracy = task->accur[(task->pkt_queue_index + j) & 63];
510 task_gen_apply_accur_pos(task, pkt_hdr[j], accuracy);
515 static void task_gen_apply_all_sig(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
520 for (uint16_t j = 0; j < count; ++j) {
521 if ((mbufs[j]->udata64 & MBUF_ARP) == 0) {
522 task_gen_apply_sig(task, pkt_hdr[j]);
527 static void task_gen_apply_unique_id(struct task_gen *task, uint8_t *pkt_hdr, const struct unique_id *id)
529 struct unique_id *dst = (struct unique_id *)(pkt_hdr + task->packet_id_pos);
534 static void task_gen_apply_all_unique_id(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
536 if (!task->packet_id_pos)
539 for (uint16_t i = 0; i < count; ++i) {
540 if ((mbufs[i]->udata64 & MBUF_ARP) == 0) {
542 unique_id_init(&id, task->generator_id, task->pkt_queue_index++);
543 task_gen_apply_unique_id(task, pkt_hdr[i], &id);
548 static void task_gen_checksum_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
550 if (!(task->runtime_flags & TASK_TX_CRC))
553 if (!task->runtime_checksum_needed)
556 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - count);
557 for (uint16_t i = 0; i < count; ++i) {
558 if ((mbufs[i]->udata64 & MBUF_ARP) == 0) {
559 struct pkt_template *pkt_template = &task->pkt_template[pkt_idx];
560 checksum_packet(pkt_hdr[i], mbufs[i], pkt_template, task->cksum_offload);
561 pkt_idx = task_gen_next_pkt_idx(task, pkt_idx);
566 static void task_gen_consume_tokens(struct task_gen *task, uint32_t tokens, uint32_t send_count)
568 /* If max burst has been sent, we can't keep up so just assume
569 that we can (leaving a "gap" in the packet stream on the
571 task->token_time.bytes_now -= tokens;
572 if (send_count == task->max_bulk_size && task->token_time.bytes_now > tokens) {
573 task->token_time.bytes_now = tokens;
577 static uint64_t task_gen_calc_bulk_duration(struct task_gen *task, uint32_t count)
579 uint32_t pkt_idx = task_gen_offset_pkt_idx(task, - 1);
580 struct pkt_template *last_pkt_template = &task->pkt_template[pkt_idx];
581 uint32_t last_pkt_len = pkt_len_to_wire_size(last_pkt_template->len);
582 uint64_t last_pkt_duration = bytes_to_tsc(task, last_pkt_len);
583 uint64_t bulk_duration = task->pkt_tsc_offset[count - 1] + last_pkt_duration;
585 return bulk_duration;
588 static uint64_t task_gen_write_latency(struct task_gen *task, uint8_t **pkt_hdr, uint32_t count)
590 if (!task->lat_enabled)
593 uint64_t tx_tsc, delta_t;
594 uint64_t tsc_before_tx = 0;
596 /* Just before sending the packets, apply the time stamp
597 relative to when the first packet will be sent. The first
598 packet will be sent now. The time is read for each packet
599 to reduce the error towards the actual time the packet will
601 uint64_t write_tsc_after, write_tsc_before;
603 write_tsc_before = rte_rdtsc();
605 /* The time it took previously to write the time stamps in the
606 packets is used as an estimate for how long it will take to
607 write the time stamps now. The estimated time at which the
608 packets will actually be sent will be at tx_tsc. */
609 tx_tsc = write_tsc_before + task->write_duration_estimate;
611 /* The offset delta_t tracks the difference between the actual
612 time and the time written in the packets. Adding the offset
613 to the actual time insures that the time written in the
614 packets is monotonically increasing. At the same time,
615 simply sleeping until delta_t is zero would leave a period
616 of silence on the line. The error has been introduced
617 earlier, but the packets have already been sent. */
618 if (tx_tsc < task->earliest_tsc_next_pkt)
619 delta_t = task->earliest_tsc_next_pkt - tx_tsc;
623 for (uint16_t i = 0; i < count; ++i) {
624 uint32_t *pos = (uint32_t *)(pkt_hdr[i] + task->lat_pos);
625 const uint64_t pkt_tsc = tx_tsc + delta_t + task->pkt_tsc_offset[i];
627 *pos = pkt_tsc >> LATENCY_ACCURACY;
630 uint64_t bulk_duration = task_gen_calc_bulk_duration(task, count);
632 task->earliest_tsc_next_pkt = tx_tsc + delta_t + bulk_duration;
633 write_tsc_after = rte_rdtsc();
634 task->write_duration_estimate = write_tsc_after - write_tsc_before;
636 /* Make sure that the time stamps that were written
637 are valid. The offset must be taken into account */
639 tsc_before_tx = rte_rdtsc();
640 } while (tsc_before_tx < tx_tsc);
641 return tsc_before_tx;
644 static void task_gen_store_accuracy(struct task_gen *task, uint32_t count, uint64_t tsc_before_tx)
646 if (!task->accur_pos)
649 uint64_t accur = rte_rdtsc() - tsc_before_tx;
650 uint64_t first_accuracy_idx = task->pkt_queue_index - count;
652 for (uint32_t i = 0; i < count; ++i) {
653 uint32_t accuracy_idx = (first_accuracy_idx + i) & 63;
655 task->accur[accuracy_idx] = accur;
659 static void task_gen_load_and_prefetch(struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
661 for (uint16_t i = 0; i < count; ++i)
662 rte_prefetch0(mbufs[i]);
663 for (uint16_t i = 0; i < count; ++i)
664 pkt_hdr[i] = rte_pktmbuf_mtod(mbufs[i], uint8_t *);
665 for (uint16_t i = 0; i < count; ++i)
666 rte_prefetch0(pkt_hdr[i]);
669 static void task_gen_build_packets(struct task_gen *task, struct rte_mbuf **mbufs, uint8_t **pkt_hdr, uint32_t count)
671 uint64_t will_send_bytes = 0;
673 for (uint16_t i = 0; i < count; ++i) {
674 struct pkt_template *pktpl = &task->pkt_template[task->pkt_idx];
675 struct pkt_template *pkt_template = &task->pkt_template[task->pkt_idx];
676 pkt_template_init_mbuf(pkt_template, mbufs[i], pkt_hdr[i]);
677 mbufs[i]->udata64 = task->pkt_idx & TEMPLATE_INDEX_MASK;
678 struct ether_hdr *hdr = (struct ether_hdr *)pkt_hdr[i];
679 if (task->lat_enabled) {
680 task->pkt_tsc_offset[i] = bytes_to_tsc(task, will_send_bytes);
681 will_send_bytes += pkt_len_to_wire_size(pkt_template->len);
683 task->pkt_idx = task_gen_next_pkt_idx(task, task->pkt_idx);
687 static void task_gen_update_config(struct task_gen *task)
689 if (task->token_time.cfg.bpp != task->new_rate_bps)
690 task_gen_reset_token_time(task);
693 static inline void handle_arp_pkts(struct task_gen *task, struct rte_mbuf **mbufs, uint16_t n_pkts)
697 struct ether_hdr_arp *hdr;
698 uint8_t out[MAX_PKT_BURST];
699 static struct my_arp_t arp_reply = {
706 static struct my_arp_t arp_request = {
714 for (j = 0; j < n_pkts; ++j) {
717 for (j = 0; j < n_pkts; ++j) {
718 PREFETCH0(rte_pktmbuf_mtod(mbufs[j], void *));
720 for (j = 0; j < n_pkts; ++j) {
721 hdr = rte_pktmbuf_mtod(mbufs[j], struct ether_hdr_arp *);
722 if (hdr->ether_hdr.ether_type == ETYPE_ARP) {
723 if (memcmp(&hdr->arp, &arp_reply, 8) == 0) {
724 uint32_t ip = hdr->arp.data.spa;
725 // plog_info("Received ARP Reply for IP %x\n",ip);
726 if (ip == task->gw_ip) {
727 memcpy(&task->gw_mac, &hdr->arp.data.sha, 6);;
728 task->flags |= FLAG_DST_MAC_KNOWN;
729 out[j] = OUT_HANDLED;
731 } else if ((task->n_pkts >= 4) || (task->flags & FLAG_RANDOM_IPS)) {
732 // Ideally, we should add the key when making the arp request,
733 // We should only store the mac address key was created.
734 // Here we are storing MAC we did not asked for...
735 ret = rte_hash_add_key(task->mac_hash, (const void *)&ip);
737 plogx_info("Unable add ip %d.%d.%d.%d in mac_hash\n", IP4(ip));
738 out[j] = OUT_DISCARD;
740 task->dst_mac[ret] = *(uint64_t *)&(hdr->arp.data.sha);
741 out[j] = OUT_HANDLED;
745 // Need to find template back...
746 // Only try this if there are few templates
747 for (unsigned int idx = 0; idx < task->n_pkts; idx++) {
748 struct pkt_template *pktpl = &task->pkt_template[idx];
749 uint32_t ip_dst_pos = pktpl->ip_dst_pos;
750 uint32_t *ip_dst = (uint32_t *)(((uint8_t *)pktpl->buf) + ip_dst_pos);
752 pktpl->dst_mac = *(uint64_t *)&(hdr->arp.data.sha);
754 out[j] = OUT_HANDLED;
756 } else if (memcmp(&hdr->arp, &arp_request, 8) == 0) {
757 struct ether_addr s_addr;
759 create_mac(hdr, &s_addr);
760 prepare_arp_reply(hdr, &s_addr);
761 memcpy(hdr->ether_hdr.d_addr.addr_bytes, hdr->ether_hdr.s_addr.addr_bytes, 6);
762 memcpy(hdr->ether_hdr.s_addr.addr_bytes, &s_addr, 6);
764 } else if (hdr->arp.data.tpa == task->src_ip) {
765 prepare_arp_reply(hdr, &task->src_mac);
766 memcpy(hdr->ether_hdr.d_addr.addr_bytes, hdr->ether_hdr.s_addr.addr_bytes, 6);
767 memcpy(hdr->ether_hdr.s_addr.addr_bytes, &task->src_mac, 6);
770 out[j] = OUT_DISCARD;
771 plogx_dbg("Received ARP on unexpected IP %x, expecting %x\n", rte_be_to_cpu_32(hdr->arp.data.tpa), rte_be_to_cpu_32(task->src_ip));
775 out[j] = OUT_DISCARD;
778 ret = task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
781 static int handle_gen_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
783 struct task_gen *task = (struct task_gen *)tbase;
784 uint8_t out[MAX_PKT_BURST] = {0};
789 if (unlikely((task->flags & FLAG_L3_GEN) && (n_pkts != 0))) {
790 handle_arp_pkts(task, mbufs, n_pkts);
793 task_gen_update_config(task);
795 if (task->pkt_count == 0) {
796 task_gen_reset_token_time(task);
799 if (!task->token_time.cfg.bpp)
802 token_time_update(&task->token_time, rte_rdtsc());
804 uint32_t would_send_bytes;
805 const uint32_t send_bulk = task_gen_calc_send_bulk(task, &would_send_bytes);
809 task_gen_take_count(task, send_bulk);
810 task_gen_consume_tokens(task, would_send_bytes, send_bulk);
812 struct rte_mbuf **new_pkts = local_mbuf_refill_and_take(&task->local_mbuf, send_bulk);
813 if (new_pkts == NULL)
815 uint8_t *pkt_hdr[MAX_RING_BURST];
817 task_gen_load_and_prefetch(new_pkts, pkt_hdr, send_bulk);
818 task_gen_build_packets(task, new_pkts, pkt_hdr, send_bulk);
819 task_gen_apply_all_random_fields(task, pkt_hdr, send_bulk);
820 if (task_gen_write_dst_mac(task, new_pkts, pkt_hdr, send_bulk) < 0)
822 task_gen_apply_all_accur_pos(task, new_pkts, pkt_hdr, send_bulk);
823 task_gen_apply_all_sig(task, new_pkts, pkt_hdr, send_bulk);
824 task_gen_apply_all_unique_id(task, new_pkts, pkt_hdr, send_bulk);
826 uint64_t tsc_before_tx;
828 tsc_before_tx = task_gen_write_latency(task, pkt_hdr, send_bulk);
829 task_gen_checksum_packets(task, new_pkts, pkt_hdr, send_bulk);
830 ret = task->base.tx_pkt(&task->base, new_pkts, send_bulk, out);
831 task_gen_store_accuracy(task, send_bulk, tsc_before_tx);
835 static void init_task_gen_seeds(struct task_gen *task)
837 for (size_t i = 0; i < sizeof(task->rand)/sizeof(task->rand[0]); ++i)
838 random_init_seed(&task->rand[i].state);
841 static uint32_t pcap_count_pkts(pcap_t *handle)
843 struct pcap_pkthdr header;
846 long pkt1_fpos = ftell(pcap_file(handle));
848 while ((buf = pcap_next(handle, &header))) {
851 int ret2 = fseek(pcap_file(handle), pkt1_fpos, SEEK_SET);
852 PROX_PANIC(ret2 != 0, "Failed to reset reading pcap file\n");
856 static uint64_t avg_time_stamp(uint64_t *time_stamp, uint32_t n)
858 uint64_t tot_inter_pkt = 0;
860 for (uint32_t i = 0; i < n; ++i)
861 tot_inter_pkt += time_stamp[i];
862 return (tot_inter_pkt + n / 2)/n;
865 static int pcap_read_pkts(pcap_t *handle, const char *file_name, uint32_t n_pkts, struct pkt_template *proto, uint64_t *time_stamp)
867 struct pcap_pkthdr header;
871 for (uint32_t i = 0; i < n_pkts; ++i) {
872 buf = pcap_next(handle, &header);
874 PROX_PANIC(buf == NULL, "Failed to read packet %d from pcap %s\n", i, file_name);
875 proto[i].len = header.len;
876 len = RTE_MIN(header.len, sizeof(proto[i].buf));
877 if (header.len > len)
878 plogx_warn("Packet truncated from %u to %zu bytes\n", header.len, len);
881 static struct timeval beg;
887 tv = tv_diff(&beg, &header.ts);
888 tv_to_tsc(&tv, time_stamp + i);
890 rte_memcpy(proto[i].buf, buf, len);
893 if (time_stamp && n_pkts) {
894 for (uint32_t i = n_pkts - 1; i > 0; --i)
895 time_stamp[i] -= time_stamp[i - 1];
896 /* Since the handle function will loop the packets,
897 there is one time-stamp that is not provided by the
898 pcap file. This is the time between the last and
899 the first packet. This implementation takes the
900 average of the inter-packet times here. */
902 time_stamp[0] = avg_time_stamp(time_stamp + 1, n_pkts - 1);
908 static int check_pkt_size(struct task_gen *task, uint32_t pkt_size, int do_panic)
910 const uint16_t min_len = sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr);
911 const uint16_t max_len = ETHER_MAX_LEN - 4;
914 PROX_PANIC(pkt_size == 0, "Invalid packet size length (no packet defined?)\n");
915 PROX_PANIC(pkt_size > max_len, "pkt_size out of range (must be <= %u)\n", max_len);
916 PROX_PANIC(pkt_size < min_len, "pkt_size out of range (must be >= %u)\n", min_len);
920 plog_err("Invalid packet size length (no packet defined?)\n");
923 if (pkt_size > max_len) {
924 plog_err("pkt_size out of range (must be <= %u)\n", max_len);
927 if (pkt_size < min_len) {
928 plog_err("pkt_size out of range (must be >= %u)\n", min_len);
935 static int check_all_pkt_size(struct task_gen *task, int do_panic)
938 for (uint32_t i = 0; i < task->n_pkts;++i) {
939 if ((rc = check_pkt_size(task, task->pkt_template[i].len, do_panic)) != 0)
945 static void check_fields_in_bounds(struct task_gen *task)
947 const uint32_t pkt_size = task->pkt_template[0].len;
949 if (task->lat_enabled) {
950 uint32_t pos_beg = task->lat_pos;
951 uint32_t pos_end = task->lat_pos + 3U;
953 PROX_PANIC(pkt_size <= pos_end, "Writing latency at %u-%u, but packet size is %u bytes\n",
954 pos_beg, pos_end, pkt_size);
956 if (task->packet_id_pos) {
957 uint32_t pos_beg = task->packet_id_pos;
958 uint32_t pos_end = task->packet_id_pos + 4U;
960 PROX_PANIC(pkt_size <= pos_end, "Writing packet at %u-%u, but packet size is %u bytes\n",
961 pos_beg, pos_end, pkt_size);
963 if (task->accur_pos) {
964 uint32_t pos_beg = task->accur_pos;
965 uint32_t pos_end = task->accur_pos + 3U;
967 PROX_PANIC(pkt_size <= pos_end, "Writing accuracy at %u%-u, but packet size is %u bytes\n",
968 pos_beg, pos_end, pkt_size);
972 static void task_gen_pkt_template_recalc_metadata(struct task_gen *task)
974 struct pkt_template *template;
976 for (size_t i = 0; i < task->n_pkts; ++i) {
977 template = &task->pkt_template[i];
978 parse_l2_l3_len(template->buf, &template->l2_len, &template->l3_len, template->len);
982 static void task_gen_pkt_template_recalc_checksum(struct task_gen *task)
984 struct pkt_template *template;
987 task->runtime_checksum_needed = 0;
988 for (size_t i = 0; i < task->n_pkts; ++i) {
989 template = &task->pkt_template[i];
990 if (template->l2_len == 0)
992 ip = (struct ipv4_hdr *)(template->buf + template->l2_len);
994 ip->hdr_checksum = 0;
995 prox_ip_cksum_sw(ip);
996 uint32_t l4_len = rte_bswap16(ip->total_length) - template->l3_len;
998 if (ip->next_proto_id == IPPROTO_UDP) {
999 struct udp_hdr *udp = (struct udp_hdr *)(((uint8_t *)ip) + template->l3_len);
1000 prox_udp_cksum_sw(udp, l4_len, ip->src_addr, ip->dst_addr);
1001 } else if (ip->next_proto_id == IPPROTO_TCP) {
1002 struct tcp_hdr *tcp = (struct tcp_hdr *)(((uint8_t *)ip) + template->l3_len);
1003 prox_tcp_cksum_sw(tcp, l4_len, ip->src_addr, ip->dst_addr);
1006 /* The current implementation avoids checksum
1007 calculation by determining that at packet
1008 construction time, no fields are applied that would
1009 require a recalculation of the checksum. */
1010 if (task->lat_enabled && task->lat_pos > template->l2_len)
1011 task->runtime_checksum_needed = 1;
1012 if (task->accur_pos > template->l2_len)
1013 task->runtime_checksum_needed = 1;
1014 if (task->packet_id_pos > template->l2_len)
1015 task->runtime_checksum_needed = 1;
1019 static void task_gen_pkt_template_recalc_all(struct task_gen *task)
1021 task_gen_pkt_template_recalc_metadata(task);
1022 task_gen_pkt_template_recalc_checksum(task);
1025 static void task_gen_reset_pkt_templates_len(struct task_gen *task)
1027 struct pkt_template *src, *dst;
1029 for (size_t i = 0; i < task->n_pkts; ++i) {
1030 src = &task->pkt_template_orig[i];
1031 dst = &task->pkt_template[i];
1032 dst->len = src->len;
1036 static void task_gen_reset_pkt_templates_content(struct task_gen *task)
1038 struct pkt_template *src, *dst;
1040 for (size_t i = 0; i < task->n_pkts; ++i) {
1041 src = &task->pkt_template_orig[i];
1042 dst = &task->pkt_template[i];
1043 memcpy(dst->buf, src->buf, dst->len);
1047 static void task_gen_reset_pkt_templates(struct task_gen *task)
1049 task_gen_reset_pkt_templates_len(task);
1050 task_gen_reset_pkt_templates_content(task);
1051 task_gen_pkt_template_recalc_all(task);
1054 static void task_init_gen_load_pkt_inline(struct task_gen *task, struct task_args *targ)
1056 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
1058 if (targ->pkt_size > sizeof(task->pkt_template[0].buf))
1059 targ->pkt_size = sizeof(task->pkt_template[0].buf);
1062 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
1063 task->pkt_template = prox_zmalloc(mem_size, socket_id);
1064 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
1066 PROX_PANIC(task->pkt_template == NULL ||
1067 task->pkt_template_orig == NULL,
1068 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1070 rte_memcpy(task->pkt_template_orig[0].buf, targ->pkt_inline, targ->pkt_size);
1071 task->pkt_template_orig[0].len = targ->pkt_size;
1072 task_gen_reset_pkt_templates(task);
1073 check_all_pkt_size(task, 1);
1074 check_fields_in_bounds(task);
1077 static void task_init_gen_load_pcap(struct task_gen *task, struct task_args *targ)
1079 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
1080 char err[PCAP_ERRBUF_SIZE];
1081 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1082 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1084 task->n_pkts = pcap_count_pkts(handle);
1085 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1088 task->n_pkts = RTE_MIN(task->n_pkts, targ->n_pkts);
1089 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
1090 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1091 size_t mem_size = task->n_pkts * sizeof(*task->pkt_template);
1092 task->pkt_template = prox_zmalloc(mem_size, socket_id);
1093 task->pkt_template_orig = prox_zmalloc(mem_size, socket_id);
1094 PROX_PANIC(task->pkt_template == NULL ||
1095 task->pkt_template_orig == NULL,
1096 "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1098 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->pkt_template_orig, NULL);
1100 task_gen_reset_pkt_templates(task);
1103 static struct rte_mempool *task_gen_create_mempool(struct task_args *targ)
1105 static char name[] = "gen_pool";
1106 struct rte_mempool *ret;
1107 const int sock_id = rte_lcore_to_socket_id(targ->lconf->id);
1110 ret = rte_mempool_create(name, targ->nb_mbuf - 1, MBUF_SIZE,
1111 targ->nb_cache_mbuf, sizeof(struct rte_pktmbuf_pool_private),
1112 rte_pktmbuf_pool_init, NULL, rte_pktmbuf_init, 0,
1114 PROX_PANIC(ret == NULL, "Failed to allocate dummy memory pool on socket %u with %u elements\n",
1115 sock_id, targ->nb_mbuf - 1);
1119 void task_gen_set_pkt_count(struct task_base *tbase, uint32_t count)
1121 struct task_gen *task = (struct task_gen *)tbase;
1123 task->pkt_count = count;
1126 int task_gen_set_pkt_size(struct task_base *tbase, uint32_t pkt_size)
1128 struct task_gen *task = (struct task_gen *)tbase;
1131 task->pkt_template[0].len = pkt_size;
1132 if ((rc = check_all_pkt_size(task, 0)) != 0)
1134 check_fields_in_bounds(task);
1138 void task_gen_set_gateway_ip(struct task_base *tbase, uint32_t ip)
1140 struct task_gen *task = (struct task_gen *)tbase;
1142 task->flags &= ~FLAG_DST_MAC_KNOWN;
1145 void task_gen_set_rate(struct task_base *tbase, uint64_t bps)
1147 struct task_gen *task = (struct task_gen *)tbase;
1149 task->new_rate_bps = bps;
1152 void task_gen_reset_randoms(struct task_base *tbase)
1154 struct task_gen *task = (struct task_gen *)tbase;
1156 for (uint32_t i = 0; i < task->n_rands; ++i) {
1157 task->rand[i].rand_mask = 0;
1158 task->rand[i].fixed_bits = 0;
1159 task->rand[i].rand_offset = 0;
1162 task->flags &= ~FLAG_RANDOM_IPS;
1165 int task_gen_set_value(struct task_base *tbase, uint32_t value, uint32_t offset, uint32_t len)
1167 struct task_gen *task = (struct task_gen *)tbase;
1169 for (size_t i = 0; i < task->n_pkts; ++i) {
1170 uint32_t to_write = rte_cpu_to_be_32(value) >> ((4 - len) * 8);
1171 uint8_t *dst = task->pkt_template[i].buf;
1173 rte_memcpy(dst + offset, &to_write, len);
1176 task_gen_pkt_template_recalc_all(task);
1181 void task_gen_reset_values(struct task_base *tbase)
1183 struct task_gen *task = (struct task_gen *)tbase;
1185 task_gen_reset_pkt_templates_content(task);
1188 uint32_t task_gen_get_n_randoms(struct task_base *tbase)
1190 struct task_gen *task = (struct task_gen *)tbase;
1192 return task->n_rands;
1195 static void init_task_gen_pcap(struct task_base *tbase, struct task_args *targ)
1197 struct task_gen_pcap *task = (struct task_gen_pcap *)tbase;
1198 const uint32_t sockid = rte_lcore_to_socket_id(targ->lconf->id);
1200 task->loop = targ->loop;
1202 task->hz = rte_get_tsc_hz();
1204 task->local_mbuf.mempool = task_gen_create_mempool(targ);
1206 PROX_PANIC(!strcmp(targ->pcap_file, ""), "No pcap file defined\n");
1208 char err[PCAP_ERRBUF_SIZE];
1209 pcap_t *handle = pcap_open_offline(targ->pcap_file, err);
1210 PROX_PANIC(handle == NULL, "Failed to open PCAP file: %s\n", err);
1212 task->n_pkts = pcap_count_pkts(handle);
1213 plogx_info("%u packets in pcap file '%s'\n", task->n_pkts, targ->pcap_file);
1216 plogx_info("Configured to load %u packets\n", targ->n_pkts);
1217 if (task->n_pkts > targ->n_pkts)
1218 task->n_pkts = targ->n_pkts;
1220 PROX_PANIC(task->n_pkts > MAX_TEMPLATE_INDEX, "Too many packets specified in pcap - increase MAX_TEMPLATE_INDEX\n");
1222 plogx_info("Loading %u packets from pcap\n", task->n_pkts);
1224 size_t mem_size = task->n_pkts * (sizeof(*task->proto) + sizeof(*task->proto_tsc));
1225 uint8_t *mem = prox_zmalloc(mem_size, sockid);
1227 PROX_PANIC(mem == NULL, "Failed to allocate %lu bytes (in huge pages) for pcap file\n", mem_size);
1228 task->proto = (struct pkt_template *) mem;
1229 task->proto_tsc = (uint64_t *)(mem + task->n_pkts * sizeof(*task->proto));
1231 pcap_read_pkts(handle, targ->pcap_file, task->n_pkts, task->proto, task->proto_tsc);
1235 static int task_gen_find_random_with_offset(struct task_gen *task, uint32_t offset)
1237 for (uint32_t i = 0; i < task->n_rands; ++i) {
1238 if (task->rand[i].rand_offset == offset) {
1246 int task_gen_add_rand(struct task_base *tbase, const char *rand_str, uint32_t offset, uint32_t rand_id)
1248 struct task_gen *task = (struct task_gen *)tbase;
1249 uint32_t existing_rand;
1251 if (rand_id == UINT32_MAX && task->n_rands == 64) {
1252 plog_err("Too many randoms\n");
1255 uint32_t mask, fixed, len;
1257 if (parse_random_str(&mask, &fixed, &len, rand_str)) {
1258 plog_err("%s\n", get_parse_err());
1261 task->runtime_checksum_needed = 1;
1263 existing_rand = task_gen_find_random_with_offset(task, offset);
1264 if (existing_rand != UINT32_MAX) {
1265 plog_warn("Random at offset %d already set => overwriting len = %d %s\n", offset, len, rand_str);
1266 rand_id = existing_rand;
1267 task->rand[rand_id].rand_len = len;
1268 task->rand[rand_id].rand_offset = offset;
1269 task->rand[rand_id].rand_mask = mask;
1270 task->rand[rand_id].fixed_bits = fixed;
1274 task->rand[task->n_rands].rand_len = len;
1275 task->rand[task->n_rands].rand_offset = offset;
1276 task->rand[task->n_rands].rand_mask = mask;
1277 task->rand[task->n_rands].fixed_bits = fixed;
1279 struct pkt_template *pktpl = &task->pkt_template[0];
1280 if (!((offset >= pktpl->ip_dst_pos + 4) || (offset + len < pktpl->ip_dst_pos))) {
1281 plog_info("\tUsing randoms IP destinations\n");
1282 task->flags |= FLAG_RANDOM_IPS;
1289 static void init_task_gen_early(struct task_args *targ)
1291 uint8_t *generator_count = prox_sh_find_system("generator_count");
1293 if (generator_count == NULL) {
1294 generator_count = prox_zmalloc(sizeof(*generator_count), 0);
1295 prox_sh_add_system("generator_count", generator_count);
1297 targ->generator_id = *generator_count;
1298 (*generator_count)++;
1301 static void init_task_gen(struct task_base *tbase, struct task_args *targ)
1303 struct task_gen *task = (struct task_gen *)tbase;
1305 task->packet_id_pos = targ->packet_id_pos;
1307 task->local_mbuf.mempool = task_gen_create_mempool(targ);
1308 PROX_PANIC(task->local_mbuf.mempool == NULL, "Failed to create mempool\n");
1310 task->hz = rte_get_tsc_hz();
1311 task->lat_pos = targ->lat_pos;
1312 task->accur_pos = targ->accur_pos;
1313 task->sig_pos = targ->sig_pos;
1314 task->sig = targ->sig;
1315 task->new_rate_bps = targ->rate_bps;
1317 struct token_time_cfg tt_cfg = token_time_cfg_create(1250000000, rte_get_tsc_hz(), -1);
1319 token_time_init(&task->token_time, &tt_cfg);
1320 init_task_gen_seeds(task);
1322 task->min_bulk_size = targ->min_bulk_size;
1323 task->max_bulk_size = targ->max_bulk_size;
1324 if (task->min_bulk_size < 1)
1325 task->min_bulk_size = 1;
1326 if (task->max_bulk_size < 1)
1327 task->max_bulk_size = 64;
1328 PROX_PANIC(task->max_bulk_size > 64, "max_bulk_size higher than 64\n");
1329 PROX_PANIC(task->max_bulk_size < task->min_bulk_size, "max_bulk_size must be > than min_bulk_size\n");
1331 task->pkt_count = -1;
1332 task->lat_enabled = targ->lat_enabled;
1333 task->runtime_flags = targ->runtime_flags;
1334 PROX_PANIC((task->lat_pos || task->accur_pos) && !task->lat_enabled, "lat not enabled by lat pos or accur pos configured\n");
1336 task->generator_id = targ->generator_id;
1337 task->link_speed = UINT64_MAX;
1338 if (targ->nb_txrings == 0 && targ->nb_txports == 1)
1339 task->link_speed = 1250000000;
1341 if (!strcmp(targ->pcap_file, "")) {
1342 plog_info("\tUsing inline definition of a packet\n");
1343 task_init_gen_load_pkt_inline(task, targ);
1345 plog_info("Loading from pcap %s\n", targ->pcap_file);
1346 task_init_gen_load_pcap(task, targ);
1349 if ((targ->flags & DSF_KEEP_SRC_MAC) == 0 && (targ->nb_txrings || targ->nb_txports)) {
1350 uint8_t *src_addr = prox_port_cfg[tbase->tx_params_hw.tx_port_queue->port].eth_addr.addr_bytes;
1351 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1352 rte_memcpy(&task->pkt_template[i].buf[6], src_addr, 6);
1355 memcpy(&task->src_mac, &prox_port_cfg[task->base.tx_params_hw.tx_port_queue->port].eth_addr, sizeof(struct ether_addr));
1356 if (!strcmp(targ->task_init->sub_mode_str, "l3")) {
1357 // In L3 GEN, we need to receive ARP replies
1358 task->flags = FLAG_L3_GEN;
1359 task->gw_ip = rte_cpu_to_be_32(targ->gateway_ipv4);
1362 if (targ->number_gen_ip == 0)
1363 n_entries = 1048576;
1365 n_entries = targ->number_gen_ip;
1367 static char hash_name[30];
1368 sprintf(hash_name, "A%03d_mac_table", targ->lconf->id);
1370 struct rte_hash_parameters hash_params = {
1372 .entries = n_entries,
1373 .key_len = sizeof(uint32_t),
1374 .hash_func = rte_hash_crc,
1375 .hash_func_init_val = 0,
1377 task->mac_hash = rte_hash_create(&hash_params);
1378 PROX_PANIC(task->mac_hash == NULL, "Failed to set up mac hash table for %d IP\n", n_entries);
1380 const uint32_t socket = rte_lcore_to_socket_id(targ->lconf->id);
1381 task->dst_mac = (uint64_t *)prox_zmalloc(n_entries * sizeof(uint64_t), socket);
1382 PROX_PANIC(task->dst_mac == NULL, "Failed to allocate mac table for %d IP\n", n_entries);
1384 for (uint32_t i = 0; i < task->n_pkts; ++i) {
1385 // For all destination IP, ARP request will need to be sent
1386 // Store position of Destination IP in template
1389 int l2_len = sizeof(struct ether_hdr);
1390 struct vlan_hdr *vlan_hdr;
1391 uint8_t *pkt = task->pkt_template[i].buf;
1392 struct ether_hdr *eth_hdr = (struct ether_hdr*)pkt;
1393 struct ipv4_hdr *ip;
1394 uint16_t ether_type = eth_hdr->ether_type;
1396 // Unstack VLAN tags
1397 while (((ether_type == ETYPE_8021ad) || (ether_type == ETYPE_VLAN)) && (l2_len + sizeof(struct vlan_hdr) < task->pkt_template[i].len)) {
1398 vlan_hdr = (struct vlan_hdr *)(pkt + l2_len);
1400 ether_type = vlan_hdr->eth_proto;
1402 if ((ether_type == ETYPE_MPLSU) || (ether_type == ETYPE_MPLSM)) {
1406 if ((ether_type == ETYPE_IPv4) || maybe_ipv4) {
1407 struct ipv4_hdr *ip = (struct ipv4_hdr *)(pkt + l2_len);
1408 PROX_PANIC(ip->version_ihl >> 4 != 4, "IPv4 ether_type but IP version = %d != 4", ip->version_ihl >> 4);
1409 // Even if IPv4 header contains options, options are after ip src and dst
1410 ip_dst_pos = l2_len + sizeof(struct ipv4_hdr) - sizeof(uint32_t);
1411 uint32_t *p = ((uint32_t *)(task->pkt_template[i].buf + ip_dst_pos - sizeof(uint32_t)));
1412 task->pkt_template[i].ip_dst_pos = ip_dst_pos;
1413 task->pkt_template[i].ip_src = *p;
1414 uint32_t *p1 = ((uint32_t *)(task->pkt_template[i].buf + ip_dst_pos));
1415 plog_info("\tip_dst_pos = %d, ip_dst = %x\n", ip_dst_pos, *p1);
1418 task->src_ip = rte_cpu_to_be_32(targ->local_ipv4);
1420 for (uint32_t i = 0; i < targ->n_rand_str; ++i) {
1421 PROX_PANIC(task_gen_add_rand(tbase, targ->rand_str[i], targ->rand_offset[i], UINT32_MAX),
1422 "Failed to add random\n");
1425 struct prox_port_cfg *port = find_reachable_port(targ);
1427 task->cksum_offload = port->capabilities.tx_offload_cksum;
1431 static struct task_init task_init_gen = {
1433 .init = init_task_gen,
1434 .handle = handle_gen_bulk,
1437 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1438 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1439 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
1441 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1443 .size = sizeof(struct task_gen)
1446 static struct task_init task_init_gen_l3 = {
1448 .sub_mode_str = "l3",
1449 .init = init_task_gen,
1450 .handle = handle_gen_bulk,
1453 // For SOFT_CRC, no offload is needed. If both NOOFFLOADS and NOMULTSEGS flags are set the
1454 // vector mode is used by DPDK, resulting (theoretically) in higher performance.
1455 .flag_features = TASK_FEATURE_ZERO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS|TASK_FEATURE_ZERO_RX,
1457 .flag_features = TASK_FEATURE_ZERO_RX,
1459 .size = sizeof(struct task_gen)
1462 static struct task_init task_init_gen_pcap = {
1464 .sub_mode_str = "pcap",
1465 .init = init_task_gen_pcap,
1466 .handle = handle_gen_pcap_bulk,
1467 .start = start_pcap,
1469 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX | TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS | TASK_FEATURE_TXQ_FLAGS_NOMULTSEGS,
1471 .flag_features = TASK_FEATURE_NEVER_DISCARDS | TASK_FEATURE_NO_RX,
1473 .size = sizeof(struct task_gen_pcap)
1476 __attribute__((constructor)) static void reg_task_gen(void)
1478 reg_task(&task_init_gen);
1479 reg_task(&task_init_gen_l3);
1480 reg_task(&task_init_gen_pcap);