2 // Copyright (c) 2010-2019 Intel Corporation
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
8 // http://www.apache.org/licenses/LICENSE-2.0
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
19 #include <rte_cycles.h>
23 #include "handle_gen.h"
24 #include "prox_malloc.h"
25 #include "mbuf_utils.h"
26 #include "handle_lat.h"
28 #include "task_init.h"
29 #include "task_base.h"
34 #include "prox_shared.h"
35 #include "prox_port_cfg.h"
37 #define DEFAULT_BUCKET_SIZE 11
38 #define ACCURACY_BUFFER_SIZE (2 * ACCURACY_WINDOW)
41 uint32_t rx_packet_index;
42 uint64_t tx_packet_index;
47 uint16_t port_queue_id;
57 struct delayed_latency_entry {
58 uint32_t rx_packet_id;
59 uint32_t tx_packet_id;
63 uint64_t pkt_tx_time; // Time written into packets by gen. Unit is TSC >> LATENCY_ACCURACY
67 static struct delayed_latency_entry *delayed_latency_get(struct delayed_latency_entry **delayed_latency_entries, uint8_t generator_id, uint32_t packet_id)
69 struct delayed_latency_entry *delayed_latency_entry = &delayed_latency_entries[generator_id][packet_id % ACCURACY_BUFFER_SIZE];
70 if (delayed_latency_entry->packet_id == packet_id)
71 return delayed_latency_entry;
76 static struct delayed_latency_entry *delayed_latency_create(struct delayed_latency_entry **delayed_latency_entries, uint8_t generator_id, uint32_t packet_id)
78 struct delayed_latency_entry *delayed_latency_entry = &delayed_latency_entries[generator_id][packet_id % ACCURACY_BUFFER_SIZE];
79 delayed_latency_entry->packet_id = packet_id;
80 return delayed_latency_entry;
83 struct rx_pkt_meta_data {
86 uint32_t bytes_after_in_bulk;
90 struct task_base base;
92 uint64_t rx_packet_index;
93 uint64_t last_pkts_tsc;
94 struct delayed_latency_entry **delayed_latency_entries;
95 struct lat_info *latency_buffer;
96 uint32_t latency_buffer_idx;
97 uint32_t latency_buffer_size;
100 uint16_t unique_id_pos;
104 volatile uint16_t use_lt; /* which lt to use, */
105 volatile uint16_t using_lt; /* 0 or 1 depending on which of the 2 measurements are used */
106 struct lat_test lt[2];
107 struct lat_test *lat_test;
108 uint32_t generator_count;
109 uint16_t min_pkt_len;
110 struct early_loss_detect *eld;
111 struct rx_pkt_meta_data *rx_pkt_meta;
112 // Following fields are only used when starting or stopping, not in general runtime
113 uint64_t *prev_tx_packet_index;
116 struct prox_port_cfg *port;
117 uint64_t *bytes_to_tsc;
118 uint64_t *previous_packet;
120 /* This function calculate the difference between rx and tx_time
121 * Both values are uint32_t (see handle_lat_bulk)
122 * rx time should be higher than tx_time...except every UINT32_MAX
123 * cycles, when rx_time overflows.
124 * As the return value is also uint32_t, returning (rx_time - tx_time)
125 * is also fine when it overflows.
127 static uint32_t diff_time(uint32_t rx_time, uint32_t tx_time)
129 return rx_time - tx_time;
132 uint32_t task_lat_get_latency_bucket_size(struct task_lat *task)
134 return task->lat_test->bucket_size;
137 struct lat_test *task_lat_get_latency_meassurement(struct task_lat *task)
139 if (task->use_lt == task->using_lt)
140 return &task->lt[!task->using_lt];
144 void task_lat_use_other_latency_meassurement(struct task_lat *task)
146 task->use_lt = !task->using_lt;
149 static void task_lat_update_lat_test(struct task_lat *task)
151 if (task->use_lt != task->using_lt) {
152 task->using_lt = task->use_lt;
153 task->lat_test = &task->lt[task->using_lt];
154 task->lat_test->accuracy_limit_tsc = task->limit;
158 static int compare_tx_time(const void *val1, const void *val2)
160 const struct lat_info *ptr1 = val1;
161 const struct lat_info *ptr2 = val2;
163 return ptr1->tx_time > ptr2->tx_time ? 1 : -1;
166 static int compare_tx_packet_index(const void *val1, const void *val2)
168 const struct lat_info *ptr1 = val1;
169 const struct lat_info *ptr2 = val2;
171 return ptr1->tx_packet_index > ptr2->tx_packet_index ? 1 : -1;
174 static void fix_latency_buffer_tx_packet_index(struct lat_info *lat, uint32_t count)
176 uint32_t tx_packet_index, old_tx_packet_index = lat->tx_packet_index, n_overflow = 0;
177 uint32_t small = UINT32_MAX >> 1;
181 /* Buffer is sorted so far by RX time.
182 * We might have packets being reordered by SUT.
183 * => consider small differences as re-order and big ones as overflow of tx_packet_index.
185 * - overflow only happens if receiving and storing 4 billions packets...
186 * - a absolute difference of less than 2 billion packets is not considered as an overflow
188 for (uint32_t i = 1; i < count; i++) {
189 tx_packet_index = lat->tx_packet_index;
190 if (tx_packet_index > old_tx_packet_index) {
191 if (tx_packet_index - old_tx_packet_index < small) {
192 // The diff is small => increasing index count
194 // The diff is big => it is more likely that the previous packet was overflow
198 if (old_tx_packet_index - tx_packet_index < small) {
199 // The diff is small => packet reorder
201 // The diff is big => it is more likely that this is an overflow
205 lat->tx_packet_index += ((uint64_t)UINT32_MAX + 1) * n_overflow;
206 old_tx_packet_index = tx_packet_index;
211 static void fix_latency_buffer_tx_time(struct lat_info *lat, uint32_t count)
213 uint32_t tx_time, old_tx_time = lat->tx_time, n_overflow = 0;
214 uint32_t small = UINT32_MAX >> 1;
218 * Same algorithm as above, but with time.
220 * - overflow happens after 4 billions "cycles" (shifted by LATENCY_ACCURACY) = ~4sec
221 * - a absolute difference up to 2 billion (shifted) cycles (~=2sec) is not considered as an overflow
222 * => algorithm does not work if receiving less than 1 packet every 2 seconds
224 for (uint32_t i = 1; i < count; i++) {
225 tx_time = lat->tx_time;
226 if (tx_time > old_tx_time) {
227 if (tx_time - old_tx_time > small) {
231 if (old_tx_time - tx_time > small) {
235 lat->tx_time += ((uint64_t)UINT32_MAX + 1) * n_overflow;
236 old_tx_time = tx_time;
241 static void task_lat_count_remaining_lost_packets(struct task_lat *task)
243 struct lat_test *lat_test = task->lat_test;
245 for (uint32_t j = 0; j < task->generator_count; j++) {
246 struct early_loss_detect *eld = &task->eld[j];
248 lat_test->lost_packets += early_loss_detect_count_remaining_loss(eld);
252 static void task_lat_reset_eld(struct task_lat *task)
254 for (uint32_t j = 0; j < task->generator_count; j++) {
255 early_loss_detect_reset(&task->eld[j]);
259 static uint64_t lat_latency_buffer_get_min_tsc(struct task_lat *task)
261 uint64_t min_tsc = UINT64_MAX;
263 for (uint32_t i = 0; i < task->latency_buffer_idx; i++) {
264 if (min_tsc > task->latency_buffer[i].tx_time)
265 min_tsc = task->latency_buffer[i].tx_time;
268 return min_tsc << LATENCY_ACCURACY;
271 static uint64_t lat_info_get_lat_tsc(struct lat_info *lat_info)
273 uint64_t lat = diff_time(lat_info->rx_time, lat_info->tx_time);
275 return lat << LATENCY_ACCURACY;
278 static uint64_t lat_info_get_tx_err_tsc(const struct lat_info *lat_info)
280 return ((uint64_t)lat_info->tx_err) << LATENCY_ACCURACY;
283 static uint64_t lat_info_get_rx_err_tsc(const struct lat_info *lat_info)
285 return ((uint64_t)lat_info->rx_err) << LATENCY_ACCURACY;
288 static uint64_t lat_info_get_rx_tsc(const struct lat_info *lat_info)
290 return ((uint64_t)lat_info->rx_time) << LATENCY_ACCURACY;
293 static uint64_t lat_info_get_tx_tsc(const struct lat_info *lat_info)
295 return ((uint64_t)lat_info->tx_time) << LATENCY_ACCURACY;
298 static void lat_write_latency_to_file(struct task_lat *task)
303 min_tsc = lat_latency_buffer_get_min_tsc(task);
305 // Dumping all packet statistics
306 fprintf(task->fp_rx, "Latency stats for %u packets, ordered by rx time\n", task->latency_buffer_idx);
307 fprintf(task->fp_rx, "rx index; queue; tx index; lat (nsec);tx time;\n");
308 for (uint32_t i = 0; i < task->latency_buffer_idx ; i++) {
309 struct lat_info *lat_info = &task->latency_buffer[i];
310 uint64_t lat_tsc = lat_info_get_lat_tsc(lat_info);
311 uint64_t rx_tsc = lat_info_get_rx_tsc(lat_info);
312 uint64_t tx_tsc = lat_info_get_tx_tsc(lat_info);
314 fprintf(task->fp_rx, "%u;%u;%lu;%lu;%lu;%lu\n",
315 lat_info->rx_packet_index,
316 lat_info->port_queue_id,
317 lat_info->tx_packet_index,
318 tsc_to_nsec(lat_tsc),
319 tsc_to_nsec(rx_tsc - min_tsc),
320 tsc_to_nsec(tx_tsc - min_tsc));
323 // To detect dropped packets, we need to sort them based on TX
324 if (task->unique_id_pos) {
325 plogx_info("Adapting tx_packet_index\n");
326 fix_latency_buffer_tx_packet_index(task->latency_buffer, task->latency_buffer_idx);
327 plogx_info("Sorting packets based on tx_packet_index\n");
328 qsort (task->latency_buffer, task->latency_buffer_idx, sizeof(struct lat_info), compare_tx_packet_index);
329 plogx_info("Sorted packets based on packet_index\n");
331 plogx_info("Adapting tx_time\n");
332 fix_latency_buffer_tx_time(task->latency_buffer, task->latency_buffer_idx);
333 plogx_info("Sorting packets based on tx_time\n");
334 qsort (task->latency_buffer, task->latency_buffer_idx, sizeof(struct lat_info), compare_tx_time);
335 plogx_info("Sorted packets based on packet_time\n");
338 // A packet is marked as dropped if 2 packets received from the same queue are not consecutive
339 fprintf(task->fp_tx, "Latency stats for %u packets, sorted by tx time\n", task->latency_buffer_idx);
340 fprintf(task->fp_tx, "queue;tx index; rx index; lat (nsec);tx time; rx time; tx_err;rx_err\n");
342 for (uint32_t i = 0; i < task->generator_count;i++)
343 task->prev_tx_packet_index[i] = -1;
345 for (uint32_t i = 0; i < task->latency_buffer_idx; i++) {
346 struct lat_info *lat_info = &task->latency_buffer[i];
347 uint64_t lat_tsc = lat_info_get_lat_tsc(lat_info);
348 uint64_t tx_err_tsc = lat_info_get_tx_err_tsc(lat_info);
349 uint64_t rx_err_tsc = lat_info_get_rx_err_tsc(lat_info);
350 uint64_t rx_tsc = lat_info_get_rx_tsc(lat_info);
351 uint64_t tx_tsc = lat_info_get_tx_tsc(lat_info);
353 /* Packet n + ACCURACY_WINDOW delivers the TX error for packet n,
354 hence the last ACCURACY_WINDOW packets do no have TX error. */
355 if (i + ACCURACY_WINDOW >= task->latency_buffer_idx) {
359 if (lat_info->port_queue_id >= task->generator_count) {
360 plog_err("Unexpected generator id %u for packet %lu - skipping packet\n",
361 lat_info->port_queue_id, lat_info->tx_packet_index);
364 // Log dropped packet
365 n_loss = lat_info->tx_packet_index - task->prev_tx_packet_index[lat_info->port_queue_id] - 1;
367 fprintf(task->fp_tx, "===> %u;%lu;0;0;0;0;0;0 lost %lu packets <===\n",
368 lat_info->port_queue_id,
369 lat_info->tx_packet_index - n_loss, n_loss);
371 fprintf(task->fp_tx, "%u;%lu;%u;%lu;%lu;%lu;%lu;%lu",
372 lat_info->port_queue_id,
373 lat_info->tx_packet_index,
374 lat_info->rx_packet_index,
375 tsc_to_nsec(lat_tsc),
376 tsc_to_nsec(tx_tsc - min_tsc),
377 tsc_to_nsec(rx_tsc - min_tsc),
378 tsc_to_nsec(tx_err_tsc),
379 tsc_to_nsec(rx_err_tsc));
381 fprintf(task->fp_tx, ";%u from %u;%lu;%lu;%lu",
382 lat_info->id_in_bulk,
384 tsc_to_nsec(lat_info->begin - min_tsc),
385 tsc_to_nsec(lat_info->before - min_tsc),
386 tsc_to_nsec(lat_info->after - min_tsc));
388 fprintf(task->fp_tx, "\n");
389 task->prev_tx_packet_index[lat_info->port_queue_id] = lat_info->tx_packet_index;
393 task->latency_buffer_idx = 0;
396 static void lat_stop(struct task_base *tbase)
398 struct task_lat *task = (struct task_lat *)tbase;
400 if (task->unique_id_pos) {
401 task_lat_count_remaining_lost_packets(task);
402 task_lat_reset_eld(task);
404 if (task->latency_buffer)
405 lat_write_latency_to_file(task);
409 static void task_lat_store_lat_debug(struct task_lat *task, uint32_t rx_packet_index, uint32_t id_in_bulk, uint32_t bulk_size)
411 struct lat_info *lat_info = &task->latency_buffer[rx_packet_index];
413 lat_info->bulk_size = bulk_size;
414 lat_info->id_in_bulk = id_in_bulk;
415 lat_info->begin = task->begin;
416 lat_info->before = task->base.aux->tsc_rx.before;
417 lat_info->after = task->base.aux->tsc_rx.after;
421 static void task_lat_store_lat_buf(struct task_lat *task, uint64_t rx_packet_index, uint64_t rx_time, uint64_t tx_time, uint64_t rx_err, uint64_t tx_err, uint32_t packet_id, uint8_t generator_id)
423 struct lat_info *lat_info;
425 /* If unique_id_pos is specified then latency is stored per
426 packet being sent. Lost packets are detected runtime, and
427 latency stored for those packets will be 0 */
428 lat_info = &task->latency_buffer[task->latency_buffer_idx++];
429 lat_info->rx_packet_index = rx_packet_index;
430 lat_info->tx_packet_index = packet_id;
431 lat_info->port_queue_id = generator_id;
432 lat_info->rx_time = rx_time;
433 lat_info->tx_time = tx_time;
434 lat_info->rx_err = rx_err;
435 lat_info->tx_err = tx_err;
438 static uint32_t task_lat_early_loss_detect(struct task_lat *task, uint32_t packet_id, uint8_t generator_id)
440 struct early_loss_detect *eld = &task->eld[generator_id];
441 return early_loss_detect_add(eld, packet_id);
444 static void lat_test_check_duplicate(struct task_lat *task, struct lat_test *lat_test, uint32_t packet_id, uint8_t generator_id)
446 struct early_loss_detect *eld = &task->eld[generator_id];
447 uint32_t old_queue_id, queue_pos;
449 queue_pos = packet_id & PACKET_QUEUE_MASK;
450 old_queue_id = eld->entries[queue_pos];
451 if ((packet_id >> PACKET_QUEUE_BITS) == old_queue_id)
452 lat_test->duplicate++;
455 static uint64_t tsc_extrapolate_backward(struct task_lat *task, uint64_t tsc_from, uint64_t bytes, uint64_t tsc_minimum)
457 #ifdef NO_LAT_EXTRAPOLATION
458 uint64_t tsc = tsc_from;
460 uint64_t tsc = tsc_from - task->bytes_to_tsc[bytes];
462 if (likely(tsc > tsc_minimum))
468 static void lat_test_histogram_add(struct lat_test *lat_test, uint64_t lat_tsc)
470 uint64_t bucket_id = (lat_tsc >> lat_test->bucket_size);
471 size_t bucket_count = sizeof(lat_test->buckets)/sizeof(lat_test->buckets[0]);
473 bucket_id = bucket_id < bucket_count? bucket_id : (bucket_count - 1);
474 lat_test->buckets[bucket_id]++;
477 static void lat_test_check_ordering(struct task_lat *task, struct lat_test *lat_test, uint32_t packet_id, uint8_t generator_id)
479 if (packet_id < task->previous_packet[generator_id]) {
480 lat_test->mis_ordered++;
481 lat_test->extent += task->previous_packet[generator_id] - packet_id;
483 task->previous_packet[generator_id] = packet_id;
486 static void lat_test_add_lost(struct lat_test *lat_test, uint64_t lost_packets)
488 lat_test->lost_packets += lost_packets;
491 static void lat_test_add_latency(struct lat_test *lat_test, uint64_t lat_tsc, uint64_t error)
493 if (error > lat_test->accuracy_limit_tsc)
495 lat_test->tot_pkts++;
497 lat_test->tot_lat += lat_tsc;
498 lat_test->tot_lat_error += error;
500 /* (a +- b)^2 = a^2 +- (2ab + b^2) */
501 lat_test->var_lat += lat_tsc * lat_tsc;
502 lat_test->var_lat_error += 2 * lat_tsc * error;
503 lat_test->var_lat_error += error * error;
505 if (lat_tsc > lat_test->max_lat) {
506 lat_test->max_lat = lat_tsc;
507 lat_test->max_lat_error = error;
509 if (lat_tsc < lat_test->min_lat) {
510 lat_test->min_lat = lat_tsc;
511 lat_test->min_lat_error = error;
514 #ifdef LATENCY_HISTOGRAM
515 lat_test_histogram_add(lat_test, lat_tsc);
519 static int task_lat_can_store_latency(struct task_lat *task)
521 return task->latency_buffer_idx < task->latency_buffer_size;
524 static void task_lat_store_lat(struct task_lat *task, uint64_t rx_packet_index, uint64_t rx_time, uint64_t tx_time, uint64_t rx_error, uint64_t tx_error, uint32_t packet_id, uint8_t generator_id)
526 uint32_t lat_tsc = diff_time(rx_time, tx_time) << LATENCY_ACCURACY;
528 lat_test_add_latency(task->lat_test, lat_tsc, rx_error + tx_error);
530 if (task_lat_can_store_latency(task)) {
531 task_lat_store_lat_buf(task, rx_packet_index, rx_time, tx_time, rx_error, tx_error, packet_id, generator_id);
535 static int handle_lat_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
537 struct task_lat *task = (struct task_lat *)tbase;
541 task->begin = tbase->aux->tsc_rx.before;
545 task_lat_update_lat_test(task);
547 // Remember those packets with bad length or bad signature
548 uint32_t non_dp_count = 0;
549 uint64_t pkt_bad_len_sig = 0;
550 #define BIT64_SET(a64, bit) a64 |= (((uint64_t)1) << (bit & 63))
551 #define BIT64_CLR(a64, bit) a64 &= ~(((uint64_t)1) << (bit & 63))
552 #define BIT64_TEST(a64, bit) a64 & (((uint64_t)1) << (bit & 63))
554 /* Go once through all received packets and read them. If
555 packet has just been modified by another core, the cost of
556 latency will be partialy amortized though the bulk size */
557 for (uint16_t j = 0; j < n_pkts; ++j) {
558 struct rte_mbuf *mbuf = mbufs[j];
559 task->rx_pkt_meta[j].hdr = rte_pktmbuf_mtod(mbuf, uint8_t *);
561 // Remember those packets which are too short to hold the values that we expect
562 if (unlikely(rte_pktmbuf_pkt_len(mbuf) < task->min_pkt_len)) {
563 BIT64_SET(pkt_bad_len_sig, j);
566 BIT64_CLR(pkt_bad_len_sig, j);
570 for (uint16_t j = 0; j < n_pkts; ++j) {
571 if (unlikely(BIT64_TEST(pkt_bad_len_sig, j)))
573 // Remember those packets with bad signature
574 if (likely(*(uint32_t *)(task->rx_pkt_meta[j].hdr + task->sig_pos) == task->sig))
575 task->rx_pkt_meta[j].pkt_tx_time = *(uint32_t *)(task->rx_pkt_meta[j].hdr + task->lat_pos);
577 BIT64_SET(pkt_bad_len_sig, j);
582 for (uint16_t j = 0; j < n_pkts; ++j) {
583 if (unlikely(BIT64_TEST(pkt_bad_len_sig, j)))
585 task->rx_pkt_meta[j].pkt_tx_time = *(uint32_t *)(task->rx_pkt_meta[j].hdr + task->lat_pos);
589 uint32_t bytes_total_in_bulk = 0;
590 // Find RX time of first packet, for RX accuracy
591 for (uint16_t j = 0; j < n_pkts; ++j) {
592 uint16_t flipped = n_pkts - 1 - j;
594 task->rx_pkt_meta[flipped].bytes_after_in_bulk = bytes_total_in_bulk;
595 bytes_total_in_bulk += mbuf_wire_size(mbufs[flipped]);
598 const uint64_t rx_tsc = tbase->aux->tsc_rx.after;
600 uint64_t rx_time_err;
601 uint64_t pkt_rx_time64 = tsc_extrapolate_backward(task, rx_tsc, task->rx_pkt_meta[0].bytes_after_in_bulk, task->last_pkts_tsc) >> LATENCY_ACCURACY;
602 if (unlikely((task->begin >> LATENCY_ACCURACY) > pkt_rx_time64)) {
603 // Extrapolation went up to BEFORE begin => packets were stuck in the NIC but we were not seeing them
604 rx_time_err = pkt_rx_time64 - (task->last_pkts_tsc >> LATENCY_ACCURACY);
606 rx_time_err = pkt_rx_time64 - (task->begin >> LATENCY_ACCURACY);
609 TASK_STATS_ADD_RX_NON_DP(&tbase->aux->stats, non_dp_count);
610 for (uint16_t j = 0; j < n_pkts; ++j) {
611 // Used to display % of packets within accuracy limit vs. total number of packets (used_col)
612 task->lat_test->tot_all_pkts++;
614 // Skip those packets with bad length or bad signature
615 if (unlikely(BIT64_TEST(pkt_bad_len_sig, j)))
618 struct rx_pkt_meta_data *rx_pkt_meta = &task->rx_pkt_meta[j];
619 uint8_t *hdr = rx_pkt_meta->hdr;
621 uint32_t pkt_rx_time = tsc_extrapolate_backward(task, rx_tsc, rx_pkt_meta->bytes_after_in_bulk, task->last_pkts_tsc) >> LATENCY_ACCURACY;
622 uint32_t pkt_tx_time = rx_pkt_meta->pkt_tx_time;
624 uint8_t generator_id;
626 if (task->unique_id_pos) {
627 struct unique_id *unique_id = (struct unique_id *)(hdr + task->unique_id_pos);
628 unique_id_get(unique_id, &generator_id, &packet_id);
630 if (unlikely(generator_id >= task->generator_count)) {
631 /* No need to remember unexpected packet at this stage
632 BIT64_SET(pkt_bad_len_sig, j);
634 // Skip unexpected packet
637 lat_test_check_ordering(task, task->lat_test, packet_id, generator_id);
638 lat_test_check_duplicate(task, task->lat_test, packet_id, generator_id);
639 lat_test_add_lost(task->lat_test, task_lat_early_loss_detect(task, packet_id, generator_id));
642 packet_id = task->rx_packet_index;
645 /* If accuracy is enabled, latency is reported with a
646 delay of ACCURACY_WINDOW packets since the generator puts the
647 accuracy for packet N into packet N + ACCURACY_WINDOW. The delay
648 ensures that all reported latencies have both rx
650 if (task->accur_pos) {
651 uint32_t tx_time_err = *(uint32_t *)(hdr + task->accur_pos);
653 struct delayed_latency_entry *delayed_latency_entry = delayed_latency_get(task->delayed_latency_entries, generator_id, packet_id - ACCURACY_WINDOW);
655 if (delayed_latency_entry) {
656 task_lat_store_lat(task,
657 delayed_latency_entry->rx_packet_id,
658 delayed_latency_entry->pkt_rx_time,
659 delayed_latency_entry->pkt_tx_time,
660 delayed_latency_entry->rx_time_err,
662 delayed_latency_entry->tx_packet_id,
663 delayed_latency_entry->generator_id);
666 delayed_latency_entry = delayed_latency_create(task->delayed_latency_entries, generator_id, packet_id);
667 delayed_latency_entry->pkt_rx_time = pkt_rx_time;
668 delayed_latency_entry->pkt_tx_time = pkt_tx_time;
669 delayed_latency_entry->rx_time_err = rx_time_err;
670 delayed_latency_entry->rx_packet_id = task->rx_packet_index;
671 delayed_latency_entry->tx_packet_id = packet_id;
672 delayed_latency_entry->generator_id = generator_id;
674 task_lat_store_lat(task, task->rx_packet_index, pkt_rx_time, pkt_tx_time, 0, 0, packet_id, generator_id);
677 // Bad/unexpected packets do not need to be indexed
678 task->rx_packet_index++;
681 if (n_pkts < MAX_PKT_BURST)
682 task->begin = tbase->aux->tsc_rx.before;
683 task->last_pkts_tsc = tbase->aux->tsc_rx.after;
685 rc = task->base.tx_pkt(&task->base, mbufs, n_pkts, NULL);
686 // non_dp_count should not be drop-handled, as there are all by definition considered as not handled
687 // RX = DISCARDED + HANDLED + NON_DP + (TX - TX_NON_DP) + TX_FAIL
688 TASK_STATS_ADD_DROP_HANDLED(&tbase->aux->stats, -non_dp_count);
692 static void init_task_lat_latency_buffer(struct task_lat *task, uint32_t core_id)
694 const int socket_id = rte_lcore_to_socket_id(core_id);
696 size_t latency_buffer_mem_size = 0;
698 if (task->latency_buffer_size > UINT32_MAX - MAX_RING_BURST)
699 task->latency_buffer_size = UINT32_MAX - MAX_RING_BURST;
701 latency_buffer_mem_size = sizeof(struct lat_info) * task->latency_buffer_size;
703 task->latency_buffer = prox_zmalloc(latency_buffer_mem_size, socket_id);
704 PROX_PANIC(task->latency_buffer == NULL, "Failed to allocate %zu kbytes for latency_buffer\n", latency_buffer_mem_size / 1024);
706 sprintf(name, "latency.rx_%u.txt", core_id);
707 task->fp_rx = fopen(name, "w+");
708 PROX_PANIC(task->fp_rx == NULL, "Failed to open %s\n", name);
710 sprintf(name, "latency.tx_%u.txt", core_id);
711 task->fp_tx = fopen(name, "w+");
712 PROX_PANIC(task->fp_tx == NULL, "Failed to open %s\n", name);
714 task->prev_tx_packet_index = prox_zmalloc(sizeof(task->prev_tx_packet_index[0]) * task->generator_count, socket_id);
715 PROX_PANIC(task->prev_tx_packet_index == NULL, "Failed to allocated prev_tx_packet_index\n");
718 static void task_init_generator_count(struct task_lat *task)
720 uint8_t *generator_count = prox_sh_find_system("generator_count");
722 if (generator_count == NULL) {
723 task->generator_count = 1;
724 plog_info("\tNo generators found, hard-coding to %u generators\n", task->generator_count);
726 task->generator_count = *generator_count;
727 plog_info("\t\tLatency using %u generators\n", task->generator_count);
730 static void task_lat_init_eld(struct task_lat *task, uint8_t socket_id)
734 eld_mem_size = sizeof(task->eld[0]) * task->generator_count;
735 task->eld = prox_zmalloc(eld_mem_size, socket_id);
736 PROX_PANIC(task->eld == NULL, "Failed to allocate eld\n");
739 void task_lat_set_accuracy_limit(struct task_lat *task, uint32_t accuracy_limit_nsec)
741 task->limit = nsec_to_tsc(accuracy_limit_nsec);
744 static void lat_start(struct task_base *tbase)
746 struct task_lat *task = (struct task_lat *)tbase;
750 static void init_task_lat(struct task_base *tbase, struct task_args *targ)
752 struct task_lat *task = (struct task_lat *)tbase;
753 const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
755 task->lat_pos = targ->lat_pos;
756 task->accur_pos = targ->accur_pos;
757 task->sig_pos = targ->sig_pos;
758 task->sig = targ->sig;
760 task->unique_id_pos = targ->packet_id_pos;
761 task->latency_buffer_size = targ->latency_buffer_size;
763 PROX_PANIC(task->lat_pos == 0, "Missing 'lat pos' parameter in config file\n");
764 uint16_t min_pkt_len = task->lat_pos + sizeof(uint32_t);
765 if (task->unique_id_pos && (
766 min_pkt_len < task->unique_id_pos + sizeof(struct unique_id)))
767 min_pkt_len = task->unique_id_pos + sizeof(struct unique_id);
768 if (task->accur_pos && (
769 min_pkt_len < task->accur_pos + sizeof(uint32_t)))
770 min_pkt_len = task->accur_pos + sizeof(uint32_t);
771 if (task->sig_pos && (
772 min_pkt_len < task->sig_pos + sizeof(uint32_t)))
773 min_pkt_len = task->sig_pos + sizeof(uint32_t);
774 task->min_pkt_len = min_pkt_len;
776 task_init_generator_count(task);
778 if (task->latency_buffer_size) {
779 init_task_lat_latency_buffer(task, targ->lconf->id);
782 if (targ->bucket_size < DEFAULT_BUCKET_SIZE) {
783 targ->bucket_size = DEFAULT_BUCKET_SIZE;
786 if (task->accur_pos) {
787 task->delayed_latency_entries = prox_zmalloc(sizeof(*task->delayed_latency_entries) * task->generator_count , socket_id);
788 PROX_PANIC(task->delayed_latency_entries == NULL, "Failed to allocate array for storing delayed latency entries\n");
789 for (uint i = 0; i < task->generator_count; i++) {
790 task->delayed_latency_entries[i] = prox_zmalloc(sizeof(**task->delayed_latency_entries) * ACCURACY_BUFFER_SIZE, socket_id);
791 PROX_PANIC(task->delayed_latency_entries[i] == NULL, "Failed to allocate array for storing delayed latency entries\n");
793 if (task->unique_id_pos == 0) {
794 /* When using accuracy feature, the accuracy from TX is written ACCURACY_WINDOW packets later
795 * We can only retrieve the good packet if a packet id is written to it.
796 * Otherwise we will use the packet RECEIVED ACCURACY_WINDOW packets ago which is OK if
797 * packets are not re-ordered. If packets are re-ordered, then the matching between
798 * the TX accuracy and the latency is wrong.
800 plog_warn("\tWhen accuracy feature is used, a unique id should ideally also be used\n");
804 task->lt[0].min_lat = -1;
805 task->lt[1].min_lat = -1;
806 task->lt[0].bucket_size = targ->bucket_size;
807 task->lt[1].bucket_size = targ->bucket_size;
808 if (task->unique_id_pos) {
809 task_lat_init_eld(task, socket_id);
810 task_lat_reset_eld(task);
811 task->previous_packet = prox_zmalloc(sizeof(task->previous_packet) * task->generator_count , socket_id);
812 PROX_PANIC(task->previous_packet == NULL, "Failed to allocate array for storing previous packet\n");
814 task->lat_test = &task->lt[task->using_lt];
816 task_lat_set_accuracy_limit(task, targ->accuracy_limit_nsec);
817 task->rx_pkt_meta = prox_zmalloc(MAX_PKT_BURST * sizeof(*task->rx_pkt_meta), socket_id);
818 PROX_PANIC(task->rx_pkt_meta == NULL, "unable to allocate memory to store RX packet meta data");
820 uint32_t max_frame_size = MAX_PKT_SIZE;
821 uint64_t bytes_per_hz = UINT64_MAX;
822 if (targ->nb_rxports) {
823 struct prox_port_cfg *port = &prox_port_cfg[targ->rx_port_queue[0].port];
824 max_frame_size = port->mtu + PROX_RTE_ETHER_HDR_LEN + PROX_RTE_ETHER_CRC_LEN + 2 * PROX_VLAN_TAG_SIZE;
826 // port->max_link_speed reports the maximum, non negotiated ink speed in Mbps e.g. 40k for a 40 Gbps NIC.
827 // It can be UINT32_MAX (virtual devices or not supported by DPDK < 16.04)
828 if (port->max_link_speed != UINT32_MAX) {
829 bytes_per_hz = port->max_link_speed * 125000L;
830 plog_info("\t\tPort %u: max link speed is %ld Mbps\n",
831 (uint8_t)(port - prox_port_cfg), 8 * bytes_per_hz / 1000000);
834 task->bytes_to_tsc = prox_zmalloc(max_frame_size * sizeof(task->bytes_to_tsc[0]) * MAX_PKT_BURST, rte_lcore_to_socket_id(targ->lconf->id));
835 PROX_PANIC(task->bytes_to_tsc == NULL,
836 "Failed to allocate %u bytes (in huge pages) for bytes_to_tsc\n", max_frame_size);
838 // There are cases where hz estimate might be slighly over-estimated
839 // This results in too much extrapolation
840 // Only account for 99% of extrapolation to handle cases with up to 1% error clocks
841 for (unsigned int i = 0; i < max_frame_size * MAX_PKT_BURST ; i++) {
842 if (bytes_per_hz == UINT64_MAX)
843 task->bytes_to_tsc[i] = 0;
845 task->bytes_to_tsc[i] = (rte_get_tsc_hz() * i * 0.99) / bytes_per_hz;
849 static struct task_init task_init_lat = {
851 .init = init_task_lat,
852 .handle = handle_lat_bulk,
855 .flag_features = TASK_FEATURE_TSC_RX | TASK_FEATURE_ZERO_RX | TASK_FEATURE_NEVER_DISCARDS,
856 .size = sizeof(struct task_lat)
859 __attribute__((constructor)) static void reg_task_lat(void)
861 reg_task(&task_init_lat);