Merge "Initial support for DPDK 18.05"
[samplevnf.git] / VNFs / DPPD-PROX / task_init.c
1 /*
2 // Copyright (c) 2010-2017 Intel Corporation
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 //     http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 */
16
17 #include <string.h>
18 #include <stdio.h>
19 #include <rte_version.h>
20
21 #include "prox_port_cfg.h"
22 #include "prox_malloc.h"
23 #include "task_init.h"
24 #include "rx_pkt.h"
25 #include "tx_pkt.h"
26 #include "log.h"
27 #include "quit.h"
28 #include "lconf.h"
29 #include "thread_generic.h"
30 #include "prox_assert.h"
31
32 #if RTE_VERSION < RTE_VERSION_NUM(1,8,0,0)
33 #define RTE_CACHE_LINE_SIZE CACHE_LINE_SIZE
34 #endif
35
36 static unsigned first_task = 1;
37 LIST_HEAD(,task_init) head;
38
39 void reg_task(struct task_init* t)
40 {
41         // PROX_PANIC(t->handle == NULL, "No handle function specified for task with name %d\n", t->mode);
42
43         if (t->thread_x == NULL)
44                 t->thread_x = thread_generic;
45
46         if (first_task) {
47                 first_task = 0;
48                 LIST_INIT(&head);
49         }
50
51         LIST_INSERT_HEAD(&head, t, entries);
52 }
53
54 struct task_init *to_task_init(const char *mode_str, const char *sub_mode_str)
55 {
56         struct task_init *cur_t;
57
58         LIST_FOREACH(cur_t, &head, entries) {
59                 if (!strcmp(mode_str, cur_t->mode_str) &&
60                     !strcmp(sub_mode_str, cur_t->sub_mode_str)) {
61                         return cur_t;
62                 }
63         }
64
65         return NULL;
66 }
67
68 static int compare_strcmp(const void *a, const void *b)
69 {
70         return strcmp(*(const char * const *)a, *(const char * const *)b);
71 }
72
73 int task_is_master(struct task_args *targ)
74 {
75         return (targ->lconf->id == prox_cfg.master);
76 }
77
78 void tasks_list(void)
79 {
80         struct task_init *cur_t;
81         char buf[sizeof(cur_t->mode_str) + sizeof(cur_t->sub_mode_str) + 4];
82
83         int nb_modes = 1; /* master */
84         LIST_FOREACH(cur_t, &head, entries) {
85                 ++nb_modes;
86         }
87
88         char **modes = calloc(nb_modes, sizeof(*modes));
89         char **cur_m = modes;
90         *cur_m++ = strdup("master");
91         LIST_FOREACH(cur_t, &head, entries) {
92                 snprintf(buf, sizeof(buf), "%s%s%s",
93                         cur_t->mode_str,
94                         (cur_t->sub_mode_str[0] == 0) ? "" : " / ",
95                         cur_t->sub_mode_str);
96                 *cur_m++ = strdup(buf);
97         }
98         qsort(modes, nb_modes, sizeof(*modes), compare_strcmp);
99
100         plog_info("=== List of supported task modes / sub modes ===\n");
101         for (cur_m = modes; nb_modes; ++cur_m, --nb_modes) {
102                 plog_info("\t%s\n", *cur_m);
103                 free(*cur_m);
104         }
105         free(modes);
106 }
107
108 static size_t calc_memsize(struct task_args *targ, size_t task_size)
109 {
110         size_t memsize = task_size;
111
112         memsize += sizeof(struct task_base_aux);
113
114         if (targ->nb_rxports != 0) {
115                 memsize += 2 * sizeof(uint8_t)*targ->nb_rxports;
116         }
117         if (targ->nb_rxrings != 0 || targ->tx_opt_ring_task) {
118                 memsize += sizeof(struct rte_ring *)*targ->nb_rxrings;
119         }
120         if (targ->nb_txrings != 0) {
121                 memsize += sizeof(struct rte_ring *) * targ->nb_txrings;
122                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
123                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txrings;
124         }
125         else if (targ->nb_txports != 0) {
126                 memsize += sizeof(struct port_queue) * targ->nb_txports;
127                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
128                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txports;
129         }
130         else {
131                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
132                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
133         }
134
135         return memsize;
136 }
137
138 static void *flush_function(struct task_args *targ)
139 {
140         if (targ->flags & TASK_ARG_DROP) {
141                 return targ->nb_txrings ? flush_queues_sw : flush_queues_hw;
142         }
143         else {
144                 return targ->nb_txrings ? flush_queues_no_drop_sw : flush_queues_no_drop_hw;
145         }
146 }
147
148 static size_t init_rx_tx_rings_ports(struct task_args *targ, struct task_base *tbase, size_t offset)
149 {
150         if (targ->tx_opt_ring_task) {
151                 tbase->rx_pkt = rx_pkt_self;
152         }
153         else if (targ->nb_rxrings != 0) {
154
155                 if (targ->nb_rxrings == 1) {
156                         tbase->rx_pkt = rx_pkt_sw1;
157                         tbase->rx_params_sw1.rx_ring = targ->rx_rings[0];
158                 }
159                 else {
160                         tbase->rx_pkt = rx_pkt_sw;
161                         tbase->rx_params_sw.nb_rxrings = targ->nb_rxrings;
162                         tbase->rx_params_sw.rx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
163                         offset += sizeof(struct rte_ring *)*tbase->rx_params_sw.nb_rxrings;
164
165                         for (uint8_t i = 0; i < tbase->rx_params_sw.nb_rxrings; ++i) {
166                                 tbase->rx_params_sw.rx_rings[i] = targ->rx_rings[i];
167                         }
168
169                         if (rte_is_power_of_2(targ->nb_rxrings)) {
170                                 tbase->rx_pkt = rx_pkt_sw_pow2;
171                                 tbase->rx_params_sw.rxrings_mask = targ->nb_rxrings - 1;
172                         }
173                 }
174         }
175         else {
176                 if (targ->nb_rxports == 1) {
177                         if (targ->flags & TASK_ARG_L3)
178                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi_l3 : rx_pkt_hw1_l3;
179                         else
180                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi : rx_pkt_hw1;
181                         tbase->rx_params_hw1.rx_pq.port =  targ->rx_port_queue[0].port;
182                         tbase->rx_params_hw1.rx_pq.queue = targ->rx_port_queue[0].queue;
183                 }
184                 else {
185                         PROX_ASSERT((targ->nb_rxports != 0) || (targ->task_init->flag_features & TASK_FEATURE_NO_RX));
186                         if (targ->flags & TASK_ARG_L3)
187                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi_l3 : rx_pkt_hw_l3;
188                         else
189                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi : rx_pkt_hw;
190                         tbase->rx_params_hw.nb_rxports = targ->nb_rxports;
191                         tbase->rx_params_hw.rx_pq = (struct port_queue *)(((uint8_t *)tbase) + offset);
192                         offset += sizeof(struct port_queue) * tbase->rx_params_hw.nb_rxports;
193                         for (int i = 0; i< targ->nb_rxports; i++) {
194                                 tbase->rx_params_hw.rx_pq[i].port = targ->rx_port_queue[i].port;
195                                 tbase->rx_params_hw.rx_pq[i].queue = targ->rx_port_queue[i].queue;
196                         }
197
198                         if (rte_is_power_of_2(targ->nb_rxports)) {
199                                 if (targ->flags & TASK_ARG_L3)
200                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi_l3 : rx_pkt_hw_pow2_l3;
201                                 else
202                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi : rx_pkt_hw_pow2;
203                                 tbase->rx_params_hw.rxport_mask = targ->nb_rxports - 1;
204                         }
205                 }
206         }
207
208         if ((targ->nb_txrings != 0) && (!targ->tx_opt_ring) && (!(targ->flags & TASK_ARG_DROP))) {
209                 // Transmitting to a ring in NO DROP. We need to make sure the receiving task in not running on the same core.
210                 // Otherwise we might end up in a dead lock: trying in a loop to transmit to a task which cannot receive anymore
211                 // (as npt being scheduled).
212                 struct core_task ct;
213                 struct task_args *dtarg;
214                 for (unsigned int j = 0; j < targ->nb_txrings; j++) {
215                         ct = targ->core_task_set[0].core_task[j];
216                         PROX_PANIC(ct.core == targ->lconf->id, "Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
217                         //plog_info("Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
218                 }
219         }
220         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
221                 /* Transmitting to multiple rings and one port */
222                 plog_info("Initializing with 1 port %d queue %d nb_rings=%d\n", targ->tx_port_queue[0].port, targ->tx_port_queue[0].queue, targ->nb_txrings);
223                 tbase->tx_params_hw_sw.tx_port_queue.port =  targ->tx_port_queue[0].port;
224                 tbase->tx_params_hw_sw.tx_port_queue.queue =  targ->tx_port_queue[0].queue;
225                 if (!targ->tx_opt_ring) {
226                         tbase->tx_params_hw_sw.nb_txrings = targ->nb_txrings;
227                         tbase->tx_params_hw_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
228                         offset += sizeof(struct rte_ring *)*tbase->tx_params_hw_sw.nb_txrings;
229
230                         for (uint8_t i = 0; i < tbase->tx_params_hw_sw.nb_txrings; ++i) {
231                                 tbase->tx_params_hw_sw.tx_rings[i] = targ->tx_rings[i];
232                         }
233
234                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
235                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
236                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw_sw.nb_txrings;
237                 }
238         }
239         else if (!targ->tx_opt_ring) {
240                 if (targ->nb_txrings != 0) {
241                         tbase->tx_params_sw.nb_txrings = targ->nb_txrings;
242                         tbase->tx_params_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
243                         offset += sizeof(struct rte_ring *)*tbase->tx_params_sw.nb_txrings;
244
245                         for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
246                                 tbase->tx_params_sw.tx_rings[i] = targ->tx_rings[i];
247                         }
248
249                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
250                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
251                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_sw.nb_txrings;
252                 }
253                 else if (targ->nb_txports != 0) {
254                         tbase->tx_params_hw.nb_txports = targ->nb_txports;
255                         tbase->tx_params_hw.tx_port_queue = (struct port_queue *)(((uint8_t *)tbase) + offset);
256                         offset += sizeof(struct port_queue) * tbase->tx_params_hw.nb_txports;
257                         for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
258                                 tbase->tx_params_hw.tx_port_queue[i].port = targ->tx_port_queue[i].port;
259                                 tbase->tx_params_hw.tx_port_queue[i].queue = targ->tx_port_queue[i].queue;
260                         }
261
262                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
263                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
264                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw.nb_txports;
265                 }
266                 else {
267                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
268                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
269                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
270                 }
271
272                 struct ws_mbuf* w = tbase->ws_mbuf;
273                 struct task_args *prev = targ->tx_opt_ring_task;
274
275                 while (prev) {
276                         prev->tbase->ws_mbuf = w;
277                         prev = prev->tx_opt_ring_task;
278                 }
279         }
280         if (targ->nb_txrings == 1 || targ->nb_txports == 1 || targ->tx_opt_ring) {
281                 if (targ->task_init->flag_features & TASK_FEATURE_NEVER_DISCARDS) {
282                         if (targ->tx_opt_ring) {
283                                 tbase->tx_pkt = tx_pkt_never_discard_self;
284                         }
285                         else if (targ->flags & TASK_ARG_DROP) {
286                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
287                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_thrpt_opt;
288                                 else
289                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_lat_opt;
290                         }
291                         else {
292                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
293                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_thrpt_opt;
294                                 else
295                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_lat_opt;
296                         }
297                         if ((targ->nb_txrings) || ((targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT) == 0))
298                                 tbase->flags |= FLAG_NEVER_FLUSH;
299                         else
300                                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
301                 }
302                 else {
303                         if (targ->tx_opt_ring) {
304                                 tbase->tx_pkt = tx_pkt_self;
305                         }
306                         else if (targ->flags & TASK_ARG_DROP) {
307                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw1 : tx_pkt_hw1;
308                         }
309                         else {
310                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw1 : tx_pkt_no_drop_hw1;
311                         }
312                         tbase->flags |= FLAG_NEVER_FLUSH;
313                 }
314         }
315         else {
316                 if (targ->flags & TASK_ARG_DROP) {
317                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw : tx_pkt_hw;
318                 }
319                 else {
320                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw : tx_pkt_no_drop_hw;
321                 }
322
323                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
324         }
325
326         if (targ->task_init->flag_features & TASK_FEATURE_NO_RX) {
327                 tbase->rx_pkt = rx_pkt_dummy;
328         }
329
330         if (targ->nb_txrings == 0 && targ->nb_txports == 0) {
331                 tbase->tx_pkt = tx_pkt_drop_all;
332         }
333
334         return offset;
335 }
336
337 struct task_base *init_task_struct(struct task_args *targ)
338 {
339         struct task_init* t = targ->task_init;
340         size_t offset = 0;
341         size_t memsize = calc_memsize(targ, t->size);
342         uint8_t task_socket = rte_lcore_to_socket_id(targ->lconf->id);
343         struct task_base *tbase = prox_zmalloc(memsize, task_socket);
344         PROX_PANIC(tbase == NULL, "Failed to allocate memory for task (%zu bytes)", memsize);
345         offset += t->size;
346
347         if (targ->nb_txrings == 0 && targ->nb_txports == 0)
348                 tbase->flags |= FLAG_NEVER_FLUSH;
349
350         offset = init_rx_tx_rings_ports(targ, tbase, offset);
351         tbase->aux = (struct task_base_aux *)(((uint8_t *)tbase) + offset);
352
353         if (targ->nb_txports != 0) {
354                 if (targ->flags & TASK_ARG_L3) {
355                         tbase->aux->tx_pkt_l2 = tbase->tx_pkt;
356                         tbase->tx_pkt = tx_pkt_l3;
357                 }
358         }
359
360         if (targ->task_init->flag_features & TASK_FEATURE_RX_ALL) {
361                 task_base_add_rx_pkt_function(tbase, rx_pkt_all);
362                 tbase->aux->all_mbufs = prox_zmalloc(MAX_RX_PKT_ALL * sizeof(* tbase->aux->all_mbufs), task_socket);
363         }
364         if (targ->task_init->flag_features & TASK_FEATURE_TSC_RX) {
365                 task_base_add_rx_pkt_function(tbase, rx_pkt_tsc);
366         }
367
368         offset += sizeof(struct task_base_aux);
369
370         tbase->handle_bulk = t->handle;
371
372         if (targ->flags & TASK_ARG_L3) {
373                 plog_info("\tTask configured in L3 mode\n");
374                 tbase->l3.ctrl_plane_ring = targ->ctrl_plane_ring;
375         }
376         if (targ->nb_txports != 0) {
377                 if (targ->flags & TASK_ARG_L3)
378                         task_init_l3(tbase, targ);
379         }
380
381         targ->tbase = tbase;
382         if (t->init) {
383                 t->init(tbase, targ);
384         }
385         tbase->aux->start = t->start;
386         tbase->aux->stop = t->stop;
387         tbase->aux->start_first = t->start_first;
388         tbase->aux->stop_last = t->stop_last;
389         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
390                 tbase->aux->tx_pkt_hw = tx_pkt_no_drop_never_discard_hw1_no_pointer;
391         }
392         if (targ->tx_opt_ring) {
393                 tbase->aux->tx_pkt_try = tx_try_self;
394         } else if (targ->nb_txrings == 1) {
395                 tbase->aux->tx_pkt_try = tx_try_sw1;
396         } else if (targ->nb_txports) {
397                 tbase->aux->tx_pkt_try = tx_try_hw1;
398         }
399
400         return tbase;
401 }
402
403 struct task_args *find_reachable_task_sending_to_port(struct task_args *from)
404 {
405         if (!from->nb_txrings) {
406                 if (from->tx_port_queue[0].port != OUT_DISCARD)
407                         return from;
408                 else
409                         return NULL;
410         }
411
412         struct core_task ct;
413         struct task_args *dtarg, *ret;
414
415         for (uint32_t i = 0; i < from->nb_txrings; ++i) {
416                 ct = from->core_task_set[0].core_task[i];
417                 dtarg = core_targ_get(ct.core, ct.task);
418                 ret = find_reachable_task_sending_to_port(dtarg);
419                 if (ret)
420                         return ret;
421         }
422         return NULL;
423 }
424
425 struct prox_port_cfg *find_reachable_port(struct task_args *from)
426 {
427         struct task_args *dst = find_reachable_task_sending_to_port(from);
428
429         if (dst) {
430                 int port_id = dst->tx_port_queue[0].port;
431
432                 return &prox_port_cfg[port_id];
433         }
434         return NULL;
435 }