VNFs/DPPD-PROX/task_init.c

   1 /*
   2 // Copyright (c) 2010-2017 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 #include <string.h>
  18 #include <stdio.h>
  19 #include <rte_version.h>
  20
  21 #include "prox_port_cfg.h"
  22 #include "prox_malloc.h"
  23 #include "task_init.h"
  24 #include "rx_pkt.h"
  25 #include "tx_pkt.h"
  26 #include "log.h"
  27 #include "quit.h"
  28 #include "lconf.h"
  29 #include "thread_generic.h"
  30 #include "prox_assert.h"
  31
  32 #if RTE_VERSION < RTE_VERSION_NUM(1,8,0,0)
  33 #define RTE_CACHE_LINE_SIZE CACHE_LINE_SIZE
  34 #endif
  35
  36 static unsigned first_task = 1;
  37 LIST_HEAD(,task_init) head;
  38
  39 void reg_task(struct task_init* t)
  40 {
  41         // PROX_PANIC(t->handle == NULL, "No handle function specified for task with name %d\n", t->mode);
  42
  43         if (t->thread_x == NULL)
  44                 t->thread_x = thread_generic;
  45
  46         if (first_task) {
  47                 first_task = 0;
  48                 LIST_INIT(&head);
  49         }
  50
  51         LIST_INSERT_HEAD(&head, t, entries);
  52 }
  53
  54 struct task_init *to_task_init(const char *mode_str, const char *sub_mode_str)
  55 {
  56         struct task_init *cur_t;
  57
  58         LIST_FOREACH(cur_t, &head, entries) {
  59                 if (!strcmp(mode_str, cur_t->mode_str) &&
  60                     !strcmp(sub_mode_str, cur_t->sub_mode_str)) {
  61                         return cur_t;
  62                 }
  63         }
  64
  65         return NULL;
  66 }
  67
  68 static int compare_strcmp(const void *a, const void *b)
  69 {
  70         return strcmp(*(const char * const *)a, *(const char * const *)b);
  71 }
  72
  73 int task_is_master(struct task_args *targ)
  74 {
  75         return (targ->lconf->id == prox_cfg.master);
  76 }
  77
  78 void tasks_list(void)
  79 {
  80         struct task_init *cur_t;
  81         char buf[sizeof(cur_t->mode_str) + sizeof(cur_t->sub_mode_str) + 4];
  82
  83         int nb_modes = 1; /* master */
  84         LIST_FOREACH(cur_t, &head, entries) {
  85                 ++nb_modes;
  86         }
  87
  88         char **modes = calloc(nb_modes, sizeof(*modes));
  89         char **cur_m = modes;
  90         *cur_m++ = strdup("master");
  91         LIST_FOREACH(cur_t, &head, entries) {
  92                 snprintf(buf, sizeof(buf), "%s%s%s",
  93                         cur_t->mode_str,
  94                         (cur_t->sub_mode_str[0] == 0) ? "" : " / ",
  95                         cur_t->sub_mode_str);
  96                 *cur_m++ = strdup(buf);
  97         }
  98         qsort(modes, nb_modes, sizeof(*modes), compare_strcmp);
  99
 100         plog_info("=== List of supported task modes / sub modes ===\n");
 101         for (cur_m = modes; nb_modes; ++cur_m, --nb_modes) {
 102                 plog_info("\t%s\n", *cur_m);
 103                 free(*cur_m);
 104         }
 105         free(modes);
 106 }
 107
 108 static size_t calc_memsize(struct task_args *targ, size_t task_size)
 109 {
 110         size_t memsize = task_size;
 111
 112         memsize += sizeof(struct task_base_aux);
 113
 114         if (targ->nb_rxports != 0) {
 115                 memsize += 2 * sizeof(uint8_t)*targ->nb_rxports;
 116         }
 117         if (targ->nb_rxrings != 0 || targ->tx_opt_ring_task) {
 118                 memsize += sizeof(struct rte_ring *)*targ->nb_rxrings;
 119         }
 120         if (targ->nb_txrings != 0) {
 121                 memsize += sizeof(struct rte_ring *) * targ->nb_txrings;
 122                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 123                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txrings;
 124         }
 125         else if (targ->nb_txports != 0) {
 126                 memsize += sizeof(struct port_queue) * targ->nb_txports;
 127                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 128                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txports;
 129         }
 130         else {
 131                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 132                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
 133         }
 134
 135         return memsize;
 136 }
 137
 138 static void *flush_function(struct task_args *targ)
 139 {
 140         if (targ->flags & TASK_ARG_DROP) {
 141                 return targ->nb_txrings ? flush_queues_sw : flush_queues_hw;
 142         }
 143         else {
 144                 return targ->nb_txrings ? flush_queues_no_drop_sw : flush_queues_no_drop_hw;
 145         }
 146 }
 147
 148 static size_t init_rx_tx_rings_ports(struct task_args *targ, struct task_base *tbase, size_t offset)
 149 {
 150         if (targ->tx_opt_ring_task) {
 151                 tbase->rx_pkt = rx_pkt_self;
 152         }
 153         else if (targ->nb_rxrings != 0) {
 154
 155                 if (targ->nb_rxrings == 1) {
 156                         tbase->rx_pkt = rx_pkt_sw1;
 157                         tbase->rx_params_sw1.rx_ring = targ->rx_rings[0];
 158                 }
 159                 else {
 160                         tbase->rx_pkt = rx_pkt_sw;
 161                         tbase->rx_params_sw.nb_rxrings = targ->nb_rxrings;
 162                         tbase->rx_params_sw.rx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 163                         offset += sizeof(struct rte_ring *)*tbase->rx_params_sw.nb_rxrings;
 164
 165                         for (uint8_t i = 0; i < tbase->rx_params_sw.nb_rxrings; ++i) {
 166                                 tbase->rx_params_sw.rx_rings[i] = targ->rx_rings[i];
 167                         }
 168
 169                         if (rte_is_power_of_2(targ->nb_rxrings)) {
 170                                 tbase->rx_pkt = rx_pkt_sw_pow2;
 171                                 tbase->rx_params_sw.rxrings_mask = targ->nb_rxrings - 1;
 172                         }
 173                 }
 174         }
 175         else {
 176                 if (targ->nb_rxports == 1) {
 177                         if (targ->flags & TASK_ARG_L3)
 178                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi_l3 : rx_pkt_hw1_l3;
 179                         else
 180                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi : rx_pkt_hw1;
 181                         tbase->rx_params_hw1.rx_pq.port =  targ->rx_port_queue[0].port;
 182                         tbase->rx_params_hw1.rx_pq.queue = targ->rx_port_queue[0].queue;
 183                 }
 184                 else {
 185                         PROX_ASSERT((targ->nb_rxports != 0) || (targ->task_init->flag_features & TASK_FEATURE_NO_RX));
 186                         if (targ->flags & TASK_ARG_L3)
 187                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi_l3 : rx_pkt_hw_l3;
 188                         else
 189                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi : rx_pkt_hw;
 190                         tbase->rx_params_hw.nb_rxports = targ->nb_rxports;
 191                         tbase->rx_params_hw.rx_pq = (struct port_queue *)(((uint8_t *)tbase) + offset);
 192                         offset += sizeof(struct port_queue) * tbase->rx_params_hw.nb_rxports;
 193                         for (int i = 0; i< targ->nb_rxports; i++) {
 194                                 tbase->rx_params_hw.rx_pq[i].port = targ->rx_port_queue[i].port;
 195                                 tbase->rx_params_hw.rx_pq[i].queue = targ->rx_port_queue[i].queue;
 196                         }
 197
 198                         if (rte_is_power_of_2(targ->nb_rxports)) {
 199                                 if (targ->flags & TASK_ARG_L3)
 200                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi_l3 : rx_pkt_hw_pow2_l3;
 201                                 else
 202                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi : rx_pkt_hw_pow2;
 203                                 tbase->rx_params_hw.rxport_mask = targ->nb_rxports - 1;
 204                         }
 205                 }
 206         }
 207
 208         if ((targ->nb_txrings != 0) && (!targ->tx_opt_ring) && (!(targ->flags & TASK_ARG_DROP))) {
 209                 // Transmitting to a ring in NO DROP. We need to make sure the receiving task in not running on the same core.
 210                 // Otherwise we might end up in a dead lock: trying in a loop to transmit to a task which cannot receive anymore
 211                 // (as npt being scheduled).
 212                 struct core_task ct;
 213                 struct task_args *dtarg;
 214                 for (unsigned int j = 0; j < targ->nb_txrings; j++) {
 215                         ct = targ->core_task_set[0].core_task[j];
 216                         PROX_PANIC(ct.core == targ->lconf->id, "Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
 217                         //plog_info("Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
 218                 }
 219         }
 220         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
 221                 /* Transmitting to multiple rings and one port */
 222                 plog_info("Initializing with 1 port %d queue %d nb_rings=%d\n", targ->tx_port_queue[0].port, targ->tx_port_queue[0].queue, targ->nb_txrings);
 223                 tbase->tx_params_hw_sw.tx_port_queue.port =  targ->tx_port_queue[0].port;
 224                 tbase->tx_params_hw_sw.tx_port_queue.queue =  targ->tx_port_queue[0].queue;
 225                 if (!targ->tx_opt_ring) {
 226                         tbase->tx_params_hw_sw.nb_txrings = targ->nb_txrings;
 227                         tbase->tx_params_hw_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 228                         offset += sizeof(struct rte_ring *)*tbase->tx_params_hw_sw.nb_txrings;
 229
 230                         for (uint8_t i = 0; i < tbase->tx_params_hw_sw.nb_txrings; ++i) {
 231                                 tbase->tx_params_hw_sw.tx_rings[i] = targ->tx_rings[i];
 232                         }
 233
 234                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 235                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 236                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw_sw.nb_txrings;
 237                 }
 238         }
 239         else if (!targ->tx_opt_ring) {
 240                 if (targ->nb_txrings != 0) {
 241                         tbase->tx_params_sw.nb_txrings = targ->nb_txrings;
 242                         tbase->tx_params_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 243                         offset += sizeof(struct rte_ring *)*tbase->tx_params_sw.nb_txrings;
 244
 245                         for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
 246                                 tbase->tx_params_sw.tx_rings[i] = targ->tx_rings[i];
 247                         }
 248
 249                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 250                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 251                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_sw.nb_txrings;
 252                 }
 253                 else if (targ->nb_txports != 0) {
 254                         tbase->tx_params_hw.nb_txports = targ->nb_txports;
 255                         tbase->tx_params_hw.tx_port_queue = (struct port_queue *)(((uint8_t *)tbase) + offset);
 256                         offset += sizeof(struct port_queue) * tbase->tx_params_hw.nb_txports;
 257                         for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
 258                                 tbase->tx_params_hw.tx_port_queue[i].port = targ->tx_port_queue[i].port;
 259                                 tbase->tx_params_hw.tx_port_queue[i].queue = targ->tx_port_queue[i].queue;
 260                         }
 261
 262                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 263                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 264                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw.nb_txports;
 265                 }
 266                 else {
 267                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 268                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 269                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
 270                 }
 271
 272                 struct ws_mbuf* w = tbase->ws_mbuf;
 273                 struct task_args *prev = targ->tx_opt_ring_task;
 274
 275                 while (prev) {
 276                         prev->tbase->ws_mbuf = w;
 277                         prev = prev->tx_opt_ring_task;
 278                 }
 279         }
 280         if (targ->nb_txrings == 1 || targ->nb_txports == 1 || targ->tx_opt_ring) {
 281                 if (targ->task_init->flag_features & TASK_FEATURE_NEVER_DISCARDS) {
 282                         if (targ->tx_opt_ring) {
 283                                 tbase->tx_pkt = tx_pkt_never_discard_self;
 284                         }
 285                         else if (targ->flags & TASK_ARG_DROP) {
 286                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
 287                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_thrpt_opt;
 288                                 else
 289                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_lat_opt;
 290                         }
 291                         else {
 292                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
 293                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_thrpt_opt;
 294                                 else
 295                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_lat_opt;
 296                         }
 297                         if ((targ->nb_txrings) || ((targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT) == 0))
 298                                 tbase->flags |= FLAG_NEVER_FLUSH;
 299                         else
 300                                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
 301                 }
 302                 else {
 303                         if (targ->tx_opt_ring) {
 304                                 tbase->tx_pkt = tx_pkt_self;
 305                         }
 306                         else if (targ->flags & TASK_ARG_DROP) {
 307                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw1 : tx_pkt_hw1;
 308                         }
 309                         else {
 310                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw1 : tx_pkt_no_drop_hw1;
 311                         }
 312                         tbase->flags |= FLAG_NEVER_FLUSH;
 313                 }
 314         }
 315         else {
 316                 if (targ->flags & TASK_ARG_DROP) {
 317                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw : tx_pkt_hw;
 318                 }
 319                 else {
 320                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw : tx_pkt_no_drop_hw;
 321                 }
 322
 323                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
 324         }
 325
 326         if (targ->task_init->flag_features & TASK_FEATURE_NO_RX) {
 327                 tbase->rx_pkt = rx_pkt_dummy;
 328         }
 329
 330         if (targ->nb_txrings == 0 && targ->nb_txports == 0) {
 331                 tbase->tx_pkt = tx_pkt_drop_all;
 332         }
 333
 334         return offset;
 335 }
 336
 337 struct task_base *init_task_struct(struct task_args *targ)
 338 {
 339         struct task_init* t = targ->task_init;
 340         size_t offset = 0;
 341         size_t memsize = calc_memsize(targ, t->size);
 342         uint8_t task_socket = rte_lcore_to_socket_id(targ->lconf->id);
 343         struct task_base *tbase = prox_zmalloc(memsize, task_socket);
 344         PROX_PANIC(tbase == NULL, "Failed to allocate memory for task (%zu bytes)", memsize);
 345         offset += t->size;
 346
 347         if (targ->nb_txrings == 0 && targ->nb_txports == 0)
 348                 tbase->flags |= FLAG_NEVER_FLUSH;
 349
 350         offset = init_rx_tx_rings_ports(targ, tbase, offset);
 351         tbase->aux = (struct task_base_aux *)(((uint8_t *)tbase) + offset);
 352
 353         if (targ->nb_txports != 0) {
 354                 if (targ->flags & TASK_ARG_L3) {
 355                         tbase->aux->tx_pkt_l2 = tbase->tx_pkt;
 356                         tbase->tx_pkt = tx_pkt_l3;
 357                 }
 358         }
 359
 360         if (targ->task_init->flag_features & TASK_FEATURE_RX_ALL) {
 361                 task_base_add_rx_pkt_function(tbase, rx_pkt_all);
 362                 tbase->aux->all_mbufs = prox_zmalloc(MAX_RX_PKT_ALL * sizeof(* tbase->aux->all_mbufs), task_socket);
 363         }
 364         if (targ->task_init->flag_features & TASK_FEATURE_TSC_RX) {
 365                 task_base_add_rx_pkt_function(tbase, rx_pkt_tsc);
 366         }
 367
 368         offset += sizeof(struct task_base_aux);
 369
 370         tbase->handle_bulk = t->handle;
 371
 372         if (targ->flags & TASK_ARG_L3) {
 373                 plog_info("\tTask configured in L3 mode\n");
 374                 tbase->l3.ctrl_plane_ring = targ->ctrl_plane_ring;
 375         }
 376         if (targ->nb_txports != 0) {
 377                 if (targ->flags & TASK_ARG_L3)
 378                         task_init_l3(tbase, targ);
 379         }
 380
 381         targ->tbase = tbase;
 382         if (t->init) {
 383                 t->init(tbase, targ);
 384         }
 385         tbase->aux->start = t->start;
 386         tbase->aux->stop = t->stop;
 387         tbase->aux->start_first = t->start_first;
 388         tbase->aux->stop_last = t->stop_last;
 389         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
 390                 tbase->aux->tx_pkt_hw = tx_pkt_no_drop_never_discard_hw1_no_pointer;
 391         }
 392         if (targ->tx_opt_ring) {
 393                 tbase->aux->tx_pkt_try = tx_try_self;
 394         } else if (targ->nb_txrings == 1) {
 395                 tbase->aux->tx_pkt_try = tx_try_sw1;
 396         } else if (targ->nb_txports) {
 397                 tbase->aux->tx_pkt_try = tx_try_hw1;
 398         }
 399
 400         return tbase;
 401 }
 402
 403 struct task_args *find_reachable_task_sending_to_port(struct task_args *from)
 404 {
 405         if (!from->nb_txrings) {
 406                 if (from->tx_port_queue[0].port != OUT_DISCARD)
 407                         return from;
 408                 else
 409                         return NULL;
 410         }
 411
 412         struct core_task ct;
 413         struct task_args *dtarg, *ret;
 414
 415         for (uint32_t i = 0; i < from->nb_txrings; ++i) {
 416                 ct = from->core_task_set[0].core_task[i];
 417                 dtarg = core_targ_get(ct.core, ct.task);
 418                 ret = find_reachable_task_sending_to_port(dtarg);
 419                 if (ret)
 420                         return ret;
 421         }
 422         return NULL;
 423 }
 424
 425 struct prox_port_cfg *find_reachable_port(struct task_args *from)
 426 {
 427         struct task_args *dst = find_reachable_task_sending_to_port(from);
 428
 429         if (dst) {
 430                 int port_id = dst->tx_port_queue[0].port;
 431
 432                 return &prox_port_cfg[port_id];
 433         }
 434         return NULL;
 435 }