VNFs/DPPD-PROX/task_init.c

   1 /*
   2 // Copyright (c) 2010-2017 Intel Corporation
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 */
  16
  17 #include <string.h>
  18 #include <stdio.h>
  19 #include <rte_version.h>
  20
  21 #include "prox_port_cfg.h"
  22 #include "prox_malloc.h"
  23 #include "task_init.h"
  24 #include "rx_pkt.h"
  25 #include "tx_pkt.h"
  26 #include "log.h"
  27 #include "quit.h"
  28 #include "lconf.h"
  29 #include "thread_generic.h"
  30 #include "prox_assert.h"
  31
  32 #if RTE_VERSION < RTE_VERSION_NUM(1,8,0,0)
  33 #define RTE_CACHE_LINE_SIZE CACHE_LINE_SIZE
  34 #endif
  35
  36 static unsigned first_task = 1;
  37 LIST_HEAD(,task_init) head;
  38
  39 void reg_task(struct task_init* t)
  40 {
  41         // PROX_PANIC(t->handle == NULL, "No handle function specified for task with name %d\n", t->mode);
  42
  43         if (t->thread_x == NULL)
  44                 t->thread_x = thread_generic;
  45
  46         if (first_task) {
  47                 first_task = 0;
  48                 LIST_INIT(&head);
  49         }
  50
  51         LIST_INSERT_HEAD(&head, t, entries);
  52 }
  53
  54 struct task_init *to_task_init(const char *mode_str, const char *sub_mode_str)
  55 {
  56         struct task_init *cur_t;
  57
  58         LIST_FOREACH(cur_t, &head, entries) {
  59                 if (!strcmp(mode_str, cur_t->mode_str) &&
  60                     !strcmp(sub_mode_str, cur_t->sub_mode_str)) {
  61                         return cur_t;
  62                 }
  63         }
  64
  65         return NULL;
  66 }
  67
  68 static int compare_strcmp(const void *a, const void *b)
  69 {
  70         return strcmp(*(const char * const *)a, *(const char * const *)b);
  71 }
  72
  73 int task_is_master(struct task_args *targ)
  74 {
  75         return (targ->lconf->id == prox_cfg.master);
  76 }
  77
  78 void tasks_list(void)
  79 {
  80         struct task_init *cur_t;
  81         char buf[sizeof(cur_t->mode_str) + sizeof(cur_t->sub_mode_str) + 4];
  82
  83         int nb_modes = 1; /* master */
  84         LIST_FOREACH(cur_t, &head, entries) {
  85                 ++nb_modes;
  86         }
  87
  88         char **modes = calloc(nb_modes, sizeof(*modes));
  89         char **cur_m = modes;
  90         *cur_m++ = strdup("master");
  91         LIST_FOREACH(cur_t, &head, entries) {
  92                 snprintf(buf, sizeof(buf), "%s%s%s",
  93                         cur_t->mode_str,
  94                         (cur_t->sub_mode_str[0] == 0) ? "" : " / ",
  95                         cur_t->sub_mode_str);
  96                 *cur_m++ = strdup(buf);
  97         }
  98         qsort(modes, nb_modes, sizeof(*modes), compare_strcmp);
  99
 100         plog_info("=== List of supported task modes / sub modes ===\n");
 101         for (cur_m = modes; nb_modes; ++cur_m, --nb_modes) {
 102                 plog_info("\t%s\n", *cur_m);
 103                 free(*cur_m);
 104         }
 105         free(modes);
 106 }
 107
 108 static size_t calc_memsize(struct task_args *targ, size_t task_size)
 109 {
 110         size_t memsize = task_size;
 111
 112         memsize += sizeof(struct task_base_aux);
 113
 114         if (targ->nb_rxports != 0) {
 115                 memsize += 2 * sizeof(uint8_t)*targ->nb_rxports;
 116         }
 117         if (targ->nb_rxrings != 0 || targ->tx_opt_ring_task) {
 118                 memsize += sizeof(struct rte_ring *)*targ->nb_rxrings;
 119         }
 120         if (targ->nb_txrings != 0) {
 121                 memsize += sizeof(struct rte_ring *) * targ->nb_txrings;
 122                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 123                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txrings;
 124         }
 125         else if (targ->nb_txports != 0) {
 126                 memsize += sizeof(struct port_queue) * targ->nb_txports;
 127                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 128                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * targ->nb_txports;
 129         }
 130         else {
 131                 memsize = RTE_ALIGN_CEIL(memsize, RTE_CACHE_LINE_SIZE);
 132                 memsize += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
 133         }
 134
 135         return memsize;
 136 }
 137
 138 static void *flush_function(struct task_args *targ)
 139 {
 140         if (targ->flags & TASK_ARG_DROP) {
 141                 return targ->nb_txrings ? flush_queues_sw : flush_queues_hw;
 142         }
 143         else {
 144                 return targ->nb_txrings ? flush_queues_no_drop_sw : flush_queues_no_drop_hw;
 145         }
 146 }
 147
 148 static size_t init_rx_tx_rings_ports(struct task_args *targ, struct task_base *tbase, size_t offset)
 149 {
 150         if (targ->tx_opt_ring_task) {
 151                 tbase->rx_pkt = rx_pkt_self;
 152         }
 153         else if (targ->nb_rxrings != 0) {
 154
 155                 if (targ->nb_rxrings == 1) {
 156                         tbase->rx_pkt = rx_pkt_sw1;
 157                         tbase->rx_params_sw1.rx_ring = targ->rx_rings[0];
 158                 }
 159                 else {
 160                         tbase->rx_pkt = rx_pkt_sw;
 161                         tbase->rx_params_sw.nb_rxrings = targ->nb_rxrings;
 162                         tbase->rx_params_sw.rx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 163                         offset += sizeof(struct rte_ring *)*tbase->rx_params_sw.nb_rxrings;
 164
 165                         for (uint8_t i = 0; i < tbase->rx_params_sw.nb_rxrings; ++i) {
 166                                 tbase->rx_params_sw.rx_rings[i] = targ->rx_rings[i];
 167                         }
 168
 169                         if (rte_is_power_of_2(targ->nb_rxrings)) {
 170                                 tbase->rx_pkt = rx_pkt_sw_pow2;
 171                                 tbase->rx_params_sw.rxrings_mask = targ->nb_rxrings - 1;
 172                         }
 173                 }
 174         }
 175         else {
 176                 if (targ->nb_rxports == 1) {
 177                         if (targ->flags & TASK_ARG_L3)
 178                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi_l3 : rx_pkt_hw1_l3;
 179                         else
 180                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw1_multi : rx_pkt_hw1;
 181                         tbase->rx_params_hw1.rx_pq.port =  targ->rx_port_queue[0].port;
 182                         tbase->rx_params_hw1.rx_pq.queue = targ->rx_port_queue[0].queue;
 183                 }
 184                 else {
 185                         PROX_ASSERT((targ->nb_rxports != 0) || (targ->task_init->flag_features & TASK_FEATURE_NO_RX));
 186                         if (targ->flags & TASK_ARG_L3)
 187                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi_l3 : rx_pkt_hw_l3;
 188                         else
 189                                 tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_multi : rx_pkt_hw;
 190                         tbase->rx_params_hw.nb_rxports = targ->nb_rxports;
 191                         tbase->rx_params_hw.rx_pq = (struct port_queue *)(((uint8_t *)tbase) + offset);
 192                         offset += sizeof(struct port_queue) * tbase->rx_params_hw.nb_rxports;
 193                         for (int i = 0; i< targ->nb_rxports; i++) {
 194                                 tbase->rx_params_hw.rx_pq[i].port = targ->rx_port_queue[i].port;
 195                                 tbase->rx_params_hw.rx_pq[i].queue = targ->rx_port_queue[i].queue;
 196                         }
 197
 198                         if (rte_is_power_of_2(targ->nb_rxports)) {
 199                                 if (targ->flags & TASK_ARG_L3)
 200                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi_l3 : rx_pkt_hw_pow2_l3;
 201                                 else
 202                                         tbase->rx_pkt = (targ->task_init->flag_features & TASK_FEATURE_MULTI_RX)? rx_pkt_hw_pow2_multi : rx_pkt_hw_pow2;
 203                                 tbase->rx_params_hw.rxport_mask = targ->nb_rxports - 1;
 204                         }
 205                 }
 206         }
 207
 208         if ((targ->nb_txrings != 0) && (!targ->tx_opt_ring) && (!(targ->flags & TASK_ARG_DROP))) {
 209                 // Transmitting to a ring in NO DROP. We need to make sure the receiving task in not running on the same core.
 210                 // Otherwise we might end up in a dead lock: trying in a loop to transmit to a task which cannot receive anymore
 211                 // (as not being scheduled).
 212                 struct core_task ct;
 213                 struct task_args *dtarg;
 214                 for (unsigned int j = 0; j < targ->nb_txrings; j++) {
 215                         ct = targ->core_task_set[0].core_task[j];
 216                         PROX_PANIC(ct.core == targ->lconf->id, "Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
 217                         //plog_info("Core %d, task %d: NO_DROP task transmitting to another task (core %d, task %d) running on on same core => potential deadlock\n", targ->lconf->id, targ->id, ct.core, ct.task);
 218                 }
 219         }
 220         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
 221                 /* Transmitting to multiple rings and one port */
 222                 plog_info("Initializing with 1 port %d queue %d nb_rings=%d\n", targ->tx_port_queue[0].port, targ->tx_port_queue[0].queue, targ->nb_txrings);
 223                 tbase->tx_params_hw_sw.tx_port_queue.port =  targ->tx_port_queue[0].port;
 224                 tbase->tx_params_hw_sw.tx_port_queue.queue =  targ->tx_port_queue[0].queue;
 225                 if (!targ->tx_opt_ring) {
 226                         tbase->tx_params_hw_sw.nb_txrings = targ->nb_txrings;
 227                         tbase->tx_params_hw_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 228                         offset += sizeof(struct rte_ring *)*tbase->tx_params_hw_sw.nb_txrings;
 229
 230                         for (uint8_t i = 0; i < tbase->tx_params_hw_sw.nb_txrings; ++i) {
 231                                 tbase->tx_params_hw_sw.tx_rings[i] = targ->tx_rings[i];
 232                         }
 233
 234                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 235                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 236                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw_sw.nb_txrings;
 237                 }
 238         }
 239         else if (!targ->tx_opt_ring) {
 240                 if (targ->nb_txrings != 0) {
 241                         tbase->tx_params_sw.nb_txrings = targ->nb_txrings;
 242                         tbase->tx_params_sw.tx_rings = (struct rte_ring **)(((uint8_t *)tbase) + offset);
 243                         offset += sizeof(struct rte_ring *)*tbase->tx_params_sw.nb_txrings;
 244
 245                         for (uint8_t i = 0; i < tbase->tx_params_sw.nb_txrings; ++i) {
 246                                 tbase->tx_params_sw.tx_rings[i] = targ->tx_rings[i];
 247                         }
 248
 249                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 250                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 251                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_sw.nb_txrings;
 252                 }
 253                 else if (targ->nb_txports != 0) {
 254                         tbase->tx_params_hw.nb_txports = targ->nb_txports;
 255                         tbase->tx_params_hw.tx_port_queue = (struct port_queue *)(((uint8_t *)tbase) + offset);
 256                         offset += sizeof(struct port_queue) * tbase->tx_params_hw.nb_txports;
 257                         for (uint8_t i = 0; i < tbase->tx_params_hw.nb_txports; ++i) {
 258                                 tbase->tx_params_hw.tx_port_queue[i].port = targ->tx_port_queue[i].port;
 259                                 tbase->tx_params_hw.tx_port_queue[i].queue = targ->tx_port_queue[i].queue;
 260                         }
 261
 262                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 263                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 264                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]) * tbase->tx_params_hw.nb_txports;
 265                 }
 266                 else {
 267                         offset = RTE_ALIGN_CEIL(offset, RTE_CACHE_LINE_SIZE);
 268                         tbase->ws_mbuf = (struct ws_mbuf *)(((uint8_t *)tbase) + offset);
 269                         offset += sizeof(struct ws_mbuf) + sizeof(((struct ws_mbuf*)0)->mbuf[0]);
 270                 }
 271
 272                 struct ws_mbuf* w = tbase->ws_mbuf;
 273                 struct task_args *prev = targ->tx_opt_ring_task;
 274
 275                 while (prev) {
 276                         prev->tbase->ws_mbuf = w;
 277                         prev = prev->tx_opt_ring_task;
 278                 }
 279         }
 280
 281         if (targ->nb_txrings == 1 || targ->nb_txports == 1 || targ->tx_opt_ring) {
 282                 if (targ->task_init->flag_features & TASK_FEATURE_NEVER_DISCARDS) {
 283                         if (targ->tx_opt_ring) {
 284                                 tbase->tx_pkt = tx_pkt_never_discard_self;
 285                         }
 286                         else if (targ->flags & TASK_ARG_DROP) {
 287                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
 288                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_thrpt_opt;
 289                                 else
 290                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_never_discard_sw1 : tx_pkt_never_discard_hw1_lat_opt;
 291                         }
 292                         else {
 293                                 if (targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT)
 294                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_thrpt_opt;
 295                                 else
 296                                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_never_discard_sw1 : tx_pkt_no_drop_never_discard_hw1_lat_opt;
 297                         }
 298                         if ((targ->nb_txrings) || ((targ->task_init->flag_features & TASK_FEATURE_THROUGHPUT_OPT) == 0))
 299                                 tbase->flags |= FLAG_NEVER_FLUSH;
 300                         else
 301                                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
 302                 }
 303                 else {
 304                         if (targ->tx_opt_ring) {
 305                                 tbase->tx_pkt = tx_pkt_self;
 306                         }
 307                         else if (targ->flags & TASK_ARG_DROP) {
 308                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw1 : tx_pkt_hw1;
 309                         }
 310                         else {
 311                                 tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw1 : tx_pkt_no_drop_hw1;
 312                         }
 313                         tbase->flags |= FLAG_NEVER_FLUSH;
 314                 }
 315         }
 316         else {
 317                 if (targ->flags & TASK_ARG_DROP) {
 318                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_sw : tx_pkt_hw;
 319                 }
 320                 else {
 321                         tbase->tx_pkt = targ->nb_txrings ? tx_pkt_no_drop_sw : tx_pkt_no_drop_hw;
 322                 }
 323
 324                 targ->lconf->flush_queues[targ->task] = flush_function(targ);
 325         }
 326
 327         if (targ->task_init->flag_features & TASK_FEATURE_NO_RX) {
 328                 tbase->rx_pkt = rx_pkt_dummy;
 329         }
 330
 331         if (targ->nb_txrings == 0 && targ->nb_txports == 0) {
 332                 tbase->tx_pkt = tx_pkt_drop_all;
 333         }
 334
 335         return offset;
 336 }
 337
 338 struct task_base *init_task_struct(struct task_args *targ)
 339 {
 340         struct task_init* t = targ->task_init;
 341         size_t offset = 0;
 342         size_t memsize = calc_memsize(targ, t->size);
 343         uint8_t task_socket = rte_lcore_to_socket_id(targ->lconf->id);
 344         struct task_base *tbase = prox_zmalloc(memsize, task_socket);
 345         PROX_PANIC(tbase == NULL, "Failed to allocate memory for task (%zu bytes)", memsize);
 346         offset += t->size;
 347
 348         if (targ->nb_txrings == 0 && targ->nb_txports == 0)
 349                 tbase->flags |= FLAG_NEVER_FLUSH;
 350
 351         offset = init_rx_tx_rings_ports(targ, tbase, offset);
 352         tbase->aux = (struct task_base_aux *)(((uint8_t *)tbase) + offset);
 353
 354         if (targ->task_init->flag_features & TASK_FEATURE_TSC_RX) {
 355                 task_base_add_rx_pkt_function(tbase, rx_pkt_tsc);
 356         }
 357
 358         offset += sizeof(struct task_base_aux);
 359
 360         tbase->handle_bulk = t->handle;
 361
 362         if (targ->flags & TASK_ARG_L3) {
 363                 plog_info("\tTask configured in L3 mode\n");
 364                 tbase->l3.ctrl_plane_ring = targ->ctrl_plane_ring;
 365                 if (targ->nb_txports != 0) {
 366                         tbase->aux->tx_pkt_l2 = tbase->tx_pkt;
 367                         tbase->tx_pkt = tx_pkt_l3;
 368                         // Make sure control plane packets such as arp are not dropped
 369                         tbase->aux->tx_ctrlplane_pkt = targ->nb_txrings ? tx_ctrlplane_sw : tx_ctrlplane_hw;
 370                         task_init_l3(tbase, targ);
 371                 }
 372         }
 373
 374         targ->tbase = tbase;
 375         if (t->init) {
 376                 t->init(tbase, targ);
 377         }
 378         tbase->aux->start = t->start;
 379         tbase->aux->stop = t->stop;
 380         tbase->aux->start_first = t->start_first;
 381         tbase->aux->stop_last = t->stop_last;
 382         if ((targ->nb_txrings != 0) && (targ->nb_txports == 1)) {
 383                 tbase->aux->tx_pkt_hw = tx_pkt_no_drop_never_discard_hw1_no_pointer;
 384         }
 385         if (targ->tx_opt_ring) {
 386                 tbase->aux->tx_pkt_try = tx_try_self;
 387         } else if (targ->nb_txrings == 1) {
 388                 tbase->aux->tx_pkt_try = tx_try_sw1;
 389         } else if (targ->nb_txports) {
 390                 tbase->aux->tx_pkt_try = tx_try_hw1;
 391         }
 392
 393         return tbase;
 394 }
 395
 396 struct task_args *find_reachable_task_sending_to_port(struct task_args *from)
 397 {
 398         if (!from->nb_txrings) {
 399                 if (from->tx_port_queue[0].port != OUT_DISCARD)
 400                         return from;
 401                 else
 402                         return NULL;
 403         }
 404
 405         struct core_task ct;
 406         struct task_args *dtarg, *ret;
 407
 408         for (uint32_t i = 0; i < from->nb_txrings; ++i) {
 409                 ct = from->core_task_set[0].core_task[i];
 410                 dtarg = core_targ_get(ct.core, ct.task);
 411                 ret = find_reachable_task_sending_to_port(dtarg);
 412                 if (ret)
 413                         return ret;
 414         }
 415         return NULL;
 416 }
 417
 418 struct prox_port_cfg *find_reachable_port(struct task_args *from)
 419 {
 420         struct task_args *dst = find_reachable_task_sending_to_port(from);
 421
 422         if (dst) {
 423                 int port_id = dst->tx_port_queue[0].port;
 424
 425                 return &prox_port_cfg[port_id];
 426         }
 427         return NULL;
 428 }