nfvbench/traffic_gen/trex_gen.py

   1 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   2 #
   3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 #    not use this file except in compliance with the License. You may obtain
   5 #    a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #    Unless required by applicable law or agreed to in writing, software
  10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 #    License for the specific language governing permissions and limitations
  13 #    under the License.
  14 """Driver module for TRex traffic generator."""
  15
  16 import math
  17 import os
  18 import random
  19 import time
  20 import traceback
  21
  22 from itertools import count
  23 # pylint: disable=import-error
  24 from scapy.contrib.mpls import MPLS  # flake8: noqa
  25 # pylint: enable=import-error
  26 from nfvbench.log import LOG
  27 from nfvbench.traffic_server import TRexTrafficServer
  28 from nfvbench.utils import cast_integer
  29 from nfvbench.utils import timeout
  30 from nfvbench.utils import TimeoutError
  31
  32 # pylint: disable=import-error
  33 from trex.common.services.trex_service_arp import ServiceARP
  34 from trex.stl.api import bind_layers
  35 from trex.stl.api import CTRexVmInsFixHwCs
  36 from trex.stl.api import Dot1Q
  37 from trex.stl.api import Ether
  38 from trex.stl.api import FlagsField
  39 from trex.stl.api import IP
  40 from trex.stl.api import Packet
  41 from trex.stl.api import STLClient
  42 from trex.stl.api import STLError
  43 from trex.stl.api import STLFlowLatencyStats
  44 from trex.stl.api import STLFlowStats
  45 from trex.stl.api import STLPktBuilder
  46 from trex.stl.api import STLScVmRaw
  47 from trex.stl.api import STLStream
  48 from trex.stl.api import STLTXCont
  49 from trex.stl.api import STLVmFixChecksumHw
  50 from trex.stl.api import STLVmFixIpv4
  51 from trex.stl.api import STLVmFlowVar
  52 from trex.stl.api import STLVmFlowVarRepeatableRandom
  53 from trex.stl.api import STLVmWrFlowVar
  54 from trex.stl.api import ThreeBytesField
  55 from trex.stl.api import UDP
  56 from trex.stl.api import XByteField
  57
  58 # pylint: enable=import-error
  59
  60 from .traffic_base import AbstractTrafficGenerator
  61 from .traffic_base import TrafficGeneratorException
  62 from . import traffic_utils as utils
  63 from .traffic_utils import IMIX_AVG_L2_FRAME_SIZE
  64 from .traffic_utils import IMIX_L2_SIZES
  65 from .traffic_utils import IMIX_RATIOS
  66
  67 class VXLAN(Packet):
  68     """VxLAN class."""
  69
  70     _VXLAN_FLAGS = ['R' * 27] + ['I'] + ['R' * 5]
  71     name = "VXLAN"
  72     fields_desc = [FlagsField("flags", 0x08000000, 32, _VXLAN_FLAGS),
  73                    ThreeBytesField("vni", 0),
  74                    XByteField("reserved", 0x00)]
  75
  76     def mysummary(self):
  77         """Summary."""
  78         return self.sprintf("VXLAN (vni=%VXLAN.vni%)")
  79
  80 class TRex(AbstractTrafficGenerator):
  81     """TRex traffic generator driver."""
  82
  83     LATENCY_PPS = 1000
  84     CHAIN_PG_ID_MASK = 0x007F
  85     PORT_PG_ID_MASK = 0x0080
  86     LATENCY_PG_ID_MASK = 0x0100
  87
  88     def __init__(self, traffic_client):
  89         """Trex driver."""
  90         AbstractTrafficGenerator.__init__(self, traffic_client)
  91         self.client = None
  92         self.id = count()
  93         self.port_handle = []
  94         self.chain_count = self.generator_config.service_chain_count
  95         self.rates = []
  96         self.capture_id = None
  97         self.packet_list = []
  98         self.l2_frame_size = 0
  99
 100     def get_version(self):
 101         """Get the Trex version."""
 102         return self.client.get_server_version() if self.client else ''
 103
 104     def get_pg_id(self, port, chain_id):
 105         """Calculate the packet group IDs to use for a given port/stream type/chain_id.
 106
 107         port: 0 or 1
 108         chain_id: identifies to which chain the pg_id is associated (0 to 255)
 109         return: pg_id, lat_pg_id
 110
 111         We use a bit mask to set up the 3 fields:
 112         0x007F: chain ID (8 bits for a max of 128 chains)
 113         0x0080: port bit
 114         0x0100: latency bit
 115         """
 116         pg_id = port * TRex.PORT_PG_ID_MASK | chain_id
 117         return pg_id, pg_id | TRex.LATENCY_PG_ID_MASK
 118
 119     def extract_stats(self, in_stats):
 120         """Extract stats from dict returned by Trex API.
 121
 122         :param in_stats: dict as returned by TRex api
 123         """
 124         utils.nan_replace(in_stats)
 125         # LOG.debug(in_stats)
 126
 127         result = {}
 128         # port_handles should have only 2 elements: [0, 1]
 129         # so (1 - ph) will be the index for the far end port
 130         for ph in self.port_handle:
 131             stats = in_stats[ph]
 132             far_end_stats = in_stats[1 - ph]
 133             result[ph] = {
 134                 'tx': {
 135                     'total_pkts': cast_integer(stats['opackets']),
 136                     'total_pkt_bytes': cast_integer(stats['obytes']),
 137                     'pkt_rate': cast_integer(stats['tx_pps']),
 138                     'pkt_bit_rate': cast_integer(stats['tx_bps'])
 139                 },
 140                 'rx': {
 141                     'total_pkts': cast_integer(stats['ipackets']),
 142                     'total_pkt_bytes': cast_integer(stats['ibytes']),
 143                     'pkt_rate': cast_integer(stats['rx_pps']),
 144                     'pkt_bit_rate': cast_integer(stats['rx_bps']),
 145                     # how many pkts were dropped in RX direction
 146                     # need to take the tx counter on the far end port
 147                     'dropped_pkts': cast_integer(
 148                         far_end_stats['opackets'] - stats['ipackets'])
 149                 }
 150             }
 151             self.__combine_latencies(in_stats, result[ph]['rx'], ph)
 152
 153         total_tx_pkts = result[0]['tx']['total_pkts'] + result[1]['tx']['total_pkts']
 154         result["total_tx_rate"] = cast_integer(total_tx_pkts / self.config.duration_sec)
 155         # actual offered tx rate in bps
 156         avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
 157         total_tx_bps = utils.pps_to_bps(result["total_tx_rate"], avg_packet_size)
 158         result['offered_tx_rate_bps'] = total_tx_bps
 159         result["flow_stats"] = in_stats["flow_stats"]
 160         result["latency"] = in_stats["latency"]
 161         return result
 162
 163     def get_stream_stats(self, trex_stats, if_stats, latencies, chain_idx):
 164         """Extract the aggregated stats for a given chain.
 165
 166         trex_stats: stats as returned by get_stats()
 167         if_stats: a list of 2 interface stats to update (port 0 and 1)
 168         latencies: a list of 2 Latency instances to update for this chain (port 0 and 1)
 169                    latencies[p] is the latency for packets sent on port p
 170                    if there are no latency streams, the Latency instances are not modified
 171         chain_idx: chain index of the interface stats
 172
 173         The packet counts include normal and latency streams.
 174
 175         Trex returns flows stats as follows:
 176
 177         'flow_stats': {0: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 178                    'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 179                    'rx_bytes': {0: nan, 1: nan, 'total': nan},
 180                    'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
 181                    'rx_pps': {0: 0, 1: 0, 'total': 0},
 182                    'tx_bps': {0: 0, 1: 0, 'total': 0},
 183                    'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 184                    'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
 185                    'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
 186                    'tx_pps': {0: 0, 1: 0, 'total': 0}},
 187                1: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 188                    'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 189                    'rx_bytes': {0: nan, 1: nan, 'total': nan},
 190                    'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
 191                    'rx_pps': {0: 0, 1: 0, 'total': 0},
 192                    'tx_bps': {0: 0, 1: 0, 'total': 0},
 193                    'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 194                    'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
 195                    'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
 196                    'tx_pps': {0: 0, 1: 0, 'total': 0}},
 197                 128: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 198                 'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 199                 'rx_bytes': {0: nan, 1: nan, 'total': nan},
 200                 'rx_pkts': {0: 15001, 1: 0, 'total': 15001},
 201                 'rx_pps': {0: 0, 1: 0, 'total': 0},
 202                 'tx_bps': {0: 0, 1: 0, 'total': 0},
 203                 'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 204                 'tx_bytes': {0: 0, 1: 1020068, 'total': 1020068},
 205                 'tx_pkts': {0: 0, 1: 15001, 'total': 15001},
 206                 'tx_pps': {0: 0, 1: 0, 'total': 0}},etc...
 207
 208         the pg_id (0, 1, 128,...) is the key of the dict and is obtained using the
 209         get_pg_id() method.
 210         packet counters for a given stream sent on port p are reported as:
 211         - tx_pkts[p] on port p
 212         - rx_pkts[1-p] on the far end port
 213
 214         This is a tricky/critical counter transposition operation because
 215         the results are grouped by port (not by stream):
 216         tx_pkts_port(p=0) comes from pg_id(port=0, chain_idx)['tx_pkts'][0]
 217         rx_pkts_port(p=0) comes from pg_id(port=1, chain_idx)['rx_pkts'][0]
 218         tx_pkts_port(p=1) comes from pg_id(port=1, chain_idx)['tx_pkts'][1]
 219         rx_pkts_port(p=1) comes from pg_id(port=0, chain_idx)['rx_pkts'][1]
 220
 221         or using a more generic formula:
 222         tx_pkts_port(p) comes from pg_id(port=p, chain_idx)['tx_pkts'][p]
 223         rx_pkts_port(p) comes from pg_id(port=1-p, chain_idx)['rx_pkts'][p]
 224
 225         the second formula is equivalent to
 226         rx_pkts_port(1-p) comes from pg_id(port=p, chain_idx)['rx_pkts'][1-p]
 227
 228         If there are latency streams, those same counters need to be added in the same way
 229         """
 230         def get_latency(lval):
 231             try:
 232                 return int(round(lval))
 233             except ValueError:
 234                 return 0
 235
 236         for ifs in if_stats:
 237             ifs.tx = ifs.rx = 0
 238         for port in range(2):
 239             pg_id, lat_pg_id = self.get_pg_id(port, chain_idx)
 240             for pid in [pg_id, lat_pg_id]:
 241                 try:
 242                     pg_stats = trex_stats['flow_stats'][pid]
 243                     if_stats[port].tx += pg_stats['tx_pkts'][port]
 244                     if_stats[1 - port].rx += pg_stats['rx_pkts'][1 - port]
 245                 except KeyError:
 246                     pass
 247             try:
 248                 lat = trex_stats['latency'][lat_pg_id]['latency']
 249                 # dropped_pkts += lat['err_cntrs']['dropped']
 250                 latencies[port].max_usec = get_latency(lat['total_max'])
 251                 if math.isnan(lat['total_min']):
 252                     latencies[port].min_usec = 0
 253                     latencies[port].avg_usec = 0
 254                 else:
 255                     latencies[port].min_usec = get_latency(lat['total_min'])
 256                     latencies[port].avg_usec = get_latency(lat['average'])
 257                 # pick up the HDR histogram if present (otherwise will raise KeyError)
 258                 latencies[port].hdrh = lat['hdrh']
 259             except KeyError:
 260                 pass
 261
 262     def __combine_latencies(self, in_stats, results, port_handle):
 263         """Traverse TRex result dictionary and combines chosen latency stats.
 264
 265           example of latency dict returned by trex (2 chains):
 266          'latency': {256: {'err_cntrs': {'dropped': 0,
 267                                  'dup': 0,
 268                                  'out_of_order': 0,
 269                                  'seq_too_high': 0,
 270                                  'seq_too_low': 0},
 271                             'latency': {'average': 26.5,
 272                                         'hdrh': u'HISTFAAAAEx4nJNpmSgz...bFRgxi',
 273                                         'histogram': {20: 303,
 274                                                         30: 320,
 275                                                         40: 300,
 276                                                         50: 73,
 277                                                         60: 4,
 278                                                         70: 1},
 279                                         'jitter': 14,
 280                                         'last_max': 63,
 281                                         'total_max': 63,
 282                                         'total_min': 20}},
 283                     257: {'err_cntrs': {'dropped': 0,
 284                                         'dup': 0,
 285                                         'out_of_order': 0,
 286                                         'seq_too_high': 0,
 287                                         'seq_too_low': 0},
 288                             'latency': {'average': 29.75,
 289                                         'hdrh': u'HISTFAAAAEV4nJN...CALilDG0=',
 290                                         'histogram': {20: 261,
 291                                                         30: 431,
 292                                                         40: 3,
 293                                                         50: 80,
 294                                                         60: 225},
 295                                         'jitter': 23,
 296                                         'last_max': 67,
 297                                         'total_max': 67,
 298                                         'total_min': 20}},
 299                     384: {'err_cntrs': {'dropped': 0,
 300                                         'dup': 0,
 301                                         'out_of_order': 0,
 302                                         'seq_too_high': 0,
 303                                         'seq_too_low': 0},
 304                             'latency': {'average': 18.0,
 305                                         'hdrh': u'HISTFAAAADR4nJNpm...MjCwDDxAZG',
 306                                         'histogram': {20: 987, 30: 14},
 307                                         'jitter': 0,
 308                                         'last_max': 34,
 309                                         'total_max': 34,
 310                                         'total_min': 20}},
 311                     385: {'err_cntrs': {'dropped': 0,
 312                                     'dup': 0,
 313                                     'out_of_order': 0,
 314                                     'seq_too_high': 0,
 315                                     'seq_too_low': 0},
 316                             'latency': {'average': 19.0,
 317                                         'hdrh': u'HISTFAAAADR4nJNpm...NkYmJgDdagfK',
 318                                         'histogram': {20: 989, 30: 11},
 319                                         'jitter': 0,
 320                                         'last_max': 38,
 321                                         'total_max': 38,
 322                                         'total_min': 20}},
 323                     'global': {'bad_hdr': 0, 'old_flow': 0}},
 324         """
 325         total_max = 0
 326         average = 0
 327         total_min = float("inf")
 328         for chain_id in range(self.chain_count):
 329             try:
 330                 _, lat_pg_id = self.get_pg_id(port_handle, chain_id)
 331                 lat = in_stats['latency'][lat_pg_id]['latency']
 332                 # dropped_pkts += lat['err_cntrs']['dropped']
 333                 total_max = max(lat['total_max'], total_max)
 334                 total_min = min(lat['total_min'], total_min)
 335                 average += lat['average']
 336             except KeyError:
 337                 pass
 338         if total_min == float("inf"):
 339             total_min = 0
 340         results['min_delay_usec'] = total_min
 341         results['max_delay_usec'] = total_max
 342         results['avg_delay_usec'] = int(average / self.chain_count)
 343
 344     def _bind_vxlan(self):
 345         bind_layers(UDP, VXLAN, dport=4789)
 346         bind_layers(VXLAN, Ether)
 347
 348     def _create_pkt(self, stream_cfg, l2frame_size):
 349         """Create a packet of given size.
 350
 351         l2frame_size: size of the L2 frame in bytes (including the 32-bit FCS)
 352         """
 353         # Trex will add the FCS field, so we need to remove 4 bytes from the l2 frame size
 354         frame_size = int(l2frame_size) - 4
 355         vm_param = []
 356         if stream_cfg['vxlan'] is True:
 357             self._bind_vxlan()
 358             encap_level = '1'
 359             pkt_base = Ether(src=stream_cfg['vtep_src_mac'], dst=stream_cfg['vtep_dst_mac'])
 360             if stream_cfg['vtep_vlan'] is not None:
 361                 pkt_base /= Dot1Q(vlan=stream_cfg['vtep_vlan'])
 362             pkt_base /= IP(src=stream_cfg['vtep_src_ip'], dst=stream_cfg['vtep_dst_ip'])
 363             pkt_base /= UDP(sport=random.randint(1337, 32767), dport=4789)
 364             pkt_base /= VXLAN(vni=stream_cfg['net_vni'])
 365             pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 366             # need to randomize the outer header UDP src port based on flow
 367             vxlan_udp_src_fv = STLVmFlowVar(
 368                 name="vxlan_udp_src",
 369                 min_value=1337,
 370                 max_value=32767,
 371                 size=2,
 372                 op="random")
 373             vm_param = [vxlan_udp_src_fv,
 374                         STLVmWrFlowVar(fv_name="vxlan_udp_src", pkt_offset="UDP.sport")]
 375         elif stream_cfg['mpls'] is True:
 376             encap_level = '0'
 377             pkt_base = Ether(src=stream_cfg['vtep_src_mac'], dst=stream_cfg['vtep_dst_mac'])
 378             if stream_cfg['vtep_vlan'] is not None:
 379                 pkt_base /= Dot1Q(vlan=stream_cfg['vtep_vlan'])
 380             if stream_cfg['mpls_outer_label'] is not None:
 381                 pkt_base /= MPLS(label=stream_cfg['mpls_outer_label'], cos=1, s=0, ttl=255)
 382             if stream_cfg['mpls_inner_label'] is not None:
 383                 pkt_base /= MPLS(label=stream_cfg['mpls_inner_label'], cos=1, s=1, ttl=255)
 384             #  Flow stats and MPLS labels randomization TBD
 385             pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 386         else:
 387             encap_level = '0'
 388             pkt_base = Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 389
 390         if stream_cfg['vlan_tag'] is not None:
 391             pkt_base /= Dot1Q(vlan=stream_cfg['vlan_tag'])
 392
 393         udp_args = {}
 394         if stream_cfg['udp_src_port']:
 395             udp_args['sport'] = int(stream_cfg['udp_src_port'])
 396             if stream_cfg['udp_port_step'] == 'random':
 397                 step = 1
 398             else:
 399                 step = stream_cfg['udp_port_step']
 400             udp_args['sport_step'] = int(step)
 401             udp_args['sport_max'] = int(stream_cfg['udp_src_port_max'])
 402         if stream_cfg['udp_dst_port']:
 403             udp_args['dport'] = int(stream_cfg['udp_dst_port'])
 404             if stream_cfg['udp_port_step'] == 'random':
 405                 step = 1
 406             else:
 407                 step = stream_cfg['udp_port_step']
 408             udp_args['dport_step'] = int(step)
 409             udp_args['dport_max'] = int(stream_cfg['udp_dst_port_max'])
 410
 411         pkt_base /= IP(src=stream_cfg['ip_src_addr'], dst=stream_cfg['ip_dst_addr']) / \
 412                     UDP(dport=udp_args['dport'], sport=udp_args['sport'])
 413         if stream_cfg['ip_src_static'] is True:
 414             src_max_ip_value = stream_cfg['ip_src_addr']
 415         else:
 416             src_max_ip_value = stream_cfg['ip_src_addr_max']
 417         if stream_cfg['ip_addrs_step'] == 'random':
 418             src_fv_ip = STLVmFlowVarRepeatableRandom(
 419                 name="ip_src",
 420                 min_value=stream_cfg['ip_src_addr'],
 421                 max_value=src_max_ip_value,
 422                 size=4,
 423                 seed=random.randint(0, 32767),
 424                 limit=stream_cfg['ip_src_count'])
 425             dst_fv_ip = STLVmFlowVarRepeatableRandom(
 426                 name="ip_dst",
 427                 min_value=stream_cfg['ip_dst_addr'],
 428                 max_value=stream_cfg['ip_dst_addr_max'],
 429                 size=4,
 430                 seed=random.randint(0, 32767),
 431                 limit=stream_cfg['ip_dst_count'])
 432         else:
 433             src_fv_ip = STLVmFlowVar(
 434                 name="ip_src",
 435                 min_value=stream_cfg['ip_src_addr'],
 436                 max_value=src_max_ip_value,
 437                 size=4,
 438                 op="inc",
 439                 step=stream_cfg['ip_addrs_step'])
 440             dst_fv_ip = STLVmFlowVar(
 441                 name="ip_dst",
 442                 min_value=stream_cfg['ip_dst_addr'],
 443                 max_value=stream_cfg['ip_dst_addr_max'],
 444                 size=4,
 445                 op="inc",
 446                 step=stream_cfg['ip_addrs_step'])
 447
 448         if stream_cfg['udp_port_step'] == 'random':
 449             src_fv_port = STLVmFlowVarRepeatableRandom(
 450                 name="p_src",
 451                 min_value=udp_args['sport'],
 452                 max_value=udp_args['sport_max'],
 453                 size=2,
 454                 seed=random.randint(0, 32767),
 455                 limit=stream_cfg['udp_src_count'])
 456             dst_fv_port = STLVmFlowVarRepeatableRandom(
 457                 name="p_dst",
 458                 min_value=udp_args['dport'],
 459                 max_value=udp_args['dport_max'],
 460                 size=2,
 461                 seed=random.randint(0, 32767),
 462                 limit=stream_cfg['udp_dst_count'])
 463         else:
 464             src_fv_port = STLVmFlowVar(
 465                 name="p_src",
 466                 min_value=udp_args['sport'],
 467                 max_value=udp_args['sport_max'],
 468                 size=2,
 469                 op="inc",
 470                 step=udp_args['sport_step'])
 471             dst_fv_port = STLVmFlowVar(
 472                 name="p_dst",
 473                 min_value=udp_args['dport'],
 474                 max_value=udp_args['dport_max'],
 475                 size=2,
 476                 op="inc",
 477                 step=udp_args['dport_step'])
 478         vm_param = [
 479             src_fv_ip,
 480             STLVmWrFlowVar(fv_name="ip_src", pkt_offset="IP:{}.src".format(encap_level)),
 481             src_fv_port,
 482             STLVmWrFlowVar(fv_name="p_src", pkt_offset="UDP:{}.sport".format(encap_level)),
 483             dst_fv_ip,
 484             STLVmWrFlowVar(fv_name="ip_dst", pkt_offset="IP:{}.dst".format(encap_level)),
 485             dst_fv_port,
 486             STLVmWrFlowVar(fv_name="p_dst", pkt_offset="UDP:{}.dport".format(encap_level)),
 487         ]
 488         # Use HW Offload to calculate the outter IP/UDP packet
 489         vm_param.append(STLVmFixChecksumHw(l3_offset="IP:0",
 490                                            l4_offset="UDP:0",
 491                                            l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP))
 492         # Use software to fix the inner IP/UDP payload for VxLAN packets
 493         if int(encap_level):
 494             vm_param.append(STLVmFixIpv4(offset="IP:1"))
 495         pad = max(0, frame_size - len(pkt_base)) * 'x'
 496
 497         return STLPktBuilder(pkt=pkt_base / pad,
 498                              vm=STLScVmRaw(vm_param, cache_size=int(self.config.cache_size)))
 499
 500     def generate_streams(self, port, chain_id, stream_cfg, l2frame, latency=True,
 501                          e2e=False):
 502         """Create a list of streams corresponding to a given chain and stream config.
 503
 504         port: port where the streams originate (0 or 1)
 505         chain_id: the chain to which the streams are associated to
 506         stream_cfg: stream configuration
 507         l2frame: L2 frame size (including 4-byte FCS) or 'IMIX'
 508         latency: if True also create a latency stream
 509         e2e: True if performing "end to end" connectivity check
 510         """
 511         streams = []
 512         pg_id, lat_pg_id = self.get_pg_id(port, chain_id)
 513         if self.config.no_flow_stats:
 514             LOG.info("Traffic flow statistics are disabled.")
 515         if l2frame == 'IMIX':
 516             for ratio, l2_frame_size in zip(IMIX_RATIOS, IMIX_L2_SIZES):
 517                 pkt = self._create_pkt(stream_cfg, l2_frame_size)
 518                 if e2e or stream_cfg['mpls']:
 519                     streams.append(STLStream(packet=pkt,
 520                                              mode=STLTXCont(pps=ratio)))
 521                 else:
 522                     if stream_cfg['vxlan'] is True:
 523                         streams.append(STLStream(packet=pkt,
 524                                                  flow_stats=STLFlowStats(pg_id=pg_id,
 525                                                                          vxlan=True)
 526                                                  if not self.config.no_flow_stats else None,
 527                                                  mode=STLTXCont(pps=ratio)))
 528                     else:
 529                         streams.append(STLStream(packet=pkt,
 530                                                  flow_stats=STLFlowStats(pg_id=pg_id)
 531                                                  if not self.config.no_flow_stats else None,
 532                                                  mode=STLTXCont(pps=ratio)))
 533
 534             if latency:
 535                 # for IMIX, the latency packets have the average IMIX packet size
 536                 pkt = self._create_pkt(stream_cfg, IMIX_AVG_L2_FRAME_SIZE)
 537
 538         else:
 539             l2frame_size = int(l2frame)
 540             pkt = self._create_pkt(stream_cfg, l2frame_size)
 541             if e2e or stream_cfg['mpls']:
 542                 streams.append(STLStream(packet=pkt,
 543                                          # Flow stats is disabled for MPLS now
 544                                          # flow_stats=STLFlowStats(pg_id=pg_id),
 545                                          mode=STLTXCont()))
 546             else:
 547                 if stream_cfg['vxlan'] is True:
 548                     streams.append(STLStream(packet=pkt,
 549                                              flow_stats=STLFlowStats(pg_id=pg_id,
 550                                                                      vxlan=True)
 551                                              if not self.config.no_flow_stats else None,
 552                                              mode=STLTXCont()))
 553                 else:
 554                     streams.append(STLStream(packet=pkt,
 555                                              flow_stats=STLFlowStats(pg_id=pg_id)
 556                                              if not self.config.no_flow_stats else None,
 557                                              mode=STLTXCont()))
 558             # for the latency stream, the minimum payload is 16 bytes even in case of vlan tagging
 559             # without vlan, the min l2 frame size is 64
 560             # with vlan it is 68
 561             # This only applies to the latency stream
 562             if latency and stream_cfg['vlan_tag'] and l2frame_size < 68:
 563                 pkt = self._create_pkt(stream_cfg, 68)
 564
 565         if latency:
 566             if self.config.no_latency_stats:
 567                 LOG.info("Latency flow statistics are disabled.")
 568             if stream_cfg['vxlan'] is True:
 569                 streams.append(STLStream(packet=pkt,
 570                                          flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id,
 571                                                                         vxlan=True)
 572                                          if not self.config.no_latency_stats else None,
 573                                          mode=STLTXCont(pps=self.LATENCY_PPS)))
 574             else:
 575                 streams.append(STLStream(packet=pkt,
 576                                          flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id)
 577                                          if not self.config.no_latency_stats else None,
 578                                          mode=STLTXCont(pps=self.LATENCY_PPS)))
 579         return streams
 580
 581     @timeout(5)
 582     def __connect(self, client):
 583         client.connect()
 584
 585     def __connect_after_start(self):
 586         # after start, Trex may take a bit of time to initialize
 587         # so we need to retry a few times
 588         for it in range(self.config.generic_retry_count):
 589             try:
 590                 time.sleep(1)
 591                 self.client.connect()
 592                 break
 593             except Exception as ex:
 594                 if it == (self.config.generic_retry_count - 1):
 595                     raise
 596                 LOG.info("Retrying connection to TRex (%s)...", ex.msg)
 597
 598     def connect(self):
 599         """Connect to the TRex server."""
 600         server_ip = self.generator_config.ip
 601         LOG.info("Connecting to TRex (%s)...", server_ip)
 602
 603         # Connect to TRex server
 604         self.client = STLClient(server=server_ip, sync_port=self.generator_config.zmq_rpc_port,
 605                                 async_port=self.generator_config.zmq_pub_port)
 606         try:
 607             self.__connect(self.client)
 608             if server_ip == '127.0.0.1':
 609                 config_updated = self.__check_config()
 610                 if config_updated or self.config.restart:
 611                     self.__restart()
 612         except (TimeoutError, STLError) as e:
 613             if server_ip == '127.0.0.1':
 614                 self.__start_local_server()
 615             else:
 616                 raise TrafficGeneratorException(e.message)
 617
 618         ports = list(self.generator_config.ports)
 619         self.port_handle = ports
 620         # Prepare the ports
 621         self.client.reset(ports)
 622         # Read HW information from each port
 623         # this returns an array of dict (1 per port)
 624         """
 625         Example of output for Intel XL710
 626         [{'arp': '-', 'src_ipv4': '-', u'supp_speeds': [40000], u'is_link_supported': True,
 627           'grat_arp': 'off', 'speed': 40, u'index': 0, 'link_change_supported': 'yes',
 628           u'rx': {u'counters': 127, u'caps': [u'flow_stats', u'latency']},
 629           u'is_virtual': 'no', 'prom': 'off', 'src_mac': u'3c:fd:fe:a8:24:48', 'status': 'IDLE',
 630           u'description': u'Ethernet Controller XL710 for 40GbE QSFP+',
 631           'dest': u'fa:16:3e:3c:63:04', u'is_fc_supported': False, 'vlan': '-',
 632           u'driver': u'net_i40e', 'led_change_supported': 'yes', 'rx_filter_mode': 'hardware match',
 633           'fc': 'none', 'link': 'UP', u'hw_mac': u'3c:fd:fe:a8:24:48', u'pci_addr': u'0000:5e:00.0',
 634           'mult': 'off', 'fc_supported': 'no', u'is_led_supported': True, 'rx_queue': 'off',
 635           'layer_mode': 'Ethernet', u'numa': 0}, ...]
 636         """
 637         self.port_info = self.client.get_port_info(ports)
 638         LOG.info('Connected to TRex')
 639         for id, port in enumerate(self.port_info):
 640             LOG.info('   Port %d: %s speed=%dGbps mac=%s pci=%s driver=%s',
 641                      id, port['description'], port['speed'], port['src_mac'],
 642                      port['pci_addr'], port['driver'])
 643         # Make sure the 2 ports have the same speed
 644         if self.port_info[0]['speed'] != self.port_info[1]['speed']:
 645             raise TrafficGeneratorException('Traffic generator ports speed mismatch: %d/%d Gbps' %
 646                                             (self.port_info[0]['speed'],
 647                                              self.port_info[1]['speed']))
 648
 649     def __start_local_server(self):
 650         try:
 651             LOG.info("Starting TRex ...")
 652             self.__start_server()
 653             self.__connect_after_start()
 654         except (TimeoutError, STLError) as e:
 655             LOG.error('Cannot connect to TRex')
 656             LOG.error(traceback.format_exc())
 657             logpath = '/tmp/trex.log'
 658             if os.path.isfile(logpath):
 659                 # Wait for TRex to finish writing error message
 660                 last_size = 0
 661                 for _ in range(self.config.generic_retry_count):
 662                     size = os.path.getsize(logpath)
 663                     if size == last_size:
 664                         # probably not writing anymore
 665                         break
 666                     last_size = size
 667                     time.sleep(1)
 668                 with open(logpath, 'r') as f:
 669                     message = f.read()
 670             else:
 671                 message = e.message
 672             raise TrafficGeneratorException(message)
 673
 674     def __start_server(self):
 675         server = TRexTrafficServer()
 676         server.run_server(self.generator_config)
 677
 678     def __check_config(self):
 679         server = TRexTrafficServer()
 680         return server.check_config_updated(self.generator_config)
 681
 682     def __restart(self):
 683         LOG.info("Restarting TRex ...")
 684         self.__stop_server()
 685         # Wait for server stopped
 686         for _ in range(self.config.generic_retry_count):
 687             time.sleep(1)
 688             if not self.client.is_connected():
 689                 LOG.info("TRex is stopped...")
 690                 break
 691         self.__start_local_server()
 692
 693     def __stop_server(self):
 694         if self.generator_config.ip == '127.0.0.1':
 695             ports = self.client.get_acquired_ports()
 696             LOG.info('Release ports %s and stopping TRex...', ports)
 697             try:
 698                 if ports:
 699                     self.client.release(ports=ports)
 700                 self.client.server_shutdown()
 701             except STLError as e:
 702                 LOG.warning('Unable to stop TRex. Error: %s', e)
 703         else:
 704             LOG.info('Using remote TRex. Unable to stop TRex')
 705
 706     def resolve_arp(self):
 707         """Resolve all configured remote IP addresses.
 708
 709         return: None if ARP failed to resolve for all IP addresses
 710                 else a dict of list of dest macs indexed by port#
 711                 the dest macs in the list are indexed by the chain id
 712         """
 713         self.client.set_service_mode(ports=self.port_handle)
 714         LOG.info('Polling ARP until successful...')
 715         arp_dest_macs = {}
 716         for port, device in zip(self.port_handle, self.generator_config.devices):
 717             # there should be 1 stream config per chain
 718             stream_configs = device.get_stream_configs()
 719             chain_count = len(stream_configs)
 720             ctx = self.client.create_service_ctx(port=port)
 721             # all dest macs on this port indexed by chain ID
 722             dst_macs = [None] * chain_count
 723             dst_macs_count = 0
 724             # the index in the list is the chain id
 725             if self.config.vxlan or self.config.mpls:
 726                 arps = [
 727                     ServiceARP(ctx,
 728                                src_ip=device.vtep_src_ip,
 729                                dst_ip=device.vtep_dst_ip,
 730                                vlan=device.vtep_vlan)
 731                     for cfg in stream_configs
 732                 ]
 733             else:
 734                 arps = [
 735                     ServiceARP(ctx,
 736                                src_ip=cfg['ip_src_tg_gw'],
 737                                dst_ip=cfg['mac_discovery_gw'],
 738                                # will be None if no vlan tagging
 739                                vlan=cfg['vlan_tag'])
 740                     for cfg in stream_configs
 741                 ]
 742
 743             for attempt in range(self.config.generic_retry_count):
 744                 try:
 745                     ctx.run(arps)
 746                 except STLError:
 747                     LOG.error(traceback.format_exc())
 748                     continue
 749
 750                 unresolved = []
 751                 for chain_id, mac in enumerate(dst_macs):
 752                     if not mac:
 753                         arp_record = arps[chain_id].get_record()
 754                         if arp_record.dst_mac:
 755                             dst_macs[chain_id] = arp_record.dst_mac
 756                             dst_macs_count += 1
 757                             LOG.info('   ARP: port=%d chain=%d src IP=%s dst IP=%s -> MAC=%s',
 758                                      port, chain_id,
 759                                      arp_record.src_ip,
 760                                      arp_record.dst_ip, arp_record.dst_mac)
 761                         else:
 762                             unresolved.append(arp_record.dst_ip)
 763                 if dst_macs_count == chain_count:
 764                     arp_dest_macs[port] = dst_macs
 765                     LOG.info('ARP resolved successfully for port %s', port)
 766                     break
 767
 768                 retry = attempt + 1
 769                 LOG.info('Retrying ARP for: %s (retry %d/%d)',
 770                          unresolved, retry, self.config.generic_retry_count)
 771                 if retry < self.config.generic_retry_count:
 772                     time.sleep(self.config.generic_poll_sec)
 773             else:
 774                 LOG.error('ARP timed out for port %s (resolved %d out of %d)',
 775                           port,
 776                           dst_macs_count,
 777                           chain_count)
 778                 break
 779
 780         # if the capture from the TRex console was started before the arp request step,
 781         # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
 782         if not self.config.service_mode:
 783             self.client.set_service_mode(ports=self.port_handle, enabled=False)
 784         if len(arp_dest_macs) == len(self.port_handle):
 785             return arp_dest_macs
 786         return None
 787
 788     def __is_rate_enough(self, l2frame_size, rates, bidirectional, latency):
 789         """Check if rate provided by user is above requirements. Applies only if latency is True."""
 790         intf_speed = self.generator_config.intf_speed
 791         if latency:
 792             if bidirectional:
 793                 mult = 2
 794                 total_rate = 0
 795                 for rate in rates:
 796                     r = utils.convert_rates(l2frame_size, rate, intf_speed)
 797                     total_rate += int(r['rate_pps'])
 798             else:
 799                 mult = 1
 800                 total_rate = utils.convert_rates(l2frame_size, rates[0], intf_speed)
 801             # rate must be enough for latency stream and at least 1 pps for base stream per chain
 802             required_rate = (self.LATENCY_PPS + 1) * self.config.service_chain_count * mult
 803             result = utils.convert_rates(l2frame_size,
 804                                          {'rate_pps': required_rate},
 805                                          intf_speed * mult)
 806             result['result'] = total_rate >= required_rate
 807             return result
 808
 809         return {'result': True}
 810
 811     def create_traffic(self, l2frame_size, rates, bidirectional, latency=True, e2e=False):
 812         """Program all the streams in Trex server.
 813
 814         l2frame_size: L2 frame size or IMIX
 815         rates: a list of 2 rates to run each direction
 816                each rate is a dict like {'rate_pps': '10kpps'}
 817         bidirectional: True if bidirectional
 818         latency: True if latency measurement is needed
 819         e2e: True if performing "end to end" connectivity check
 820         """
 821         r = self.__is_rate_enough(l2frame_size, rates, bidirectional, latency)
 822         if not r['result']:
 823             raise TrafficGeneratorException(
 824                 'Required rate in total is at least one of: \n{pps}pps \n{bps}bps \n{load}%.'
 825                 .format(pps=r['rate_pps'],
 826                         bps=r['rate_bps'],
 827                         load=r['rate_percent']))
 828         self.l2_frame_size = l2frame_size
 829         # a dict of list of streams indexed by port#
 830         # in case of fixed size, has self.chain_count * 2 * 2 streams
 831         # (1 normal + 1 latency stream per direction per chain)
 832         # for IMIX, has self.chain_count * 2 * 4 streams
 833         # (3 normal + 1 latency stream per direction per chain)
 834         streamblock = {}
 835         for port in self.port_handle:
 836             streamblock[port] = []
 837         stream_cfgs = [d.get_stream_configs() for d in self.generator_config.devices]
 838         self.rates = [utils.to_rate_str(rate) for rate in rates]
 839         for chain_id, (fwd_stream_cfg, rev_stream_cfg) in enumerate(zip(*stream_cfgs)):
 840             streamblock[0].extend(self.generate_streams(self.port_handle[0],
 841                                                         chain_id,
 842                                                         fwd_stream_cfg,
 843                                                         l2frame_size,
 844                                                         latency=latency,
 845                                                         e2e=e2e))
 846             if len(self.rates) > 1:
 847                 streamblock[1].extend(self.generate_streams(self.port_handle[1],
 848                                                             chain_id,
 849                                                             rev_stream_cfg,
 850                                                             l2frame_size,
 851                                                             latency=bidirectional and latency,
 852                                                             e2e=e2e))
 853
 854         for port in self.port_handle:
 855             if self.config.vxlan:
 856                 self.client.set_port_attr(ports=port, vxlan_fs=[4789])
 857             else:
 858                 self.client.set_port_attr(ports=port, vxlan_fs=None)
 859             self.client.add_streams(streamblock[port], ports=port)
 860             LOG.info('Created %d traffic streams for port %s.', len(streamblock[port]), port)
 861
 862     def clear_streamblock(self):
 863         """Clear all streams from TRex."""
 864         self.rates = []
 865         self.client.reset(self.port_handle)
 866         LOG.info('Cleared all existing streams')
 867
 868     def get_stats(self):
 869         """Get stats from Trex."""
 870         stats = self.client.get_stats()
 871         return self.extract_stats(stats)
 872
 873     def get_macs(self):
 874         """Return the Trex local port MAC addresses.
 875
 876         return: a list of MAC addresses indexed by the port#
 877         """
 878         return [port['src_mac'] for port in self.port_info]
 879
 880     def get_port_speed_gbps(self):
 881         """Return the Trex local port MAC addresses.
 882
 883         return: a list of speed in Gbps indexed by the port#
 884         """
 885         return [port['speed'] for port in self.port_info]
 886
 887     def clear_stats(self):
 888         """Clear all stats in the traffic gneerator."""
 889         if self.port_handle:
 890             self.client.clear_stats()
 891
 892     def start_traffic(self):
 893         """Start generating traffic in all ports."""
 894         for port, rate in zip(self.port_handle, self.rates):
 895             self.client.start(ports=port, mult=rate, duration=self.config.duration_sec, force=True)
 896
 897     def stop_traffic(self):
 898         """Stop generating traffic."""
 899         self.client.stop(ports=self.port_handle)
 900
 901     def start_capture(self):
 902         """Capture all packets on both ports that are unicast to us."""
 903         if self.capture_id:
 904             self.stop_capture()
 905         # Need to filter out unwanted packets so we do not end up counting
 906         # src MACs of frames that are not unicast to us
 907         src_mac_list = self.get_macs()
 908         bpf_filter = "ether dst %s or ether dst %s" % (src_mac_list[0], src_mac_list[1])
 909         # ports must be set in service in order to enable capture
 910         self.client.set_service_mode(ports=self.port_handle)
 911         self.capture_id = self.client.start_capture \
 912             (rx_ports=self.port_handle, bpf_filter=bpf_filter)
 913
 914     def fetch_capture_packets(self):
 915         """Fetch capture packets in capture mode."""
 916         if self.capture_id:
 917             self.packet_list = []
 918             self.client.fetch_capture_packets(capture_id=self.capture_id['id'],
 919                                               output=self.packet_list)
 920
 921     def stop_capture(self):
 922         """Stop capturing packets."""
 923         if self.capture_id:
 924             self.client.stop_capture(capture_id=self.capture_id['id'])
 925             self.capture_id = None
 926             # if the capture from TRex console was started before the connectivity step,
 927             # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
 928             if not self.config.service_mode:
 929                 self.client.set_service_mode(ports=self.port_handle, enabled=False)
 930
 931     def cleanup(self):
 932         """Cleanup Trex driver."""
 933         if self.client:
 934             try:
 935                 self.client.reset(self.port_handle)
 936                 self.client.disconnect()
 937             except STLError:
 938                 # TRex does not like a reset while in disconnected state
 939                 pass
 940
 941     def set_service_mode(self, enabled=True):
 942         """Enable/disable the 'service_mode'."""
 943         self.client.set_service_mode(ports=self.port_handle, enabled=enabled)