nfvbench/traffic_gen/trex_gen.py

   1 # Copyright 2016 Cisco Systems, Inc.  All rights reserved.
   2 #
   3 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
   4 #    not use this file except in compliance with the License. You may obtain
   5 #    a copy of the License at
   6 #
   7 #         http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 #    Unless required by applicable law or agreed to in writing, software
  10 #    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  11 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  12 #    License for the specific language governing permissions and limitations
  13 #    under the License.
  14 """Driver module for TRex traffic generator."""
  15
  16 import math
  17 import os
  18 import random
  19 import time
  20 import traceback
  21
  22 from itertools import count
  23 # pylint: disable=import-error
  24 from scapy.contrib.mpls import MPLS  # flake8: noqa
  25 # pylint: enable=import-error
  26 from nfvbench.log import LOG
  27 from nfvbench.traffic_server import TRexTrafficServer
  28 from nfvbench.utils import cast_integer
  29 from nfvbench.utils import timeout
  30 from nfvbench.utils import TimeoutError
  31
  32 # pylint: disable=import-error
  33 from trex.common.services.trex_service_arp import ServiceARP
  34 from trex.stl.api import bind_layers
  35 from trex.stl.api import CTRexVmInsFixHwCs
  36 from trex.stl.api import Dot1Q
  37 from trex.stl.api import Ether
  38 from trex.stl.api import FlagsField
  39 from trex.stl.api import IP
  40 from trex.stl.api import Packet
  41 from trex.stl.api import STLClient
  42 from trex.stl.api import STLError
  43 from trex.stl.api import STLFlowLatencyStats
  44 from trex.stl.api import STLFlowStats
  45 from trex.stl.api import STLPktBuilder
  46 from trex.stl.api import STLScVmRaw
  47 from trex.stl.api import STLStream
  48 from trex.stl.api import STLTXCont
  49 from trex.stl.api import STLVmFixChecksumHw
  50 from trex.stl.api import STLVmFixIpv4
  51 from trex.stl.api import STLVmFlowVar
  52 from trex.stl.api import STLVmFlowVarRepeatableRandom
  53 from trex.stl.api import STLVmWrFlowVar
  54 from trex.stl.api import ThreeBytesField
  55 from trex.stl.api import UDP
  56 from trex.stl.api import XByteField
  57
  58 # pylint: enable=import-error
  59
  60 from .traffic_base import AbstractTrafficGenerator
  61 from .traffic_base import TrafficGeneratorException
  62 from . import traffic_utils as utils
  63 from .traffic_utils import IMIX_AVG_L2_FRAME_SIZE
  64 from .traffic_utils import IMIX_L2_SIZES
  65 from .traffic_utils import IMIX_RATIOS
  66
  67 class VXLAN(Packet):
  68     """VxLAN class."""
  69
  70     _VXLAN_FLAGS = ['R' * 27] + ['I'] + ['R' * 5]
  71     name = "VXLAN"
  72     fields_desc = [FlagsField("flags", 0x08000000, 32, _VXLAN_FLAGS),
  73                    ThreeBytesField("vni", 0),
  74                    XByteField("reserved", 0x00)]
  75
  76     def mysummary(self):
  77         """Summary."""
  78         return self.sprintf("VXLAN (vni=%VXLAN.vni%)")
  79
  80 class TRex(AbstractTrafficGenerator):
  81     """TRex traffic generator driver."""
  82
  83     LATENCY_PPS = 1000
  84     CHAIN_PG_ID_MASK = 0x007F
  85     PORT_PG_ID_MASK = 0x0080
  86     LATENCY_PG_ID_MASK = 0x0100
  87
  88     def __init__(self, traffic_client):
  89         """Trex driver."""
  90         AbstractTrafficGenerator.__init__(self, traffic_client)
  91         self.client = None
  92         self.id = count()
  93         self.port_handle = []
  94         self.chain_count = self.generator_config.service_chain_count
  95         self.rates = []
  96         self.capture_id = None
  97         self.packet_list = []
  98         self.l2_frame_size = 0
  99
 100     def get_version(self):
 101         """Get the Trex version."""
 102         return self.client.get_server_version() if self.client else ''
 103
 104     def get_pg_id(self, port, chain_id):
 105         """Calculate the packet group IDs to use for a given port/stream type/chain_id.
 106
 107         port: 0 or 1
 108         chain_id: identifies to which chain the pg_id is associated (0 to 255)
 109         return: pg_id, lat_pg_id
 110
 111         We use a bit mask to set up the 3 fields:
 112         0x007F: chain ID (8 bits for a max of 128 chains)
 113         0x0080: port bit
 114         0x0100: latency bit
 115         """
 116         pg_id = port * TRex.PORT_PG_ID_MASK | chain_id
 117         return pg_id, pg_id | TRex.LATENCY_PG_ID_MASK
 118
 119     def extract_stats(self, in_stats):
 120         """Extract stats from dict returned by Trex API.
 121
 122         :param in_stats: dict as returned by TRex api
 123         """
 124         utils.nan_replace(in_stats)
 125         # LOG.debug(in_stats)
 126
 127         result = {}
 128         # port_handles should have only 2 elements: [0, 1]
 129         # so (1 - ph) will be the index for the far end port
 130         for ph in self.port_handle:
 131             stats = in_stats[ph]
 132             far_end_stats = in_stats[1 - ph]
 133             result[ph] = {
 134                 'tx': {
 135                     'total_pkts': cast_integer(stats['opackets']),
 136                     'total_pkt_bytes': cast_integer(stats['obytes']),
 137                     'pkt_rate': cast_integer(stats['tx_pps']),
 138                     'pkt_bit_rate': cast_integer(stats['tx_bps'])
 139                 },
 140                 'rx': {
 141                     'total_pkts': cast_integer(stats['ipackets']),
 142                     'total_pkt_bytes': cast_integer(stats['ibytes']),
 143                     'pkt_rate': cast_integer(stats['rx_pps']),
 144                     'pkt_bit_rate': cast_integer(stats['rx_bps']),
 145                     # how many pkts were dropped in RX direction
 146                     # need to take the tx counter on the far end port
 147                     'dropped_pkts': cast_integer(
 148                         far_end_stats['opackets'] - stats['ipackets'])
 149                 }
 150             }
 151             self.__combine_latencies(in_stats, result[ph]['rx'], ph)
 152
 153         total_tx_pkts = result[0]['tx']['total_pkts'] + result[1]['tx']['total_pkts']
 154         result["total_tx_rate"] = cast_integer(total_tx_pkts / self.config.duration_sec)
 155         # actual offered tx rate in bps
 156         avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
 157         total_tx_bps = utils.pps_to_bps(result["total_tx_rate"], avg_packet_size)
 158         result['offered_tx_rate_bps'] = total_tx_bps
 159         result["flow_stats"] = in_stats["flow_stats"]
 160         result["latency"] = in_stats["latency"]
 161         return result
 162
 163     def get_stream_stats(self, trex_stats, if_stats, latencies, chain_idx):
 164         """Extract the aggregated stats for a given chain.
 165
 166         trex_stats: stats as returned by get_stats()
 167         if_stats: a list of 2 interface stats to update (port 0 and 1)
 168         latencies: a list of 2 Latency instances to update for this chain (port 0 and 1)
 169                    latencies[p] is the latency for packets sent on port p
 170                    if there are no latency streams, the Latency instances are not modified
 171         chain_idx: chain index of the interface stats
 172
 173         The packet counts include normal and latency streams.
 174
 175         Trex returns flows stats as follows:
 176
 177         'flow_stats': {0: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 178                    'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 179                    'rx_bytes': {0: nan, 1: nan, 'total': nan},
 180                    'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
 181                    'rx_pps': {0: 0, 1: 0, 'total': 0},
 182                    'tx_bps': {0: 0, 1: 0, 'total': 0},
 183                    'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 184                    'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
 185                    'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
 186                    'tx_pps': {0: 0, 1: 0, 'total': 0}},
 187                1: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 188                    'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 189                    'rx_bytes': {0: nan, 1: nan, 'total': nan},
 190                    'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
 191                    'rx_pps': {0: 0, 1: 0, 'total': 0},
 192                    'tx_bps': {0: 0, 1: 0, 'total': 0},
 193                    'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 194                    'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
 195                    'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
 196                    'tx_pps': {0: 0, 1: 0, 'total': 0}},
 197                 128: {'rx_bps': {0: 0, 1: 0, 'total': 0},
 198                 'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 199                 'rx_bytes': {0: nan, 1: nan, 'total': nan},
 200                 'rx_pkts': {0: 15001, 1: 0, 'total': 15001},
 201                 'rx_pps': {0: 0, 1: 0, 'total': 0},
 202                 'tx_bps': {0: 0, 1: 0, 'total': 0},
 203                 'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
 204                 'tx_bytes': {0: 0, 1: 1020068, 'total': 1020068},
 205                 'tx_pkts': {0: 0, 1: 15001, 'total': 15001},
 206                 'tx_pps': {0: 0, 1: 0, 'total': 0}},etc...
 207
 208         the pg_id (0, 1, 128,...) is the key of the dict and is obtained using the
 209         get_pg_id() method.
 210         packet counters for a given stream sent on port p are reported as:
 211         - tx_pkts[p] on port p
 212         - rx_pkts[1-p] on the far end port
 213
 214         This is a tricky/critical counter transposition operation because
 215         the results are grouped by port (not by stream):
 216         tx_pkts_port(p=0) comes from pg_id(port=0, chain_idx)['tx_pkts'][0]
 217         rx_pkts_port(p=0) comes from pg_id(port=1, chain_idx)['rx_pkts'][0]
 218         tx_pkts_port(p=1) comes from pg_id(port=1, chain_idx)['tx_pkts'][1]
 219         rx_pkts_port(p=1) comes from pg_id(port=0, chain_idx)['rx_pkts'][1]
 220
 221         or using a more generic formula:
 222         tx_pkts_port(p) comes from pg_id(port=p, chain_idx)['tx_pkts'][p]
 223         rx_pkts_port(p) comes from pg_id(port=1-p, chain_idx)['rx_pkts'][p]
 224
 225         the second formula is equivalent to
 226         rx_pkts_port(1-p) comes from pg_id(port=p, chain_idx)['rx_pkts'][1-p]
 227
 228         If there are latency streams, those same counters need to be added in the same way
 229         """
 230         def get_latency(lval):
 231             try:
 232                 return int(round(lval))
 233             except ValueError:
 234                 return 0
 235
 236         for ifs in if_stats:
 237             ifs.tx = ifs.rx = 0
 238         for port in range(2):
 239             pg_id, lat_pg_id = self.get_pg_id(port, chain_idx)
 240             for pid in [pg_id, lat_pg_id]:
 241                 try:
 242                     pg_stats = trex_stats['flow_stats'][pid]
 243                     if_stats[port].tx += pg_stats['tx_pkts'][port]
 244                     if_stats[1 - port].rx += pg_stats['rx_pkts'][1 - port]
 245                 except KeyError:
 246                     pass
 247             try:
 248                 lat = trex_stats['latency'][lat_pg_id]['latency']
 249                 # dropped_pkts += lat['err_cntrs']['dropped']
 250                 latencies[port].max_usec = get_latency(lat['total_max'])
 251                 if math.isnan(lat['total_min']):
 252                     latencies[port].min_usec = 0
 253                     latencies[port].avg_usec = 0
 254                 else:
 255                     latencies[port].min_usec = get_latency(lat['total_min'])
 256                     latencies[port].avg_usec = get_latency(lat['average'])
 257                 # pick up the HDR histogram if present (otherwise will raise KeyError)
 258                 latencies[port].hdrh = lat['hdrh']
 259             except KeyError:
 260                 pass
 261
 262     def __combine_latencies(self, in_stats, results, port_handle):
 263         """Traverse TRex result dictionary and combines chosen latency stats.
 264
 265           example of latency dict returned by trex (2 chains):
 266          'latency': {256: {'err_cntrs': {'dropped': 0,
 267                                  'dup': 0,
 268                                  'out_of_order': 0,
 269                                  'seq_too_high': 0,
 270                                  'seq_too_low': 0},
 271                             'latency': {'average': 26.5,
 272                                         'hdrh': u'HISTFAAAAEx4nJNpmSgz...bFRgxi',
 273                                         'histogram': {20: 303,
 274                                                         30: 320,
 275                                                         40: 300,
 276                                                         50: 73,
 277                                                         60: 4,
 278                                                         70: 1},
 279                                         'jitter': 14,
 280                                         'last_max': 63,
 281                                         'total_max': 63,
 282                                         'total_min': 20}},
 283                     257: {'err_cntrs': {'dropped': 0,
 284                                         'dup': 0,
 285                                         'out_of_order': 0,
 286                                         'seq_too_high': 0,
 287                                         'seq_too_low': 0},
 288                             'latency': {'average': 29.75,
 289                                         'hdrh': u'HISTFAAAAEV4nJN...CALilDG0=',
 290                                         'histogram': {20: 261,
 291                                                         30: 431,
 292                                                         40: 3,
 293                                                         50: 80,
 294                                                         60: 225},
 295                                         'jitter': 23,
 296                                         'last_max': 67,
 297                                         'total_max': 67,
 298                                         'total_min': 20}},
 299                     384: {'err_cntrs': {'dropped': 0,
 300                                         'dup': 0,
 301                                         'out_of_order': 0,
 302                                         'seq_too_high': 0,
 303                                         'seq_too_low': 0},
 304                             'latency': {'average': 18.0,
 305                                         'hdrh': u'HISTFAAAADR4nJNpm...MjCwDDxAZG',
 306                                         'histogram': {20: 987, 30: 14},
 307                                         'jitter': 0,
 308                                         'last_max': 34,
 309                                         'total_max': 34,
 310                                         'total_min': 20}},
 311                     385: {'err_cntrs': {'dropped': 0,
 312                                     'dup': 0,
 313                                     'out_of_order': 0,
 314                                     'seq_too_high': 0,
 315                                     'seq_too_low': 0},
 316                             'latency': {'average': 19.0,
 317                                         'hdrh': u'HISTFAAAADR4nJNpm...NkYmJgDdagfK',
 318                                         'histogram': {20: 989, 30: 11},
 319                                         'jitter': 0,
 320                                         'last_max': 38,
 321                                         'total_max': 38,
 322                                         'total_min': 20}},
 323                     'global': {'bad_hdr': 0, 'old_flow': 0}},
 324         """
 325         total_max = 0
 326         average = 0
 327         total_min = float("inf")
 328         for chain_id in range(self.chain_count):
 329             try:
 330                 _, lat_pg_id = self.get_pg_id(port_handle, chain_id)
 331                 lat = in_stats['latency'][lat_pg_id]['latency']
 332                 # dropped_pkts += lat['err_cntrs']['dropped']
 333                 total_max = max(lat['total_max'], total_max)
 334                 total_min = min(lat['total_min'], total_min)
 335                 average += lat['average']
 336             except KeyError:
 337                 pass
 338         if total_min == float("inf"):
 339             total_min = 0
 340         results['min_delay_usec'] = total_min
 341         results['max_delay_usec'] = total_max
 342         results['avg_delay_usec'] = int(average / self.chain_count)
 343
 344     def _bind_vxlan(self):
 345         bind_layers(UDP, VXLAN, dport=4789)
 346         bind_layers(VXLAN, Ether)
 347
 348     def _create_pkt(self, stream_cfg, l2frame_size):
 349         """Create a packet of given size.
 350
 351         l2frame_size: size of the L2 frame in bytes (including the 32-bit FCS)
 352         """
 353         # Trex will add the FCS field, so we need to remove 4 bytes from the l2 frame size
 354         frame_size = int(l2frame_size) - 4
 355         vm_param = []
 356         if stream_cfg['vxlan'] is True:
 357             self._bind_vxlan()
 358             encap_level = '1'
 359             pkt_base = Ether(src=stream_cfg['vtep_src_mac'], dst=stream_cfg['vtep_dst_mac'])
 360             if stream_cfg['vtep_vlan'] is not None:
 361                 pkt_base /= Dot1Q(vlan=stream_cfg['vtep_vlan'])
 362             pkt_base /= IP(src=stream_cfg['vtep_src_ip'], dst=stream_cfg['vtep_dst_ip'])
 363             pkt_base /= UDP(sport=random.randint(1337, 32767), dport=4789)
 364             pkt_base /= VXLAN(vni=stream_cfg['net_vni'])
 365             pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 366             # need to randomize the outer header UDP src port based on flow
 367             vxlan_udp_src_fv = STLVmFlowVar(
 368                 name="vxlan_udp_src",
 369                 min_value=1337,
 370                 max_value=32767,
 371                 size=2,
 372                 op="random")
 373             vm_param = [vxlan_udp_src_fv,
 374                         STLVmWrFlowVar(fv_name="vxlan_udp_src", pkt_offset="UDP.sport")]
 375         elif stream_cfg['mpls'] is True:
 376             encap_level = '0'
 377             pkt_base = Ether(src=stream_cfg['vtep_src_mac'], dst=stream_cfg['vtep_dst_mac'])
 378             if stream_cfg['vtep_vlan'] is not None:
 379                 pkt_base /= Dot1Q(vlan=stream_cfg['vtep_vlan'])
 380             if stream_cfg['mpls_outer_label'] is not None:
 381                 pkt_base /= MPLS(label=stream_cfg['mpls_outer_label'], cos=1, s=0, ttl=255)
 382             if stream_cfg['mpls_inner_label'] is not None:
 383                 pkt_base /= MPLS(label=stream_cfg['mpls_inner_label'], cos=1, s=1, ttl=255)
 384             #  Flow stats and MPLS labels randomization TBD
 385             pkt_base /= Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 386         else:
 387             encap_level = '0'
 388             pkt_base = Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
 389
 390         if stream_cfg['vlan_tag'] is not None:
 391             pkt_base /= Dot1Q(vlan=stream_cfg['vlan_tag'])
 392
 393         udp_args = {}
 394         if stream_cfg['udp_src_port']:
 395             udp_args['sport'] = int(stream_cfg['udp_src_port'])
 396             udp_args['sport_step'] = int(stream_cfg['udp_port_step'])
 397             udp_args['sport_max'] = int(stream_cfg['udp_src_port_max'])
 398         if stream_cfg['udp_dst_port']:
 399             udp_args['dport'] = int(stream_cfg['udp_dst_port'])
 400             udp_args['dport_step'] = int(stream_cfg['udp_port_step'])
 401             udp_args['dport_max'] = int(stream_cfg['udp_dst_port_max'])
 402
 403         pkt_base /= IP(src=stream_cfg['ip_src_addr'], dst=stream_cfg['ip_dst_addr']) / \
 404                     UDP(dport=udp_args['dport'], sport=udp_args['sport'])
 405         if stream_cfg['ip_src_static'] is True:
 406             src_max_ip_value = stream_cfg['ip_src_addr']
 407         else:
 408             src_max_ip_value = stream_cfg['ip_src_addr_max']
 409         if stream_cfg['ip_addrs_step'] == 'random':
 410             src_fv_ip = STLVmFlowVarRepeatableRandom(
 411                 name="ip_src",
 412                 min_value=stream_cfg['ip_src_addr'],
 413                 max_value=src_max_ip_value,
 414                 size=4,
 415                 seed=random.randint(0, 32767),
 416                 limit=stream_cfg['ip_src_count'])
 417             dst_fv_ip = STLVmFlowVarRepeatableRandom(
 418                 name="ip_dst",
 419                 min_value=stream_cfg['ip_dst_addr'],
 420                 max_value=stream_cfg['ip_dst_addr_max'],
 421                 size=4,
 422                 seed=random.randint(0, 32767),
 423                 limit=stream_cfg['ip_dst_count'])
 424         else:
 425             src_fv_ip = STLVmFlowVar(
 426                 name="ip_src",
 427                 min_value=stream_cfg['ip_src_addr'],
 428                 max_value=src_max_ip_value,
 429                 size=4,
 430                 op="inc",
 431                 step=stream_cfg['ip_addrs_step'])
 432             dst_fv_ip = STLVmFlowVar(
 433                 name="ip_dst",
 434                 min_value=stream_cfg['ip_dst_addr'],
 435                 max_value=stream_cfg['ip_dst_addr_max'],
 436                 size=4,
 437                 op="inc",
 438                 step=stream_cfg['ip_addrs_step'])
 439
 440         if stream_cfg['udp_port_step'] == 'random':
 441             src_fv_port = STLVmFlowVarRepeatableRandom(
 442                 name="p_src",
 443                 min_value=udp_args['sport'],
 444                 max_value=udp_args['sport_max'],
 445                 size=2,
 446                 seed=random.randint(0, 32767),
 447                 limit=udp_args['udp_src_count'])
 448             dst_fv_port = STLVmFlowVarRepeatableRandom(
 449                 name="p_dst",
 450                 min_value=udp_args['dport'],
 451                 max_value=udp_args['dport_max'],
 452                 size=2,
 453                 seed=random.randint(0, 32767),
 454                 limit=stream_cfg['udp_dst_count'])
 455         else:
 456             src_fv_port = STLVmFlowVar(
 457                 name="p_src",
 458                 min_value=udp_args['sport'],
 459                 max_value=udp_args['sport_max'],
 460                 size=2,
 461                 op="inc",
 462                 step=udp_args['sport_step'])
 463             dst_fv_port = STLVmFlowVar(
 464                 name="p_dst",
 465                 min_value=udp_args['dport'],
 466                 max_value=udp_args['dport_max'],
 467                 size=2,
 468                 op="inc",
 469                 step=udp_args['dport_step'])
 470         vm_param = [
 471             src_fv_ip,
 472             STLVmWrFlowVar(fv_name="ip_src", pkt_offset="IP:{}.src".format(encap_level)),
 473             src_fv_port,
 474             STLVmWrFlowVar(fv_name="p_src", pkt_offset="UDP:{}.sport".format(encap_level)),
 475             dst_fv_ip,
 476             STLVmWrFlowVar(fv_name="ip_dst", pkt_offset="IP:{}.dst".format(encap_level)),
 477             dst_fv_port,
 478             STLVmWrFlowVar(fv_name="p_dst", pkt_offset="UDP:{}.dport".format(encap_level)),
 479         ]
 480         # Use HW Offload to calculate the outter IP/UDP packet
 481         vm_param.append(STLVmFixChecksumHw(l3_offset="IP:0",
 482                                            l4_offset="UDP:0",
 483                                            l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP))
 484         # Use software to fix the inner IP/UDP payload for VxLAN packets
 485         if int(encap_level):
 486             vm_param.append(STLVmFixIpv4(offset="IP:1"))
 487         pad = max(0, frame_size - len(pkt_base)) * 'x'
 488
 489         return STLPktBuilder(pkt=pkt_base / pad,
 490                              vm=STLScVmRaw(vm_param, cache_size=int(self.config.cache_size)))
 491
 492     def generate_streams(self, port, chain_id, stream_cfg, l2frame, latency=True,
 493                          e2e=False):
 494         """Create a list of streams corresponding to a given chain and stream config.
 495
 496         port: port where the streams originate (0 or 1)
 497         chain_id: the chain to which the streams are associated to
 498         stream_cfg: stream configuration
 499         l2frame: L2 frame size (including 4-byte FCS) or 'IMIX'
 500         latency: if True also create a latency stream
 501         e2e: True if performing "end to end" connectivity check
 502         """
 503         streams = []
 504         pg_id, lat_pg_id = self.get_pg_id(port, chain_id)
 505         if self.config.no_flow_stats:
 506             LOG.info("Traffic flow statistics are disabled.")
 507         if l2frame == 'IMIX':
 508             for ratio, l2_frame_size in zip(IMIX_RATIOS, IMIX_L2_SIZES):
 509                 pkt = self._create_pkt(stream_cfg, l2_frame_size)
 510                 if e2e or stream_cfg['mpls']:
 511                     streams.append(STLStream(packet=pkt,
 512                                              mode=STLTXCont(pps=ratio)))
 513                 else:
 514                     if stream_cfg['vxlan'] is True:
 515                         streams.append(STLStream(packet=pkt,
 516                                                  flow_stats=STLFlowStats(pg_id=pg_id,
 517                                                                          vxlan=True)
 518                                                  if not self.config.no_flow_stats else None,
 519                                                  mode=STLTXCont(pps=ratio)))
 520                     else:
 521                         streams.append(STLStream(packet=pkt,
 522                                                  flow_stats=STLFlowStats(pg_id=pg_id)
 523                                                  if not self.config.no_flow_stats else None,
 524                                                  mode=STLTXCont(pps=ratio)))
 525
 526             if latency:
 527                 # for IMIX, the latency packets have the average IMIX packet size
 528                 pkt = self._create_pkt(stream_cfg, IMIX_AVG_L2_FRAME_SIZE)
 529
 530         else:
 531             l2frame_size = int(l2frame)
 532             pkt = self._create_pkt(stream_cfg, l2frame_size)
 533             if e2e or stream_cfg['mpls']:
 534                 streams.append(STLStream(packet=pkt,
 535                                          # Flow stats is disabled for MPLS now
 536                                          # flow_stats=STLFlowStats(pg_id=pg_id),
 537                                          mode=STLTXCont()))
 538             else:
 539                 if stream_cfg['vxlan'] is True:
 540                     streams.append(STLStream(packet=pkt,
 541                                              flow_stats=STLFlowStats(pg_id=pg_id,
 542                                                                      vxlan=True)
 543                                              if not self.config.no_flow_stats else None,
 544                                              mode=STLTXCont()))
 545                 else:
 546                     streams.append(STLStream(packet=pkt,
 547                                              flow_stats=STLFlowStats(pg_id=pg_id)
 548                                              if not self.config.no_flow_stats else None,
 549                                              mode=STLTXCont()))
 550             # for the latency stream, the minimum payload is 16 bytes even in case of vlan tagging
 551             # without vlan, the min l2 frame size is 64
 552             # with vlan it is 68
 553             # This only applies to the latency stream
 554             if latency and stream_cfg['vlan_tag'] and l2frame_size < 68:
 555                 pkt = self._create_pkt(stream_cfg, 68)
 556
 557         if latency:
 558             if self.config.no_latency_stats:
 559                 LOG.info("Latency flow statistics are disabled.")
 560             if stream_cfg['vxlan'] is True:
 561                 streams.append(STLStream(packet=pkt,
 562                                          flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id,
 563                                                                         vxlan=True)
 564                                          if not self.config.no_latency_stats else None,
 565                                          mode=STLTXCont(pps=self.LATENCY_PPS)))
 566             else:
 567                 streams.append(STLStream(packet=pkt,
 568                                          flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id)
 569                                          if not self.config.no_latency_stats else None,
 570                                          mode=STLTXCont(pps=self.LATENCY_PPS)))
 571         return streams
 572
 573     @timeout(5)
 574     def __connect(self, client):
 575         client.connect()
 576
 577     def __connect_after_start(self):
 578         # after start, Trex may take a bit of time to initialize
 579         # so we need to retry a few times
 580         for it in range(self.config.generic_retry_count):
 581             try:
 582                 time.sleep(1)
 583                 self.client.connect()
 584                 break
 585             except Exception as ex:
 586                 if it == (self.config.generic_retry_count - 1):
 587                     raise
 588                 LOG.info("Retrying connection to TRex (%s)...", ex.msg)
 589
 590     def connect(self):
 591         """Connect to the TRex server."""
 592         server_ip = self.generator_config.ip
 593         LOG.info("Connecting to TRex (%s)...", server_ip)
 594
 595         # Connect to TRex server
 596         self.client = STLClient(server=server_ip, sync_port=self.generator_config.zmq_rpc_port,
 597                                 async_port=self.generator_config.zmq_pub_port)
 598         try:
 599             self.__connect(self.client)
 600             if server_ip == '127.0.0.1':
 601                 config_updated = self.__check_config()
 602                 if config_updated or self.config.restart:
 603                     self.__restart()
 604         except (TimeoutError, STLError) as e:
 605             if server_ip == '127.0.0.1':
 606                 self.__start_local_server()
 607             else:
 608                 raise TrafficGeneratorException(e.message)
 609
 610         ports = list(self.generator_config.ports)
 611         self.port_handle = ports
 612         # Prepare the ports
 613         self.client.reset(ports)
 614         # Read HW information from each port
 615         # this returns an array of dict (1 per port)
 616         """
 617         Example of output for Intel XL710
 618         [{'arp': '-', 'src_ipv4': '-', u'supp_speeds': [40000], u'is_link_supported': True,
 619           'grat_arp': 'off', 'speed': 40, u'index': 0, 'link_change_supported': 'yes',
 620           u'rx': {u'counters': 127, u'caps': [u'flow_stats', u'latency']},
 621           u'is_virtual': 'no', 'prom': 'off', 'src_mac': u'3c:fd:fe:a8:24:48', 'status': 'IDLE',
 622           u'description': u'Ethernet Controller XL710 for 40GbE QSFP+',
 623           'dest': u'fa:16:3e:3c:63:04', u'is_fc_supported': False, 'vlan': '-',
 624           u'driver': u'net_i40e', 'led_change_supported': 'yes', 'rx_filter_mode': 'hardware match',
 625           'fc': 'none', 'link': 'UP', u'hw_mac': u'3c:fd:fe:a8:24:48', u'pci_addr': u'0000:5e:00.0',
 626           'mult': 'off', 'fc_supported': 'no', u'is_led_supported': True, 'rx_queue': 'off',
 627           'layer_mode': 'Ethernet', u'numa': 0}, ...]
 628         """
 629         self.port_info = self.client.get_port_info(ports)
 630         LOG.info('Connected to TRex')
 631         for id, port in enumerate(self.port_info):
 632             LOG.info('   Port %d: %s speed=%dGbps mac=%s pci=%s driver=%s',
 633                      id, port['description'], port['speed'], port['src_mac'],
 634                      port['pci_addr'], port['driver'])
 635         # Make sure the 2 ports have the same speed
 636         if self.port_info[0]['speed'] != self.port_info[1]['speed']:
 637             raise TrafficGeneratorException('Traffic generator ports speed mismatch: %d/%d Gbps' %
 638                                             (self.port_info[0]['speed'],
 639                                              self.port_info[1]['speed']))
 640
 641     def __start_local_server(self):
 642         try:
 643             LOG.info("Starting TRex ...")
 644             self.__start_server()
 645             self.__connect_after_start()
 646         except (TimeoutError, STLError) as e:
 647             LOG.error('Cannot connect to TRex')
 648             LOG.error(traceback.format_exc())
 649             logpath = '/tmp/trex.log'
 650             if os.path.isfile(logpath):
 651                 # Wait for TRex to finish writing error message
 652                 last_size = 0
 653                 for _ in range(self.config.generic_retry_count):
 654                     size = os.path.getsize(logpath)
 655                     if size == last_size:
 656                         # probably not writing anymore
 657                         break
 658                     last_size = size
 659                     time.sleep(1)
 660                 with open(logpath, 'r') as f:
 661                     message = f.read()
 662             else:
 663                 message = e.message
 664             raise TrafficGeneratorException(message)
 665
 666     def __start_server(self):
 667         server = TRexTrafficServer()
 668         server.run_server(self.generator_config)
 669
 670     def __check_config(self):
 671         server = TRexTrafficServer()
 672         return server.check_config_updated(self.generator_config)
 673
 674     def __restart(self):
 675         LOG.info("Restarting TRex ...")
 676         self.__stop_server()
 677         # Wait for server stopped
 678         for _ in range(self.config.generic_retry_count):
 679             time.sleep(1)
 680             if not self.client.is_connected():
 681                 LOG.info("TRex is stopped...")
 682                 break
 683         self.__start_local_server()
 684
 685     def __stop_server(self):
 686         if self.generator_config.ip == '127.0.0.1':
 687             ports = self.client.get_acquired_ports()
 688             LOG.info('Release ports %s and stopping TRex...', ports)
 689             try:
 690                 if ports:
 691                     self.client.release(ports=ports)
 692                 self.client.server_shutdown()
 693             except STLError as e:
 694                 LOG.warning('Unable to stop TRex. Error: %s', e)
 695         else:
 696             LOG.info('Using remote TRex. Unable to stop TRex')
 697
 698     def resolve_arp(self):
 699         """Resolve all configured remote IP addresses.
 700
 701         return: None if ARP failed to resolve for all IP addresses
 702                 else a dict of list of dest macs indexed by port#
 703                 the dest macs in the list are indexed by the chain id
 704         """
 705         self.client.set_service_mode(ports=self.port_handle)
 706         LOG.info('Polling ARP until successful...')
 707         arp_dest_macs = {}
 708         for port, device in zip(self.port_handle, self.generator_config.devices):
 709             # there should be 1 stream config per chain
 710             stream_configs = device.get_stream_configs()
 711             chain_count = len(stream_configs)
 712             ctx = self.client.create_service_ctx(port=port)
 713             # all dest macs on this port indexed by chain ID
 714             dst_macs = [None] * chain_count
 715             dst_macs_count = 0
 716             # the index in the list is the chain id
 717             if self.config.vxlan or self.config.mpls:
 718                 arps = [
 719                     ServiceARP(ctx,
 720                                src_ip=device.vtep_src_ip,
 721                                dst_ip=device.vtep_dst_ip,
 722                                vlan=device.vtep_vlan)
 723                     for cfg in stream_configs
 724                 ]
 725             else:
 726                 arps = [
 727                     ServiceARP(ctx,
 728                                src_ip=cfg['ip_src_tg_gw'],
 729                                dst_ip=cfg['mac_discovery_gw'],
 730                                # will be None if no vlan tagging
 731                                vlan=cfg['vlan_tag'])
 732                     for cfg in stream_configs
 733                 ]
 734
 735             for attempt in range(self.config.generic_retry_count):
 736                 try:
 737                     ctx.run(arps)
 738                 except STLError:
 739                     LOG.error(traceback.format_exc())
 740                     continue
 741
 742                 unresolved = []
 743                 for chain_id, mac in enumerate(dst_macs):
 744                     if not mac:
 745                         arp_record = arps[chain_id].get_record()
 746                         if arp_record.dst_mac:
 747                             dst_macs[chain_id] = arp_record.dst_mac
 748                             dst_macs_count += 1
 749                             LOG.info('   ARP: port=%d chain=%d src IP=%s dst IP=%s -> MAC=%s',
 750                                      port, chain_id,
 751                                      arp_record.src_ip,
 752                                      arp_record.dst_ip, arp_record.dst_mac)
 753                         else:
 754                             unresolved.append(arp_record.dst_ip)
 755                 if dst_macs_count == chain_count:
 756                     arp_dest_macs[port] = dst_macs
 757                     LOG.info('ARP resolved successfully for port %s', port)
 758                     break
 759
 760                 retry = attempt + 1
 761                 LOG.info('Retrying ARP for: %s (retry %d/%d)',
 762                          unresolved, retry, self.config.generic_retry_count)
 763                 if retry < self.config.generic_retry_count:
 764                     time.sleep(self.config.generic_poll_sec)
 765             else:
 766                 LOG.error('ARP timed out for port %s (resolved %d out of %d)',
 767                           port,
 768                           dst_macs_count,
 769                           chain_count)
 770                 break
 771
 772         # if the capture from the TRex console was started before the arp request step,
 773         # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
 774         if not self.config.service_mode:
 775             self.client.set_service_mode(ports=self.port_handle, enabled=False)
 776         if len(arp_dest_macs) == len(self.port_handle):
 777             return arp_dest_macs
 778         return None
 779
 780     def __is_rate_enough(self, l2frame_size, rates, bidirectional, latency):
 781         """Check if rate provided by user is above requirements. Applies only if latency is True."""
 782         intf_speed = self.generator_config.intf_speed
 783         if latency:
 784             if bidirectional:
 785                 mult = 2
 786                 total_rate = 0
 787                 for rate in rates:
 788                     r = utils.convert_rates(l2frame_size, rate, intf_speed)
 789                     total_rate += int(r['rate_pps'])
 790             else:
 791                 mult = 1
 792                 total_rate = utils.convert_rates(l2frame_size, rates[0], intf_speed)
 793             # rate must be enough for latency stream and at least 1 pps for base stream per chain
 794             required_rate = (self.LATENCY_PPS + 1) * self.config.service_chain_count * mult
 795             result = utils.convert_rates(l2frame_size,
 796                                          {'rate_pps': required_rate},
 797                                          intf_speed * mult)
 798             result['result'] = total_rate >= required_rate
 799             return result
 800
 801         return {'result': True}
 802
 803     def create_traffic(self, l2frame_size, rates, bidirectional, latency=True, e2e=False):
 804         """Program all the streams in Trex server.
 805
 806         l2frame_size: L2 frame size or IMIX
 807         rates: a list of 2 rates to run each direction
 808                each rate is a dict like {'rate_pps': '10kpps'}
 809         bidirectional: True if bidirectional
 810         latency: True if latency measurement is needed
 811         e2e: True if performing "end to end" connectivity check
 812         """
 813         r = self.__is_rate_enough(l2frame_size, rates, bidirectional, latency)
 814         if not r['result']:
 815             raise TrafficGeneratorException(
 816                 'Required rate in total is at least one of: \n{pps}pps \n{bps}bps \n{load}%.'
 817                 .format(pps=r['rate_pps'],
 818                         bps=r['rate_bps'],
 819                         load=r['rate_percent']))
 820         self.l2_frame_size = l2frame_size
 821         # a dict of list of streams indexed by port#
 822         # in case of fixed size, has self.chain_count * 2 * 2 streams
 823         # (1 normal + 1 latency stream per direction per chain)
 824         # for IMIX, has self.chain_count * 2 * 4 streams
 825         # (3 normal + 1 latency stream per direction per chain)
 826         streamblock = {}
 827         for port in self.port_handle:
 828             streamblock[port] = []
 829         stream_cfgs = [d.get_stream_configs() for d in self.generator_config.devices]
 830         self.rates = [utils.to_rate_str(rate) for rate in rates]
 831         for chain_id, (fwd_stream_cfg, rev_stream_cfg) in enumerate(zip(*stream_cfgs)):
 832             streamblock[0].extend(self.generate_streams(self.port_handle[0],
 833                                                         chain_id,
 834                                                         fwd_stream_cfg,
 835                                                         l2frame_size,
 836                                                         latency=latency,
 837                                                         e2e=e2e))
 838             if len(self.rates) > 1:
 839                 streamblock[1].extend(self.generate_streams(self.port_handle[1],
 840                                                             chain_id,
 841                                                             rev_stream_cfg,
 842                                                             l2frame_size,
 843                                                             latency=bidirectional and latency,
 844                                                             e2e=e2e))
 845
 846         for port in self.port_handle:
 847             if self.config.vxlan:
 848                 self.client.set_port_attr(ports=port, vxlan_fs=[4789])
 849             else:
 850                 self.client.set_port_attr(ports=port, vxlan_fs=None)
 851             self.client.add_streams(streamblock[port], ports=port)
 852             LOG.info('Created %d traffic streams for port %s.', len(streamblock[port]), port)
 853
 854     def clear_streamblock(self):
 855         """Clear all streams from TRex."""
 856         self.rates = []
 857         self.client.reset(self.port_handle)
 858         LOG.info('Cleared all existing streams')
 859
 860     def get_stats(self):
 861         """Get stats from Trex."""
 862         stats = self.client.get_stats()
 863         return self.extract_stats(stats)
 864
 865     def get_macs(self):
 866         """Return the Trex local port MAC addresses.
 867
 868         return: a list of MAC addresses indexed by the port#
 869         """
 870         return [port['src_mac'] for port in self.port_info]
 871
 872     def get_port_speed_gbps(self):
 873         """Return the Trex local port MAC addresses.
 874
 875         return: a list of speed in Gbps indexed by the port#
 876         """
 877         return [port['speed'] for port in self.port_info]
 878
 879     def clear_stats(self):
 880         """Clear all stats in the traffic gneerator."""
 881         if self.port_handle:
 882             self.client.clear_stats()
 883
 884     def start_traffic(self):
 885         """Start generating traffic in all ports."""
 886         for port, rate in zip(self.port_handle, self.rates):
 887             self.client.start(ports=port, mult=rate, duration=self.config.duration_sec, force=True)
 888
 889     def stop_traffic(self):
 890         """Stop generating traffic."""
 891         self.client.stop(ports=self.port_handle)
 892
 893     def start_capture(self):
 894         """Capture all packets on both ports that are unicast to us."""
 895         if self.capture_id:
 896             self.stop_capture()
 897         # Need to filter out unwanted packets so we do not end up counting
 898         # src MACs of frames that are not unicast to us
 899         src_mac_list = self.get_macs()
 900         bpf_filter = "ether dst %s or ether dst %s" % (src_mac_list[0], src_mac_list[1])
 901         # ports must be set in service in order to enable capture
 902         self.client.set_service_mode(ports=self.port_handle)
 903         self.capture_id = self.client.start_capture \
 904             (rx_ports=self.port_handle, bpf_filter=bpf_filter)
 905
 906     def fetch_capture_packets(self):
 907         """Fetch capture packets in capture mode."""
 908         if self.capture_id:
 909             self.packet_list = []
 910             self.client.fetch_capture_packets(capture_id=self.capture_id['id'],
 911                                               output=self.packet_list)
 912
 913     def stop_capture(self):
 914         """Stop capturing packets."""
 915         if self.capture_id:
 916             self.client.stop_capture(capture_id=self.capture_id['id'])
 917             self.capture_id = None
 918             # if the capture from TRex console was started before the connectivity step,
 919             # it keeps 'service_mode' enabled, otherwise, it disables the 'service_mode'
 920             if not self.config.service_mode:
 921                 self.client.set_service_mode(ports=self.port_handle, enabled=False)
 922
 923     def cleanup(self):
 924         """Cleanup Trex driver."""
 925         if self.client:
 926             try:
 927                 self.client.reset(self.port_handle)
 928                 self.client.disconnect()
 929             except STLError:
 930                 # TRex does not like a reset while in disconnected state
 931                 pass
 932
 933     def set_service_mode(self, enabled=True):
 934         """Enable/disable the 'service_mode'."""
 935         self.client.set_service_mode(ports=self.port_handle, enabled=enabled)