1 # Copyright 2016 Cisco Systems, Inc. All rights reserved.
3 # Licensed under the Apache License, Version 2.0 (the "License"); you may
4 # not use this file except in compliance with the License. You may obtain
5 # a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12 # License for the specific language governing permissions and limitations
14 """Driver module for TRex traffic generator."""
21 from itertools import count
22 from nfvbench.log import LOG
23 from nfvbench.traffic_server import TRexTrafficServer
24 from nfvbench.utils import cast_integer
25 from nfvbench.utils import timeout
26 from nfvbench.utils import TimeoutError
27 from traffic_base import AbstractTrafficGenerator
28 from traffic_base import TrafficGeneratorException
29 import traffic_utils as utils
31 # pylint: disable=import-error
32 from trex_stl_lib.api import CTRexVmInsFixHwCs
33 from trex_stl_lib.api import Dot1Q
34 from trex_stl_lib.api import Ether
35 from trex_stl_lib.api import IP
36 from trex_stl_lib.api import STLClient
37 from trex_stl_lib.api import STLError
38 from trex_stl_lib.api import STLFlowLatencyStats
39 from trex_stl_lib.api import STLFlowStats
40 from trex_stl_lib.api import STLPktBuilder
41 from trex_stl_lib.api import STLScVmRaw
42 from trex_stl_lib.api import STLStream
43 from trex_stl_lib.api import STLTXCont
44 from trex_stl_lib.api import STLVmFixChecksumHw
45 from trex_stl_lib.api import STLVmFlowVar
46 from trex_stl_lib.api import STLVmFlowVarRepetableRandom
47 from trex_stl_lib.api import STLVmWrFlowVar
48 from trex_stl_lib.api import UDP
49 from trex_stl_lib.services.trex_stl_service_arp import STLServiceARP
52 # pylint: enable=import-error
55 class TRex(AbstractTrafficGenerator):
56 """TRex traffic generator driver."""
59 CHAIN_PG_ID_MASK = 0x007F
60 PORT_PG_ID_MASK = 0x0080
61 LATENCY_PG_ID_MASK = 0x0100
63 def __init__(self, traffic_client):
65 AbstractTrafficGenerator.__init__(self, traffic_client)
69 self.chain_count = self.generator_config.service_chain_count
71 self.capture_id = None
74 def get_version(self):
75 """Get the Trex version."""
76 return self.client.get_server_version() if self.client else ''
78 def get_pg_id(self, port, chain_id):
79 """Calculate the packet group IDs to use for a given port/stream type/chain_id.
82 chain_id: identifies to which chain the pg_id is associated (0 to 255)
83 return: pg_id, lat_pg_id
85 We use a bit mask to set up the 3 fields:
86 0x007F: chain ID (8 bits for a max of 128 chains)
90 pg_id = port * TRex.PORT_PG_ID_MASK | chain_id
91 return pg_id, pg_id | TRex.LATENCY_PG_ID_MASK
93 def extract_stats(self, in_stats):
94 """Extract stats from dict returned by Trex API.
96 :param in_stats: dict as returned by TRex api
98 utils.nan_replace(in_stats)
102 # port_handles should have only 2 elements: [0, 1]
103 # so (1 - ph) will be the index for the far end port
104 for ph in self.port_handle:
106 far_end_stats = in_stats[1 - ph]
109 'total_pkts': cast_integer(stats['opackets']),
110 'total_pkt_bytes': cast_integer(stats['obytes']),
111 'pkt_rate': cast_integer(stats['tx_pps']),
112 'pkt_bit_rate': cast_integer(stats['tx_bps'])
115 'total_pkts': cast_integer(stats['ipackets']),
116 'total_pkt_bytes': cast_integer(stats['ibytes']),
117 'pkt_rate': cast_integer(stats['rx_pps']),
118 'pkt_bit_rate': cast_integer(stats['rx_bps']),
119 # how many pkts were dropped in RX direction
120 # need to take the tx counter on the far end port
121 'dropped_pkts': cast_integer(
122 far_end_stats['opackets'] - stats['ipackets'])
125 self.__combine_latencies(in_stats, result[ph]['rx'], ph)
127 total_tx_pkts = result[0]['tx']['total_pkts'] + result[1]['tx']['total_pkts']
128 result["total_tx_rate"] = cast_integer(total_tx_pkts / self.config.duration_sec)
129 result["flow_stats"] = in_stats["flow_stats"]
130 result["latency"] = in_stats["latency"]
133 def get_stream_stats(self, trex_stats, if_stats, latencies, chain_idx):
134 """Extract the aggregated stats for a given chain.
136 trex_stats: stats as returned by get_stats()
137 if_stats: a list of 2 interface stats to update (port 0 and 1)
138 latencies: a list of 2 Latency instances to update for this chain (port 0 and 1)
139 latencies[p] is the latency for packets sent on port p
140 if there are no latency streams, the Latency instances are not modified
141 chain_idx: chain index of the interface stats
143 The packet counts include normal and latency streams.
145 Trex returns flows stats as follows:
147 'flow_stats': {0: {'rx_bps': {0: 0, 1: 0, 'total': 0},
148 'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
149 'rx_bytes': {0: nan, 1: nan, 'total': nan},
150 'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
151 'rx_pps': {0: 0, 1: 0, 'total': 0},
152 'tx_bps': {0: 0, 1: 0, 'total': 0},
153 'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
154 'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
155 'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
156 'tx_pps': {0: 0, 1: 0, 'total': 0}},
157 1: {'rx_bps': {0: 0, 1: 0, 'total': 0},
158 'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
159 'rx_bytes': {0: nan, 1: nan, 'total': nan},
160 'rx_pkts': {0: 0, 1: 15001, 'total': 15001},
161 'rx_pps': {0: 0, 1: 0, 'total': 0},
162 'tx_bps': {0: 0, 1: 0, 'total': 0},
163 'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
164 'tx_bytes': {0: 1020068, 1: 0, 'total': 1020068},
165 'tx_pkts': {0: 15001, 1: 0, 'total': 15001},
166 'tx_pps': {0: 0, 1: 0, 'total': 0}},
167 128: {'rx_bps': {0: 0, 1: 0, 'total': 0},
168 'rx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
169 'rx_bytes': {0: nan, 1: nan, 'total': nan},
170 'rx_pkts': {0: 15001, 1: 0, 'total': 15001},
171 'rx_pps': {0: 0, 1: 0, 'total': 0},
172 'tx_bps': {0: 0, 1: 0, 'total': 0},
173 'tx_bps_l1': {0: 0.0, 1: 0.0, 'total': 0.0},
174 'tx_bytes': {0: 0, 1: 1020068, 'total': 1020068},
175 'tx_pkts': {0: 0, 1: 15001, 'total': 15001},
176 'tx_pps': {0: 0, 1: 0, 'total': 0}},etc...
178 the pg_id (0, 1, 128,...) is the key of the dict and is obtained using the
180 packet counters for a given stream sent on port p are reported as:
181 - tx_pkts[p] on port p
182 - rx_pkts[1-p] on the far end port
184 This is a tricky/critical counter transposition operation because
185 the results are grouped by port (not by stream):
186 tx_pkts_port(p=0) comes from pg_id(port=0, chain_idx)['tx_pkts'][0]
187 rx_pkts_port(p=0) comes from pg_id(port=1, chain_idx)['rx_pkts'][0]
188 tx_pkts_port(p=1) comes from pg_id(port=1, chain_idx)['tx_pkts'][1]
189 rx_pkts_port(p=1) comes from pg_id(port=0, chain_idx)['rx_pkts'][1]
191 or using a more generic formula:
192 tx_pkts_port(p) comes from pg_id(port=p, chain_idx)['tx_pkts'][p]
193 rx_pkts_port(p) comes from pg_id(port=1-p, chain_idx)['rx_pkts'][p]
195 the second formula is equivalent to
196 rx_pkts_port(1-p) comes from pg_id(port=p, chain_idx)['rx_pkts'][1-p]
198 If there are latency streams, those same counters need to be added in the same way
200 def get_latency(lval):
202 return int(round(lval))
208 for port in range(2):
209 pg_id, lat_pg_id = self.get_pg_id(port, chain_idx)
210 for pid in [pg_id, lat_pg_id]:
212 pg_stats = trex_stats['flow_stats'][pid]
213 if_stats[port].tx += pg_stats['tx_pkts'][port]
214 if_stats[1 - port].rx += pg_stats['rx_pkts'][1 - port]
218 lat = trex_stats['latency'][lat_pg_id]['latency']
219 # dropped_pkts += lat['err_cntrs']['dropped']
220 latencies[port].max_usec = get_latency(lat['total_max'])
221 latencies[port].min_usec = get_latency(lat['total_min'])
222 latencies[port].avg_usec = get_latency(lat['average'])
226 def __combine_latencies(self, in_stats, results, port_handle):
227 """Traverse TRex result dictionary and combines chosen latency stats."""
230 total_min = float("inf")
231 for chain_id in range(self.chain_count):
233 _, lat_pg_id = self.get_pg_id(port_handle, chain_id)
234 lat = in_stats['latency'][lat_pg_id]['latency']
235 # dropped_pkts += lat['err_cntrs']['dropped']
236 total_max = max(lat['total_max'], total_max)
237 total_min = min(lat['total_min'], total_min)
238 average += lat['average']
241 if total_min == float("inf"):
243 results['min_delay_usec'] = total_min
244 results['max_delay_usec'] = total_max
245 results['avg_delay_usec'] = int(average / self.chain_count)
247 def _create_pkt(self, stream_cfg, l2frame_size):
248 pkt_base = Ether(src=stream_cfg['mac_src'], dst=stream_cfg['mac_dst'])
249 if stream_cfg['vlan_tag'] is not None:
250 # 50 = 14 (Ethernet II) + 4 (Vlan tag) + 4 (CRC Checksum) + 20 (IPv4) + 8 (UDP)
251 pkt_base /= Dot1Q(vlan=stream_cfg['vlan_tag'])
252 l2payload_size = int(l2frame_size) - 50
254 # 46 = 14 (Ethernet II) + 4 (CRC Checksum) + 20 (IPv4) + 8 (UDP)
255 l2payload_size = int(l2frame_size) - 46
256 payload = 'x' * l2payload_size
258 if stream_cfg['udp_src_port']:
259 udp_args['sport'] = int(stream_cfg['udp_src_port'])
260 if stream_cfg['udp_dst_port']:
261 udp_args['dport'] = int(stream_cfg['udp_dst_port'])
262 pkt_base /= IP() / UDP(**udp_args)
264 if stream_cfg['ip_addrs_step'] == 'random':
265 src_fv = STLVmFlowVarRepetableRandom(
267 min_value=stream_cfg['ip_src_addr'],
268 max_value=stream_cfg['ip_src_addr_max'],
270 seed=random.randint(0, 32767),
271 limit=stream_cfg['ip_src_count'])
272 dst_fv = STLVmFlowVarRepetableRandom(
274 min_value=stream_cfg['ip_dst_addr'],
275 max_value=stream_cfg['ip_dst_addr_max'],
277 seed=random.randint(0, 32767),
278 limit=stream_cfg['ip_dst_count'])
280 src_fv = STLVmFlowVar(
282 min_value=stream_cfg['ip_src_addr'],
283 max_value=stream_cfg['ip_src_addr'],
286 step=stream_cfg['ip_addrs_step'])
287 dst_fv = STLVmFlowVar(
289 min_value=stream_cfg['ip_dst_addr'],
290 max_value=stream_cfg['ip_dst_addr_max'],
293 step=stream_cfg['ip_addrs_step'])
297 STLVmWrFlowVar(fv_name="ip_src", pkt_offset="IP.src"),
299 STLVmWrFlowVar(fv_name="ip_dst", pkt_offset="IP.dst"),
300 STLVmFixChecksumHw(l3_offset="IP",
302 l4_type=CTRexVmInsFixHwCs.L4_TYPE_UDP)
305 return STLPktBuilder(pkt=pkt_base / payload, vm=STLScVmRaw(vm_param))
307 def generate_streams(self, port, chain_id, stream_cfg, l2frame, latency=True):
308 """Create a list of streams corresponding to a given chain and stream config.
310 port: port where the streams originate (0 or 1)
311 chain_id: the chain to which the streams are associated to
312 stream_cfg: stream configuration
313 l2frame: L2 frame size
314 latency: if True also create a latency stream
317 pg_id, lat_pg_id = self.get_pg_id(port, chain_id)
318 if l2frame == 'IMIX':
319 min_size = 64 if stream_cfg['vlan_tag'] is None else 68
320 self.adjust_imix_min_size(min_size)
321 for ratio, l2_frame_size in zip(self.imix_ratios, self.imix_l2_sizes):
322 pkt = self._create_pkt(stream_cfg, l2_frame_size)
323 streams.append(STLStream(packet=pkt,
324 flow_stats=STLFlowStats(pg_id=pg_id),
325 mode=STLTXCont(pps=ratio)))
328 # for IMIX, the latency packets have the average IMIX packet size
329 pkt = self._create_pkt(stream_cfg, self.imix_avg_l2_size)
332 pkt = self._create_pkt(stream_cfg, l2frame)
333 streams.append(STLStream(packet=pkt,
334 flow_stats=STLFlowStats(pg_id=pg_id),
338 streams.append(STLStream(packet=pkt,
339 flow_stats=STLFlowLatencyStats(pg_id=lat_pg_id),
340 mode=STLTXCont(pps=self.LATENCY_PPS)))
344 def __connect(self, client):
347 def __connect_after_start(self):
348 # after start, Trex may take a bit of time to initialize
349 # so we need to retry a few times
350 for it in xrange(self.config.generic_retry_count):
353 self.client.connect()
355 except Exception as ex:
356 if it == (self.config.generic_retry_count - 1):
358 LOG.info("Retrying connection to TRex (%s)...", ex.message)
361 """Connect to the TRex server."""
362 server_ip = self.generator_config.ip
363 LOG.info("Connecting to TRex (%s)...", server_ip)
365 # Connect to TRex server
366 self.client = STLClient(server=server_ip)
368 self.__connect(self.client)
369 except (TimeoutError, STLError) as e:
370 if server_ip == '127.0.0.1':
372 self.__start_server()
373 self.__connect_after_start()
374 except (TimeoutError, STLError) as e:
375 LOG.error('Cannot connect to TRex')
376 LOG.error(traceback.format_exc())
377 logpath = '/tmp/trex.log'
378 if os.path.isfile(logpath):
379 # Wait for TRex to finish writing error message
381 for _ in xrange(self.config.generic_retry_count):
382 size = os.path.getsize(logpath)
383 if size == last_size:
384 # probably not writing anymore
388 with open(logpath, 'r') as f:
392 raise TrafficGeneratorException(message)
394 raise TrafficGeneratorException(e.message)
396 ports = list(self.generator_config.ports)
397 self.port_handle = ports
399 self.client.reset(ports)
400 # Read HW information from each port
401 # this returns an array of dict (1 per port)
403 Example of output for Intel XL710
404 [{'arp': '-', 'src_ipv4': '-', u'supp_speeds': [40000], u'is_link_supported': True,
405 'grat_arp': 'off', 'speed': 40, u'index': 0, 'link_change_supported': 'yes',
406 u'rx': {u'counters': 127, u'caps': [u'flow_stats', u'latency']},
407 u'is_virtual': 'no', 'prom': 'off', 'src_mac': u'3c:fd:fe:a8:24:48', 'status': 'IDLE',
408 u'description': u'Ethernet Controller XL710 for 40GbE QSFP+',
409 'dest': u'fa:16:3e:3c:63:04', u'is_fc_supported': False, 'vlan': '-',
410 u'driver': u'net_i40e', 'led_change_supported': 'yes', 'rx_filter_mode': 'hardware match',
411 'fc': 'none', 'link': 'UP', u'hw_mac': u'3c:fd:fe:a8:24:48', u'pci_addr': u'0000:5e:00.0',
412 'mult': 'off', 'fc_supported': 'no', u'is_led_supported': True, 'rx_queue': 'off',
413 'layer_mode': 'Ethernet', u'numa': 0}, ...]
415 self.port_info = self.client.get_port_info(ports)
416 LOG.info('Connected to TRex')
417 for id, port in enumerate(self.port_info):
418 LOG.info(' Port %d: %s speed=%dGbps mac=%s pci=%s driver=%s',
419 id, port['description'], port['speed'], port['src_mac'],
420 port['pci_addr'], port['driver'])
421 # Make sure the 2 ports have the same speed
422 if self.port_info[0]['speed'] != self.port_info[1]['speed']:
423 raise TrafficGeneratorException('Traffic generator ports speed mismatch: %d/%d Gbps' %
424 (self.port_info[0]['speed'],
425 self.port_info[1]['speed']))
427 def __start_server(self):
428 server = TRexTrafficServer()
429 server.run_server(self.generator_config)
431 def resolve_arp(self):
432 """Resolve all configured remote IP addresses.
434 return: None if ARP failed to resolve for all IP addresses
435 else a dict of list of dest macs indexed by port#
436 the dest macs in the list are indexed by the chain id
438 self.client.set_service_mode(ports=self.port_handle)
439 LOG.info('Polling ARP until successful...')
441 for port, device in zip(self.port_handle, self.generator_config.devices):
442 # there should be 1 stream config per chain
443 stream_configs = device.get_stream_configs()
444 chain_count = len(stream_configs)
445 ctx = self.client.create_service_ctx(port=port)
446 # all dest macs on this port indexed by chain ID
447 dst_macs = [None] * chain_count
449 # the index in the list is the chain id
452 src_ip=cfg['ip_src_tg_gw'],
453 dst_ip=cfg['mac_discovery_gw'],
454 # will be None if no vlan tagging
455 vlan=cfg['vlan_tag'])
456 for cfg in stream_configs
459 for attempt in range(self.config.generic_retry_count):
463 LOG.error(traceback.format_exc())
467 for chain_id, mac in enumerate(dst_macs):
469 arp_record = arps[chain_id].get_record()
470 if arp_record.dst_mac:
471 dst_macs[chain_id] = arp_record.dst_mac
473 LOG.info(' ARP: port=%d chain=%d src IP=%s dst IP=%s -> MAC=%s',
476 arp_record.dst_ip, arp_record.dst_mac)
478 unresolved.append(arp_record.dst_ip)
479 if dst_macs_count == chain_count:
480 arp_dest_macs[port] = dst_macs
481 LOG.info('ARP resolved successfully for port %s', port)
485 LOG.info('Retrying ARP for: %s (retry %d/%d)',
486 unresolved, retry, self.config.generic_retry_count)
487 if retry < self.config.generic_retry_count:
488 time.sleep(self.config.generic_poll_sec)
490 LOG.error('ARP timed out for port %s (resolved %d out of %d)',
496 self.client.set_service_mode(ports=self.port_handle, enabled=False)
497 if len(arp_dest_macs) == len(self.port_handle):
501 def __is_rate_enough(self, l2frame_size, rates, bidirectional, latency):
502 """Check if rate provided by user is above requirements. Applies only if latency is True."""
503 intf_speed = self.generator_config.intf_speed
509 r = utils.convert_rates(l2frame_size, rate, intf_speed)
510 total_rate += int(r['rate_pps'])
513 total_rate = utils.convert_rates(l2frame_size, rates[0], intf_speed)
514 # rate must be enough for latency stream and at least 1 pps for base stream per chain
515 required_rate = (self.LATENCY_PPS + 1) * self.config.service_chain_count * mult
516 result = utils.convert_rates(l2frame_size,
517 {'rate_pps': required_rate},
519 result['result'] = total_rate >= required_rate
522 return {'result': True}
524 def create_traffic(self, l2frame_size, rates, bidirectional, latency=True):
525 """Program all the streams in Trex server.
527 l2frame_size: L2 frame size or IMIX
528 rates: a list of 2 rates to run each direction
529 each rate is a dict like {'rate_pps': '10kpps'}
530 bidirectional: True if bidirectional
531 latency: True if latency measurement is needed
533 r = self.__is_rate_enough(l2frame_size, rates, bidirectional, latency)
535 raise TrafficGeneratorException(
536 'Required rate in total is at least one of: \n{pps}pps \n{bps}bps \n{load}%.'
537 .format(pps=r['rate_pps'],
539 load=r['rate_percent']))
540 # a dict of list of streams indexed by port#
541 # in case of fixed size, has self.chain_count * 2 * 2 streams
542 # (1 normal + 1 latency stream per direction per chain)
543 # for IMIX, has self.chain_count * 2 * 4 streams
544 # (3 normal + 1 latency stream per direction per chain)
546 for port in self.port_handle:
547 streamblock[port] = []
548 stream_cfgs = [d.get_stream_configs() for d in self.generator_config.devices]
549 self.rates = [utils.to_rate_str(rate) for rate in rates]
550 for chain_id, (fwd_stream_cfg, rev_stream_cfg) in enumerate(zip(*stream_cfgs)):
551 streamblock[0].extend(self.generate_streams(self.port_handle[0],
556 if len(self.rates) > 1:
557 streamblock[1].extend(self.generate_streams(self.port_handle[1],
561 latency=bidirectional and latency))
563 for port in self.port_handle:
564 self.client.add_streams(streamblock[port], ports=port)
565 LOG.info('Created %d traffic streams for port %s.', len(streamblock[port]), port)
567 def clear_streamblock(self):
568 """Clear all streams from TRex."""
570 self.client.reset(self.port_handle)
571 LOG.info('Cleared all existing streams')
574 """Get stats from Trex."""
575 stats = self.client.get_stats()
576 return self.extract_stats(stats)
579 """Return the Trex local port MAC addresses.
581 return: a list of MAC addresses indexed by the port#
583 return [port['src_mac'] for port in self.port_info]
585 def get_port_speed_gbps(self):
586 """Return the Trex local port MAC addresses.
588 return: a list of speed in Gbps indexed by the port#
590 return [port['speed'] for port in self.port_info]
592 def clear_stats(self):
593 """Clear all stats in the traffic gneerator."""
595 self.client.clear_stats()
597 def start_traffic(self):
598 """Start generating traffic in all ports."""
599 for port, rate in zip(self.port_handle, self.rates):
600 self.client.start(ports=port, mult=rate, duration=self.config.duration_sec, force=True)
602 def stop_traffic(self):
603 """Stop generating traffic."""
604 self.client.stop(ports=self.port_handle)
606 def start_capture(self):
607 """Capture all packets on both ports that are unicast to us."""
610 # Need to filter out unwanted packets so we do not end up counting
611 # src MACs of frames that are not unicast to us
612 src_mac_list = self.get_macs()
613 bpf_filter = "ether dst %s or ether dst %s" % (src_mac_list[0], src_mac_list[1])
614 # ports must be set in service in order to enable capture
615 self.client.set_service_mode(ports=self.port_handle)
616 self.capture_id = self.client.start_capture(rx_ports=self.port_handle,
617 bpf_filter=bpf_filter)
619 def fetch_capture_packets(self):
620 """Fetch capture packets in capture mode."""
622 self.packet_list = []
623 self.client.fetch_capture_packets(capture_id=self.capture_id['id'],
624 output=self.packet_list)
626 def stop_capture(self):
627 """Stop capturing packets."""
629 self.client.stop_capture(capture_id=self.capture_id['id'])
630 self.capture_id = None
631 self.client.set_service_mode(ports=self.port_handle, enabled=False)
634 """Cleanup Trex driver."""
637 self.client.reset(self.port_handle)
638 self.client.disconnect()
640 # TRex does not like a reset while in disconnected state