[NFVBENCH-81]With some Intel X710 NIC cards, NFVbench reports erroneous RX counters

[nfvbench.git] / nfvbench / traffic_client.py

diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py
old mode 100644 (file)
new mode 100755 (executable)
index a1c4954..2ce118c
--- a/nfvbench/traffic_client.py
+++ b/nfvbench/traffic_client.py
@@ -13,6 +13,7 @@
 #    under the License.
 
 from datetime import datetime
 #    under the License.
 
 from datetime import datetime

+import re

 import socket
 import struct
 import time
 import socket
 import struct
 import time


@@ -67,6 +68,9 @@ class TrafficRunner(object):
     def poll_stats(self):
         if not self.is_running():
             return None
     def poll_stats(self):
         if not self.is_running():
             return None

+        if self.client.skip_sleep:
+            self.stop()
+            return self.client.get_stats()

         time_elapsed = self.time_elapsed()
         if time_elapsed > self.duration_sec:
             self.stop()
         time_elapsed = self.time_elapsed()
         if time_elapsed > self.duration_sec:
             self.stop()


@@ -102,10 +106,10 @@ class IpBlock(object):
         '''Reserve a range of count consecutive IP addresses spaced by step
         '''
         if self.next_free + count > self.max_available:
         '''Reserve a range of count consecutive IP addresses spaced by step
         '''
         if self.next_free + count > self.max_available:

-            raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d',
-                             self.next_free,
-                             self.max_available,
-                             count)
+            raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d' %
+                             (self.next_free,
+                              self.max_available,
+                              count))

         first_ip = self.get_ip(self.next_free)
         last_ip = self.get_ip(self.next_free + count - 1)
         self.next_free += count
         first_ip = self.get_ip(self.next_free)
         last_ip = self.get_ip(self.next_free + count - 1)
         self.next_free += count


@@ -119,7 +123,7 @@ class Device(object):
     def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
                  gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
                  gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,
     def __init__(self, port, pci, switch_port=None, vtep_vlan=None, ip=None, tg_gateway_ip=None,
                  gateway_ip=None, ip_addrs_step=None, tg_gateway_ip_addrs_step=None,
                  gateway_ip_addrs_step=None, udp_src_port=None, udp_dst_port=None,

-                 chain_count=1, flow_count=1, vlan_tagging=False):
+                 dst_mac=None, chain_count=1, flow_count=1, vlan_tagging=False):

         self.chain_count = chain_count
         self.flow_count = flow_count
         self.dst = None
         self.chain_count = chain_count
         self.flow_count = flow_count
         self.dst = None


@@ -130,6 +134,7 @@ class Device(object):
         self.vlan_tagging = vlan_tagging
         self.pci = pci
         self.mac = None
         self.vlan_tagging = vlan_tagging
         self.pci = pci
         self.mac = None

+        self.dst_mac = dst_mac

         self.vm_mac_list = None
         subnet = IPNetwork(ip)
         self.ip = subnet.ip.format()
         self.vm_mac_list = None
         subnet = IPNetwork(ip)
         self.ip = subnet.ip.format()


@@ -186,10 +191,16 @@ class Device(object):
         for chain_idx in xrange(self.chain_count):
             src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count)
             dst_ip_first, dst_ip_last = self.dst.ip_block.reserve_ip_range(cur_chain_flow_count)
         for chain_idx in xrange(self.chain_count):
             src_ip_first, src_ip_last = self.ip_block.reserve_ip_range(cur_chain_flow_count)
             dst_ip_first, dst_ip_last = self.dst.ip_block.reserve_ip_range(cur_chain_flow_count)

+
+            dst_mac = self.dst_mac[chain_idx] if self.dst_mac is not None else self.dst.mac
+            if not re.match("[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", dst_mac.lower()):
+                raise TrafficClientException("Invalid MAC address '{mac}' specified in "
+                                             "mac_addrs_left/right".format(mac=dst_mac))
+

             configs.append({
                 'count': cur_chain_flow_count,
                 'mac_src': self.mac,
             configs.append({
                 'count': cur_chain_flow_count,
                 'mac_src': self.mac,

-                'mac_dst': self.dst.mac if service_chain == ChainType.EXT else self.vm_mac_list[
+                'mac_dst': dst_mac if service_chain == ChainType.EXT else self.vm_mac_list[

                     chain_idx],
                 'ip_src_addr': src_ip_first,
                 'ip_src_addr_max': src_ip_last,
                     chain_idx],
                 'ip_src_addr': src_ip_first,
                 'ip_src_addr_max': src_ip_last,


@@ -267,6 +278,8 @@ class RunningTrafficProfile(object):
         self.src_device = None
         self.dst_device = None
         self.vm_mac_list = None
         self.src_device = None
         self.dst_device = None
         self.vm_mac_list = None

+        self.mac_addrs_left = generator_config.mac_addrs_left
+        self.mac_addrs_right = generator_config.mac_addrs_right

         self.__prep_interfaces(generator_config)
 
     def to_json(self):
         self.__prep_interfaces(generator_config)
 
     def to_json(self):


@@ -302,7 +315,8 @@ class RunningTrafficProfile(object):
             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
             'udp_src_port': generator_config.udp_src_port,
             'udp_dst_port': generator_config.udp_dst_port,
             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
             'udp_src_port': generator_config.udp_src_port,
             'udp_dst_port': generator_config.udp_dst_port,

-            'vlan_tagging': self.vlan_tagging
+            'vlan_tagging': self.vlan_tagging,
+            'dst_mac': generator_config.mac_addrs_left

         }
         dst_config = {
             'chain_count': self.service_chain_count,
         }
         dst_config = {
             'chain_count': self.service_chain_count,


@@ -315,7 +329,8 @@ class RunningTrafficProfile(object):
             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
             'udp_src_port': generator_config.udp_src_port,
             'udp_dst_port': generator_config.udp_dst_port,
             'tg_gateway_ip_addrs_step': self.tg_gateway_ip_addrs_step,
             'udp_src_port': generator_config.udp_src_port,
             'udp_dst_port': generator_config.udp_dst_port,

-            'vlan_tagging': self.vlan_tagging
+            'vlan_tagging': self.vlan_tagging,
+            'dst_mac': generator_config.mac_addrs_right

         }
 
         self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))
         }
 
         self.src_device = Device(**dict(src_config, **generator_config.interfaces[0]))


@@ -393,7 +408,7 @@ class TrafficGeneratorFactory(object):
 class TrafficClient(object):
     PORTS = [0, 1]
 
 class TrafficClient(object):
     PORTS = [0, 1]
 

-    def __init__(self, config, notifier=None):
+    def __init__(self, config, notifier=None, skip_sleep=False):

         generator_factory = TrafficGeneratorFactory(config)
         self.gen = generator_factory.get_generator_client()
         self.tool = generator_factory.get_tool()
         generator_factory = TrafficGeneratorFactory(config)
         self.gen = generator_factory.get_generator_client()
         self.tool = generator_factory.get_tool()


@@ -414,6 +429,8 @@ class TrafficClient(object):
         self.current_total_rate = {'rate_percent': '10'}
         if self.config.single_run:
             self.current_total_rate = utils.parse_rate_str(self.config.rate)
         self.current_total_rate = {'rate_percent': '10'}
         if self.config.single_run:
             self.current_total_rate = utils.parse_rate_str(self.config.rate)

+        # UT with dummy TG can bypass all sleeps
+        self.skip_sleep = skip_sleep

 
     def set_macs(self):
         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
 
     def set_macs(self):
         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):


@@ -436,52 +453,52 @@ class TrafficClient(object):
         Ensure traffic generator receives packets it has transmitted.
         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 
         Ensure traffic generator receives packets it has transmitted.
         This ensures end to end connectivity and also waits until VMs are ready to forward packets.
 

-        At this point all VMs are in active state, but forwarding does not have to work.
-        Small amount of traffic is sent to every chain. Then total of sent and received packets
-        is compared. If ratio between received and transmitted packets is higher than (N-1)/N,
-        N being number of chains, traffic flows through every chain and real measurements can be
-        performed.
+        VMs that are started and in active state may not pass traffic yet. It is imperative to make
+        sure that all VMs are passing traffic in both directions before starting any benchmarking.
+        To verify this, we need to send at a low frequency bi-directional packets and make sure
+        that we receive all packets back from all VMs. The number of flows is equal to 2 times
+        the number of chains (1 per direction) and we need to make sure we receive packets coming
+        from exactly 2 x chain count different source MAC addresses.

 
         Example:
             PVP chain (1 VM per chain)
             N = 10 (number of chains)
 
         Example:
             PVP chain (1 VM per chain)
             N = 10 (number of chains)

-            threshold = (N-1)/N = 9/10 = 0.9 (acceptable ratio ensuring working conditions)
-            if total_received/total_sent > 0.9, traffic is flowing to more than 9 VMs meaning
-            all 10 VMs are in operational state.
+            Flow count = 20 (number of flows)
+            If the number of unique source MAC addresses from received packets is 20 then
+            all 10 VMs 10 VMs are in operational state.

         """
         LOG.info('Starting traffic generator to ensure end-to-end connectivity')
         """
         LOG.info('Starting traffic generator to ensure end-to-end connectivity')

-        rate_pps = {'rate_pps': str(self.config.service_chain_count * 100)}
+        rate_pps = {'rate_pps': str(self.config.service_chain_count * 1)}

         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 
         # ensures enough traffic is coming back
         self.gen.create_traffic('64', [rate_pps, rate_pps], bidirectional=True, latency=False)
 
         # ensures enough traffic is coming back

-        threshold = (self.config.service_chain_count - 1) / float(self.config.service_chain_count)

         retry_count = (self.config.check_traffic_time_sec +
                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec
         retry_count = (self.config.check_traffic_time_sec +
                        self.config.generic_poll_sec - 1) / self.config.generic_poll_sec

+        mac_addresses = set()
+        ln = 0

         for it in xrange(retry_count):
             self.gen.clear_stats()
             self.gen.start_traffic()
         for it in xrange(retry_count):
             self.gen.clear_stats()
             self.gen.start_traffic()

+            self.gen.start_capture()

             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)
             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)

-            time.sleep(self.config.generic_poll_sec)
+            if not self.skip_sleep:
+                time.sleep(self.config.generic_poll_sec)

             self.gen.stop_traffic()
             self.gen.stop_traffic()

-            stats = self.gen.get_stats()
-
-            # compute total sent and received traffic on both ports
-            total_rx = 0
-            total_tx = 0
-            for port in self.PORTS:
-                total_rx += float(stats[port]['rx'].get('total_pkts', 0))
-                total_tx += float(stats[port]['tx'].get('total_pkts', 0))
-
-            # how much of traffic came back
-            ratio = total_rx / total_tx if total_tx else 0
-
-            if ratio > threshold:
-                self.gen.clear_stats()
-                self.gen.clear_streamblock()
-                LOG.info('End-to-end connectivity ensured')
-                return
-
-            time.sleep(self.config.generic_poll_sec)
+            self.gen.fetch_capture_packets()
+            self.gen.stop_capture()
+
+            for packet in self.gen.packet_list:
+                mac_addresses.add(packet['binary'][6:12])
+                if ln != len(mac_addresses):
+                    ln = len(mac_addresses)
+                    LOG.info('Flows passing traffic %d / %d', ln,
+                             self.config.service_chain_count * 2)
+                if len(mac_addresses) == self.config.service_chain_count * 2:
+                    LOG.info('End-to-end connectivity ensured')
+                    return
+
+            if not self.skip_sleep:
+                time.sleep(self.config.generic_poll_sec)

 
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 
 
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 


@@ -499,6 +516,10 @@ class TrafficClient(object):
             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
             if unidir_reverse_pps > 0:
                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})
             unidir_reverse_pps = int(self.config.unidir_reverse_traffic_pps)
             if unidir_reverse_pps > 0:
                 self.run_config['rates'].append({'rate_pps': str(unidir_reverse_pps)})

+        # Fix for [NFVBENCH-67], convert the rate string to PPS
+        for idx, rate in enumerate(self.run_config['rates']):
+            if 'rate_pps' not in rate:
+                self.run_config['rates'][idx] = {'rate_pps': self.__convert_rates(rate)['rate_pps']}

 
         self.gen.clear_streamblock()
         self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)
 
         self.gen.clear_streamblock()
         self.gen.create_traffic(frame_size, self.run_config['rates'], bidirectional, latency=True)


@@ -766,13 +787,11 @@ class TrafficClient(object):
     def cancel_traffic(self):
         self.runner.stop()
 
     def cancel_traffic(self):
         self.runner.stop()
 

-    def get_interface(self, port_index):
+    def get_interface(self, port_index, stats):

         port = self.gen.port_handle[port_index]
         tx, rx = 0, 0
         port = self.gen.port_handle[port_index]
         tx, rx = 0, 0

-        if not self.config.no_traffic:
-            stats = self.get_stats()
-            if port in stats:
-                tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])
+        if stats and port in stats:
+            tx, rx = int(stats[port]['tx']['total_pkts']), int(stats[port]['rx']['total_pkts'])

         return Interface('traffic-generator', self.tool.lower(), tx, rx)
 
     def get_traffic_config(self):
         return Interface('traffic-generator', self.tool.lower(), tx, rx)
 
     def get_traffic_config(self):


@@ -816,7 +835,6 @@ class TrafficClient(object):
         for direction in ['orig', 'tx', 'rx']:
             total[direction] = {}
             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:
         for direction in ['orig', 'tx', 'rx']:
             total[direction] = {}
             for unit in ['rate_percent', 'rate_bps', 'rate_pps']:

-

                 total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()])
 
         r['direction-total'] = total
                 total[direction][unit] = sum([float(x[direction][unit]) for x in r.values()])
 
         r['direction-total'] = total