[NFVBENCH-59] Add Unit Testing of the NDR/PDR convergence algorithm using the dummy...

[NFVBENCH-60] Fix pylint warnings

Change-Id: I72deec060bf25774d1be33eaeefc74b42a576483
Signed-off-by: ahothan <ahothan@cisco.com>

diff --git a/docs/development/design/ndrpdr.rst b/docs/development/design/ndrpdr.rst
index 4f611a0..5361174 100644 (file)
--- a/docs/development/design/ndrpdr.rst
+++ b/docs/development/design/ndrpdr.rst
@@ -28,7 +28,17 @@ The default value of 0.1 indicates for example that the measured NDR and PDR are
 actual NDR/PDR (e.g. 0.1% of 10Gbps is 10Mbps). It also determines how small the search range must be in the binary search.
 
 The recursion narrows down the range by half and stops when:
 actual NDR/PDR (e.g. 0.1% of 10Gbps is 10Mbps). It also determines how small the search range must be in the binary search.
 
 The recursion narrows down the range by half and stops when:

+

 - the range is smaller than the configured load_epsilon value
 - or when the search hits 100% or 0% of line rate
 
 - the range is smaller than the configured load_epsilon value
 - or when the search hits 100% or 0% of line rate
 

+One particularity of using a software traffic generator is that the requested Tx rate may not always be met due to
+resource limitations (e.g. CPU is not fast enough to generate a very high load). The algorithm should take this into
+consideration:
+
+- always monitor the actual Tx rate achieved
+- actual Tx rate is always <= requested Tx rate
+- the measured drop rate should always be relative to the actual Tx rate
+- if the actual Tx rate is < requested Tx rate and the measured drop rate is already within threshold (<NDR/PDR threshold) then the binary search must stop with proper warning
+

 
 

diff --git a/nfvbench/chain_clients.py b/nfvbench/chain_clients.py
index 57b15ee..fa21359 100644 (file)
--- a/nfvbench/chain_clients.py
+++ b/nfvbench/chain_clients.py
@@ -18,13 +18,12 @@ import os
 import re
 import time
 
 import re
 import time
 

-import compute
-from log import LOG
-

 from glanceclient.v2 import client as glanceclient
 from neutronclient.neutron import client as neutronclient
 from novaclient.client import Client
 
 from glanceclient.v2 import client as glanceclient
 from neutronclient.neutron import client as neutronclient
 from novaclient.client import Client
 

+import compute
+from log import LOG

 
 class StageClientException(Exception):
     pass
 
 class StageClientException(Exception):
     pass


@@ -109,7 +108,7 @@ class BasicStageClient(object):
                                                        phys1=network['provider:physical_network'],
                                                        phys2=physical_network))
 
                                                        phys1=network['provider:physical_network'],
                                                        phys2=physical_network))
 

-            LOG.info('Reusing existing network: ' + name)
+            LOG.info('Reusing existing network: %s', name)

             network['is_reuse'] = True
             return network
 
             network['is_reuse'] = True
             return network
 

diff --git a/nfvbench/chain_managers.py b/nfvbench/chain_managers.py
index cbc53e2..8b605aa 100644 (file)
--- a/nfvbench/chain_managers.py
+++ b/nfvbench/chain_managers.py
@@ -111,7 +111,7 @@ class StatsManager(object):
 
     def _generate_traffic(self):
         if self.config.no_traffic:
 
     def _generate_traffic(self):
         if self.config.no_traffic:

-            return
+            return {}

 
         self.interval_collector = IntervalCollector(time.time())
         self.interval_collector.attach_notifier(self.notifier)
 
         self.interval_collector = IntervalCollector(time.time())
         self.interval_collector.attach_notifier(self.notifier)

diff --git a/nfvbench/compute.py b/nfvbench/compute.py
index 575744c..af1a0d6 100644 (file)
--- a/nfvbench/compute.py
+++ b/nfvbench/compute.py
@@ -16,16 +16,15 @@ import os
 import time
 import traceback
 
 import time
 import traceback
 

-import keystoneauth1
-from log import LOG
-import novaclient
-

 from glanceclient import exc as glance_exception
 from glanceclient import exc as glance_exception

-

 try:
     from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound
 except ImportError:
     from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound
 try:
     from glanceclient.openstack.common.apiclient.exceptions import NotFound as GlanceImageNotFound
 except ImportError:
     from glanceclient.v1.apiclient.exceptions import NotFound as GlanceImageNotFound

+import keystoneauth1
+import novaclient
+
+from log import LOG

 
 
 class Compute(object):
 
 
 class Compute(object):


@@ -75,7 +74,7 @@ class Compute(object):
                       "image at the specified location %s is correct.", image_file)
             return False
         except keystoneauth1.exceptions.http.NotFound as exc:
                       "image at the specified location %s is correct.", image_file)
             return False
         except keystoneauth1.exceptions.http.NotFound as exc:

-            LOG.error("Authentication error while uploading the image:" + str(exc))
+            LOG.error("Authentication error while uploading the image: %s", str(exc))

             return False
         except Exception:
             LOG.error(traceback.format_exc())
             return False
         except Exception:
             LOG.error(traceback.format_exc())


@@ -258,7 +257,7 @@ class Compute(object):
                 if hyp.host == host:
                     return self.normalize_az_host(hyp.zone, host)
             # no match on host
                 if hyp.host == host:
                     return self.normalize_az_host(hyp.zone, host)
             # no match on host

-            LOG.error('Passed host name does not exist: ' + host)
+            LOG.error('Passed host name does not exist: %s', host)

             return None
         if self.config.availability_zone:
             return self.normalize_az_host(None, host)
             return None
         if self.config.availability_zone:
             return self.normalize_az_host(None, host)


@@ -290,7 +289,7 @@ class Compute(object):
                         return az_host
                         # else continue - another zone with same host name?
             # no match
                         return az_host
                         # else continue - another zone with same host name?
             # no match

-            LOG.error('No match for availability zone and host ' + az_host)
+            LOG.error('No match for availability zone and host %s', az_host)

             return None
         else:
             return self.auto_fill_az(host_list, az_host)
             return None
         else:
             return self.auto_fill_az(host_list, az_host)


@@ -348,7 +347,7 @@ class Compute(object):
             if not self.config.availability_zone:
                 LOG.error('Availability_zone must be configured')
             elif host_list:
             if not self.config.availability_zone:
                 LOG.error('Availability_zone must be configured')
             elif host_list:

-                LOG.error('No host matching the selection for availability zone: ' +
+                LOG.error('No host matching the selection for availability zone: %s',

                           self.config.availability_zone)
                 avail_list = []
             else:
                           self.config.availability_zone)
                 avail_list = []
             else:

diff --git a/nfvbench/config.py b/nfvbench/config.py
index 8139389..5feeda5 100644 (file)
--- a/nfvbench/config.py
+++ b/nfvbench/config.py
@@ -14,9 +14,9 @@
 #
 
 from attrdict import AttrDict
 #
 
 from attrdict import AttrDict

-from log import LOG

 import yaml
 
 import yaml
 

+from log import LOG

 
 def config_load(file_name, from_cfg=None, whitelist_keys=None):
     """Load a yaml file into a config dict, merge with from_cfg if not None
 
 def config_load(file_name, from_cfg=None, whitelist_keys=None):
     """Load a yaml file into a config dict, merge with from_cfg if not None

diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index 4c9f56c..6f59e24 100644 (file)
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -481,11 +481,11 @@ def main():
             # override default config options with start config at path parsed from CLI
             # check if it is an inline yaml/json config or a file name
             if os.path.isfile(opts.config):
             # override default config options with start config at path parsed from CLI
             # check if it is an inline yaml/json config or a file name
             if os.path.isfile(opts.config):

-                LOG.info('Loading configuration file: ' + opts.config)
+                LOG.info('Loading configuration file: %s', opts.config)

                 config = config_load(opts.config, config, whitelist_keys)
                 config.name = os.path.basename(opts.config)
             else:
                 config = config_load(opts.config, config, whitelist_keys)
                 config.name = os.path.basename(opts.config)
             else:

-                LOG.info('Loading configuration string: ' + opts.config)
+                LOG.info('Loading configuration string: %s', opts.config)

                 config = config_loads(opts.config, config, whitelist_keys)
 
         # traffic profile override options
                 config = config_loads(opts.config, config, whitelist_keys)
 
         # traffic profile override options

diff --git a/nfvbench/summarizer.py b/nfvbench/summarizer.py
index 70ad389..1676e93 100644 (file)
--- a/nfvbench/summarizer.py
+++ b/nfvbench/summarizer.py
@@ -20,9 +20,10 @@ import math
 
 import bitmath
 import pytz
 
 import bitmath
 import pytz

-from specs import ChainType

 from tabulate import tabulate
 
 from tabulate import tabulate
 

+from specs import ChainType
+

 
 class Formatter(object):
     """Collection of string formatter methods"""
 
 class Formatter(object):
     """Collection of string formatter methods"""

diff --git a/nfvbench/traffic_client.py b/nfvbench/traffic_client.py
index a1c4954..8959cab 100644 (file)
--- a/nfvbench/traffic_client.py
+++ b/nfvbench/traffic_client.py
@@ -67,6 +67,9 @@ class TrafficRunner(object):
     def poll_stats(self):
         if not self.is_running():
             return None
     def poll_stats(self):
         if not self.is_running():
             return None

+        if self.client.skip_sleep:
+            self.stop()
+            return self.client.get_stats()

         time_elapsed = self.time_elapsed()
         if time_elapsed > self.duration_sec:
             self.stop()
         time_elapsed = self.time_elapsed()
         if time_elapsed > self.duration_sec:
             self.stop()


@@ -102,10 +105,10 @@ class IpBlock(object):
         '''Reserve a range of count consecutive IP addresses spaced by step
         '''
         if self.next_free + count > self.max_available:
         '''Reserve a range of count consecutive IP addresses spaced by step
         '''
         if self.next_free + count > self.max_available:

-            raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d',
-                             self.next_free,
-                             self.max_available,
-                             count)
+            raise IndexError('No more IP addresses next free=%d max_available=%d requested=%d' %
+                             (self.next_free,
+                              self.max_available,
+                              count))

         first_ip = self.get_ip(self.next_free)
         last_ip = self.get_ip(self.next_free + count - 1)
         self.next_free += count
         first_ip = self.get_ip(self.next_free)
         last_ip = self.get_ip(self.next_free + count - 1)
         self.next_free += count


@@ -393,7 +396,7 @@ class TrafficGeneratorFactory(object):
 class TrafficClient(object):
     PORTS = [0, 1]
 
 class TrafficClient(object):
     PORTS = [0, 1]
 

-    def __init__(self, config, notifier=None):
+    def __init__(self, config, notifier=None, skip_sleep=False):

         generator_factory = TrafficGeneratorFactory(config)
         self.gen = generator_factory.get_generator_client()
         self.tool = generator_factory.get_tool()
         generator_factory = TrafficGeneratorFactory(config)
         self.gen = generator_factory.get_generator_client()
         self.tool = generator_factory.get_tool()


@@ -414,6 +417,8 @@ class TrafficClient(object):
         self.current_total_rate = {'rate_percent': '10'}
         if self.config.single_run:
             self.current_total_rate = utils.parse_rate_str(self.config.rate)
         self.current_total_rate = {'rate_percent': '10'}
         if self.config.single_run:
             self.current_total_rate = utils.parse_rate_str(self.config.rate)

+        # UT with dummy TG can bypass all sleeps
+        self.skip_sleep = skip_sleep

 
     def set_macs(self):
         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):
 
     def set_macs(self):
         for mac, device in zip(self.gen.get_macs(), self.config.generator_config.devices):


@@ -461,7 +466,8 @@ class TrafficClient(object):
             self.gen.clear_stats()
             self.gen.start_traffic()
             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)
             self.gen.clear_stats()
             self.gen.start_traffic()
             LOG.info('Waiting for packets to be received back... (%d / %d)', it + 1, retry_count)

-            time.sleep(self.config.generic_poll_sec)
+            if not self.skip_sleep:
+                time.sleep(self.config.generic_poll_sec)

             self.gen.stop_traffic()
             stats = self.gen.get_stats()
 
             self.gen.stop_traffic()
             stats = self.gen.get_stats()
 


@@ -481,7 +487,8 @@ class TrafficClient(object):
                 LOG.info('End-to-end connectivity ensured')
                 return
 
                 LOG.info('End-to-end connectivity ensured')
                 return
 

-            time.sleep(self.config.generic_poll_sec)
+            if not self.skip_sleep:
+                time.sleep(self.config.generic_poll_sec)

 
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 
 
         raise TrafficClientException('End-to-end connectivity cannot be ensured')
 

diff --git a/nfvbench/traffic_gen/dummy.py b/nfvbench/traffic_gen/dummy.py
index d8c01e9..b43030f 100644 (file)
--- a/nfvbench/traffic_gen/dummy.py
+++ b/nfvbench/traffic_gen/dummy.py
@@ -13,6 +13,7 @@
 #    under the License.
 
 from traffic_base import AbstractTrafficGenerator
 #    under the License.
 
 from traffic_base import AbstractTrafficGenerator

+import traffic_utils as utils

 
 
 class DummyTG(AbstractTrafficGenerator):
 
 
 class DummyTG(AbstractTrafficGenerator):


@@ -22,10 +23,13 @@ class DummyTG(AbstractTrafficGenerator):
     Useful for unit testing without actually generating any traffic.
     """
 
     Useful for unit testing without actually generating any traffic.
     """
 

-    def __init__(self, runner):
-        AbstractTrafficGenerator.__init__(self, runner)
+    def __init__(self, config):
+        AbstractTrafficGenerator.__init__(self, config)

         self.port_handle = []
         self.rates = []
         self.port_handle = []
         self.rates = []

+        self.l2_frame_size = 0
+        self.duration_sec = self.config.duration_sec
+        self.intf_speed = config.generator_config.intf_speed

 
     def get_version(self):
         return "0.1"
 
     def get_version(self):
         return "0.1"


@@ -33,6 +37,59 @@ class DummyTG(AbstractTrafficGenerator):
     def init(self):
         pass
 
     def init(self):
         pass
 

+    def get_tx_pps_dropped_pps(self, tx_rate):
+        '''Get actual tx packets based on requested tx rate
+
+        :param tx_rate: requested TX rate with unit ('40%', '1Mbps', '1000pps')
+
+        :return: the actual TX pps and the dropped pps corresponding to the requested TX rate
+        '''
+        dr, tx = self.__get_dr_actual_tx(tx_rate)
+        actual_tx_bps = utils.load_to_bps(tx, self.intf_speed)
+        avg_packet_size = utils.get_average_packet_size(self.l2_frame_size)
+        tx_packets = utils.bps_to_pps(actual_tx_bps, avg_packet_size)
+
+        dropped = tx_packets * dr / 100
+        # print '===get_tx_pkts_dropped_pkts req tex=', tx_rate, 'dr=', dr,
+        # 'actual tx rate=', tx, 'actual tx pkts=', tx_packets, 'dropped=', dropped
+        return int(tx_packets), int(dropped)
+
+    def set_response_curve(self, lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100):
+        '''Set traffic gen response characteristics
+
+        Specifies the drop rate curve and the actual TX curve
+        :param float lr_dr: The actual drop rate at TX line rate (in %, 0..100)
+        :param float ndr: The true NDR  (0 packet drop) in % (0..100) of line rate"
+        :param float max_actual_tx: highest actual TX when requested TX is 100%
+        :param float max_11_tx: highest requested TX that results in same actual TX
+        '''
+        self.target_ndr = ndr
+        if ndr < 100:
+            self.dr_slope = float(lr_dr) / (100 - ndr)
+        else:
+            self.dr_slope = 0
+        self.max_11_tx = max_11_tx
+        self.max_actual_tx = max_actual_tx
+        if max_11_tx < 100:
+            self.tx_slope = float(max_actual_tx - max_11_tx) / (100 - max_11_tx)
+        else:
+            self.tx_slope = 0
+
+    def __get_dr_actual_tx(self, requested_tx_rate):
+        '''Get drop rate at given requested tx rate
+        :param float requested_tx_rate: requested tx rate in % (0..100)
+        :return: the drop rate and actual tx rate at that requested_tx_rate in % (0..100)
+        '''
+        if requested_tx_rate <= self.max_11_tx:
+            actual_tx = requested_tx_rate
+        else:
+            actual_tx = self.max_11_tx + (requested_tx_rate - self.max_11_tx) * self.tx_slope
+        if actual_tx <= self.target_ndr:
+            dr = 0.0
+        else:
+            dr = (actual_tx - self.target_ndr) * self.dr_slope
+        return dr, actual_tx
+

     def connect(self):
         ports = list(self.config.generator_config.ports)
         self.port_handle = ports
     def connect(self):
         ports = list(self.config.generator_config.ports)
         self.port_handle = ports


@@ -44,32 +101,57 @@ class DummyTG(AbstractTrafficGenerator):
         pass
 
     def create_traffic(self, l2frame_size, rates, bidirectional, latency=True):
         pass
 
     def create_traffic(self, l2frame_size, rates, bidirectional, latency=True):

-        pass
+        self.rates = [utils.to_rate_str(rate) for rate in rates]
+        self.l2_frame_size = l2frame_size

 
     def clear_streamblock(self):
         pass
 
     def get_stats(self):
 
     def clear_streamblock(self):
         pass
 
     def get_stats(self):

+        '''Get stats from current run.
+
+        The binary search mainly looks at 2 results to make the decision:
+            actual tx packets
+            actual rx dropped packets
+        From the Requested TX rate - we get the Actual TX rate and the RX drop rate
+        From the Run duration and actual TX rate - we get the actual total tx packets
+        From the Actual tx packets and RX drop rate - we get the RX dropped packets
+        '''

         result = {}
         result = {}

-        for ph in self.port_handle:
+        total_tx_pps = 0
+
+        # use dummy values for all other result field as the goal is to
+        # test the ndr/pdr convergence code
+        for idx, ph in enumerate(self.port_handle):
+            requested_tx_rate = utils.get_load_from_rate(self.rates[idx])
+            tx_pps, dropped_pps = self.get_tx_pps_dropped_pps(requested_tx_rate)
+
+            # total packets sent per direction - used by binary search
+            total_pkts = tx_pps * self.duration_sec
+            dropped_pkts = dropped_pps * self.duration_sec
+            _, tx_pkt_rate = self.__get_dr_actual_tx(requested_tx_rate)

             result[ph] = {
                 'tx': {
             result[ph] = {
                 'tx': {

-                    'total_pkts': 1000,
+                    'total_pkts': total_pkts,

                     'total_pkt_bytes': 100000,
                     'total_pkt_bytes': 100000,

-                    'pkt_rate': 100,
+                    'pkt_rate': tx_pkt_rate,

                     'pkt_bit_rate': 1000000
                 },
                 'rx': {
                     'pkt_bit_rate': 1000000
                 },
                 'rx': {

-                    'total_pkts': 1000,
+                    # total packets received
+                    'total_pkts': total_pkts - dropped_pkts,

                     'total_pkt_bytes': 100000,
                     'pkt_rate': 100,
                     'pkt_bit_rate': 1000000,
                     'total_pkt_bytes': 100000,
                     'pkt_rate': 100,
                     'pkt_bit_rate': 1000000,

-                    'dropped_pkts': 0
+                    'dropped_pkts': dropped_pkts

                 }
             }
             result[ph]['rx']['max_delay_usec'] = 10.0
             result[ph]['rx']['min_delay_usec'] = 1.0
             result[ph]['rx']['avg_delay_usec'] = 2.0
                 }
             }
             result[ph]['rx']['max_delay_usec'] = 10.0
             result[ph]['rx']['min_delay_usec'] = 1.0
             result[ph]['rx']['avg_delay_usec'] = 2.0

+            total_tx_pps += tx_pps
+        # actual total tx rate in pps
+        result['total_tx_rate'] = total_tx_pps

         return result
 
     def clear_stats(self):
         return result
 
     def clear_stats(self):

diff --git a/nfvbench/traffic_gen/traffic_utils.py b/nfvbench/traffic_gen/traffic_utils.py
index e618c28..4a7f855 100644 (file)
--- a/nfvbench/traffic_gen/traffic_utils.py
+++ b/nfvbench/traffic_gen/traffic_utils.py
@@ -75,6 +75,13 @@ def weighted_avg(weight, count):
         return sum([x[0] * x[1] for x in zip(weight, count)]) / sum(weight)
     return float('nan')
 
         return sum([x[0] * x[1] for x in zip(weight, count)]) / sum(weight)
     return float('nan')
 

+def _get_bitmath_rate(rate_bps):
+    rate = rate_bps.replace('ps', '').strip()
+    bitmath_rate = bitmath.parse_string(rate)
+    if bitmath_rate.bits <= 0:
+        raise Exception('%s is out of valid range' % rate_bps)
+    return bitmath_rate
+

 def parse_rate_str(rate_str):
     if rate_str.endswith('pps'):
         rate_pps = rate_str[:-3]
 def parse_rate_str(rate_str):
     if rate_str.endswith('pps'):
         rate_pps = rate_str[:-3]


@@ -103,6 +110,26 @@ def parse_rate_str(rate_str):
     else:
         raise Exception('Unknown rate string format %s' % rate_str)
 
     else:
         raise Exception('Unknown rate string format %s' % rate_str)
 

+def get_load_from_rate(rate_str, avg_frame_size=64, line_rate='10Gbps'):
+    '''From any rate string (with unit) return the corresponding load (in % unit)
+
+    :param str rate_str: the rate to convert - must end with a unit (e.g. 1Mpps, 30%, 1Gbps)
+    :param int avg_frame_size: average frame size in bytes (needed only if pps is given)
+    :param str line_rate: line rate ending with bps unit (e.g. 1Mbps, 10Gbps) is the rate that
+                      corresponds to 100% rate
+    :return float: the corresponding rate in % of line rate
+    '''
+    rate_dict = parse_rate_str(rate_str)
+    if 'rate_percent' in rate_dict:
+        return float(rate_dict['rate_percent'])
+    lr_bps = _get_bitmath_rate(line_rate).bits
+    if 'rate_bps' in rate_dict:
+        bps = int(rate_dict['rate_bps'])
+    else:
+        # must be rate_pps
+        pps = rate_dict['rate_pps']
+        bps = pps_to_bps(pps, avg_frame_size)
+    return bps_to_load(bps, lr_bps)

 
 def divide_rate(rate, divisor):
     if 'rate_pps' in rate:
 
 def divide_rate(rate, divisor):
     if 'rate_pps' in rate:


@@ -130,8 +157,9 @@ def to_rate_str(rate):
     elif 'rate_percent' in rate:
         load = rate['rate_percent']
         return '{}%'.format(load)
     elif 'rate_percent' in rate:
         load = rate['rate_percent']
         return '{}%'.format(load)

-    else:
-        assert False
+    assert False
+    # avert pylint warning
+    return None

 
 
 def nan_replace(d):
 
 
 def nan_replace(d):

diff --git a/nfvbench/utils.py b/nfvbench/utils.py
index 412dfae..20dc588 100644 (file)
--- a/nfvbench/utils.py
+++ b/nfvbench/utils.py
@@ -61,7 +61,7 @@ def save_json_result(result, json_file, std_json_path, service_chain, service_ch
 
     if filepaths:
         for file_path in filepaths:
 
     if filepaths:
         for file_path in filepaths:

-            LOG.info('Saving results in json file: ' + file_path + "...")
+            LOG.info('Saving results in json file: %s...', file_path)

             with open(file_path, 'w') as jfp:
                 json.dump(result,
                           jfp,
             with open(file_path, 'w') as jfp:
                 json.dump(result,
                           jfp,

diff --git a/test/test_nfvbench.py b/test/test_nfvbench.py
index 2578407..05490e7 100644 (file)
--- a/test/test_nfvbench.py
+++ b/test/test_nfvbench.py
@@ -18,6 +18,8 @@ import logging
 import os
 import sys
 
 import os
 import sys
 

+import pytest
+

 from attrdict import AttrDict
 from nfvbench.config import config_loads
 from nfvbench.credentials import Credentials
 from attrdict import AttrDict
 from nfvbench.config import config_loads
 from nfvbench.credentials import Credentials


@@ -28,7 +30,6 @@ from nfvbench.network import Network
 from nfvbench.specs import ChainType
 from nfvbench.specs import Encaps
 import nfvbench.traffic_gen.traffic_utils as traffic_utils
 from nfvbench.specs import ChainType
 from nfvbench.specs import Encaps
 import nfvbench.traffic_gen.traffic_utils as traffic_utils

-import pytest

 
 __location__ = os.path.realpath(os.path.join(os.getcwd(),
                                              os.path.dirname(__file__)))
 
 __location__ = os.path.realpath(os.path.join(os.getcwd(),
                                              os.path.dirname(__file__)))


@@ -266,183 +267,6 @@ def test_pvp_chain_run(pvp_chain):
     assert result == expected_result
 """
 
     assert result == expected_result
 """
 

-# =========================================================================
-# PVVP Chain tests
-# =========================================================================
-
-"""
-@pytest.fixture
-def pvvp_chain(monkeypatch, openstack_vxlan_spec):
-    tor_vni1 = Interface('vni-4097', 'n9k', 50, 77)
-    vsw_vni1 = Interface('vxlan_tunnel0', 'vpp', 77, 48)
-    vsw_vif1 = Interface('VirtualEthernet0/0/2', 'vpp', 48, 77)
-    vsw_vif3 = Interface('VirtualEthernet0/0/0', 'vpp', 77, 47)
-    vsw_vif4 = Interface('VirtualEthernet0/0/1', 'vpp', 45, 77)
-    vsw_vif2 = Interface('VirtualEthernet0/0/3', 'vpp', 77, 44)
-    vsw_vni2 = Interface('vxlan_tunnel1', 'vpp', 43, 77)
-    tor_vni2 = Interface('vni-4098', 'n9k', 77, 40)
-
-    def mock_init(self, *args, **kwargs):
-        self.vni_ports = [4099, 4100]
-        self.v2vnet = V2VNetwork()
-        self.specs = openstack_vxlan_spec
-        self.clients = {
-            'vpp': AttrDict({
-                'get_v2v_network': lambda reverse=None: Network([vsw_vif3, vsw_vif4], reverse),
-                'set_interface_counters': lambda pvvp=None: None,
-                'set_v2v_counters': lambda: None,
-            })
-        }
-        self.worker = AttrDict({
-            'run': lambda: None,
-        })
-
-    def mock_empty(self, *args, **kwargs):
-        pass
-
-    def mock_get_network(self, traffic_port, vni_id, reverse=False):
-        if vni_id == 0:
-            return Network([tor_vni1, vsw_vni1, vsw_vif1], reverse)
-        else:
-            return Network([tor_vni2, vsw_vni2, vsw_vif2], reverse)
-
-    def mock_get_data(self):
-        return {}
-
-    monkeypatch.setattr(PVVPChain, '_get_network', mock_get_network)
-    monkeypatch.setattr(PVVPChain, '_get_data', mock_get_data)
-    monkeypatch.setattr(PVVPChain, '_setup', mock_empty)
-    monkeypatch.setattr(VxLANWorker, '_clear_interfaces', mock_empty)
-    monkeypatch.setattr(PVVPChain, '_generate_traffic', mock_empty)
-    monkeypatch.setattr(PVVPChain, '__init__', mock_init)
-
-    return PVVPChain(None, None, {'vm': None, 'vpp': None, 'tor': None, 'traffic': None}, None)
-
-
-def test_pvvp_chain_run(pvvp_chain):
-    result = pvvp_chain.run()
-
-    expected_result = {
-        'raw_data': {},
-        'stats': None,
-        'packet_analysis':
-            {'direction-forward': [
-                OrderedDict([
-                    ('interface', 'vni-4097'),
-                    ('device', 'n9k'),
-                    ('packet_count', 50)
-                ]),
-                OrderedDict([
-                    ('interface', 'vxlan_tunnel0'),
-                    ('device', 'vpp'),
-                    ('packet_count', 48),
-                    ('packet_drop_count', 2),
-                    ('packet_drop_percentage', 4.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/2'),
-                    ('device', 'vpp'),
-                    ('packet_count', 48),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/0'),
-                    ('device', 'vpp'),
-                    ('packet_count', 47),
-                    ('packet_drop_count', 1),
-                    ('packet_drop_percentage', 2.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/1'),
-                    ('device', 'vpp'),
-                    ('packet_count', 45),
-                    ('packet_drop_count', 2),
-                    ('packet_drop_percentage', 4.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/3'),
-                    ('device', 'vpp'),
-                    ('packet_count', 44),
-                    ('packet_drop_count', 1),
-                    ('packet_drop_percentage', 2.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'vxlan_tunnel1'),
-                    ('device', 'vpp'),
-                    ('packet_count', 43),
-                    ('packet_drop_count', 1),
-                    ('packet_drop_percentage', 2.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'vni-4098'),
-                    ('device', 'n9k'),
-                    ('packet_count', 40),
-                    ('packet_drop_count', 3),
-                    ('packet_drop_percentage', 6.0)
-                ])
-            ],
-            'direction-reverse': [
-                OrderedDict([
-                    ('interface', 'vni-4098'),
-                    ('device', 'n9k'),
-                    ('packet_count', 77)
-                ]),
-                OrderedDict([
-                    ('interface', 'vxlan_tunnel1'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/3'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/1'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/0'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'VirtualEthernet0/0/2'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'vxlan_tunnel0'),
-                    ('device', 'vpp'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ]),
-                OrderedDict([
-                    ('interface', 'vni-4097'),
-                    ('device', 'n9k'),
-                    ('packet_count', 77),
-                    ('packet_drop_count', 0),
-                    ('packet_drop_percentage', 0.0)
-                ])
-            ]}
-    }
-    assert result == expected_result
-"""
-
-

 # =========================================================================
 # Traffic client tests
 # =========================================================================
 # =========================================================================
 # Traffic client tests
 # =========================================================================


@@ -473,7 +297,7 @@ def test_parse_rate_str():
         except Exception:
             return True
         else:
         except Exception:
             return True
         else:

-            assert False
+            return False

 
     assert should_raise_error('101')
     assert should_raise_error('201%')
 
     assert should_raise_error('101')
     assert should_raise_error('201%')


@@ -500,6 +324,38 @@ def test_rate_conversion():
     assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
     assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
 
     assert traffic_utils.pps_to_bps(31.6066319896, 1518) == pytest.approx(388888)
     assert traffic_utils.pps_to_bps(3225895.85831, 340.3) == pytest.approx(9298322222)
 

+# pps at 10Gbps line rate for 64 byte frames
+LR_64B_PPS = 14880952
+LR_1518B_PPS = 812743
+
+def assert_equivalence(reference, value, allowance_pct=1):
+    '''Asserts if a value is equivalent to a reference value with given margin
+
+    :param float reference: reference value to compare to
+    :param float value: value to compare to reference
+    :param float allowance_pct: max allowed percentage of margin
+        0 : requires exact match
+        1 : must be equal within 1% of the reference value
+        ...
+        100: always true
+    '''
+    if reference == 0:
+        assert value == 0
+    else:
+        assert abs(value - reference) * 100 / reference <= allowance_pct
+
+def test_load_from_rate():
+    assert traffic_utils.get_load_from_rate('100%') == 100
+    assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_64B_PPS) + 'pps'))
+    assert_equivalence(50, traffic_utils.get_load_from_rate(str(LR_64B_PPS / 2) + 'pps'))
+    assert_equivalence(100, traffic_utils.get_load_from_rate('10Gbps'))
+    assert_equivalence(50, traffic_utils.get_load_from_rate('5000Mbps'))
+    assert_equivalence(1, traffic_utils.get_load_from_rate('100Mbps'))
+    assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS) + 'pps',
+                                                             avg_frame_size=1518))
+    assert_equivalence(100, traffic_utils.get_load_from_rate(str(LR_1518B_PPS * 2) + 'pps',
+                                                             avg_frame_size=1518,
+                                                             line_rate='20Gbps'))

 
 """
 @pytest.fixture
 
 """
 @pytest.fixture


@@ -513,112 +369,14 @@ def traffic_client(monkeypatch):
             'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
         }
 
             'rates': [{'rate_percent': '10'}, {'rate_pps': '1'}]
         }
 

-        self.config = AttrDict({
-            'generator_config': {
-                'intf_speed': 10000000000
-            },
-            'ndr_run': True,
-            'pdr_run': True,
-            'single_run': False,
-            'attempts': 1,
-            'measurement': {
-                'NDR': 0.0,
-                'PDR': 0.1,
-                'load_epsilon': 0.1
-            }
-        })
-
-        self.runner = AttrDict({
-            'time_elapsed': lambda: 30,
-            'stop': lambda: None,
-            'client': AttrDict({'get_stats': lambda: None})
-        })
-
-        self.current_load = None
-        self.dummy_stats = {
-            50.0: 72.6433562831,
-            25.0: 45.6095059858,
-            12.5: 0.0,
-            18.75: 27.218642979,
-            15.625: 12.68585861,
-            14.0625: 2.47154392563,
-            13.28125: 0.000663797066801,
-            12.890625: 0.0,
-            13.0859375: 0.0,
-            13.18359375: 0.00359387347122,
-            13.671875: 0.307939922531,
-            13.4765625: 0.0207718516156,
-            13.57421875: 0.0661795060969
-        }
-

     def mock_modify_load(self, load):
         self.run_config['rates'][0] = {'rate_percent': str(load)}
         self.current_load = load
 
     def mock_modify_load(self, load):
         self.run_config['rates'][0] = {'rate_percent': str(load)}
         self.current_load = load
 

-    def mock_run_traffic(self):
-        yield {
-            'overall': {
-                'drop_rate_percent': self.dummy_stats[self.current_load],
-                'rx': {
-                    'total_pkts': 1,
-                    'avg_delay_usec': 0.0,
-                    'max_delay_usec': 0.0,
-                    'min_delay_usec': 0.0
-                }
-            }
-        }
-

     monkeypatch.setattr(TrafficClient, '__init__', mock_init)
     monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)
     monkeypatch.setattr(TrafficClient, '__init__', mock_init)
     monkeypatch.setattr(TrafficClient, 'modify_load', mock_modify_load)

-    monkeypatch.setattr(TrafficClient, 'run_traffic', mock_run_traffic)

 
     return TrafficClient()
 
     return TrafficClient()

-
-
-def test_ndr_pdr_search(traffic_client):
-    expected_results = {
-        'pdr': {
-            'l2frame_size': '64',
-            'initial_rate_type': 'rate_percent',
-            'stats': {
-                'overall': {
-                    'drop_rate_percent': 0.0661795060969,
-                    'min_delay_usec': 0.0,
-                    'avg_delay_usec': 0.0,
-                    'max_delay_usec': 0.0
-                }
-            },
-            'load_percent_per_direction': 13.57421875,
-            'rate_percent': 13.57422547,
-            'rate_bps': 1357422547.0,
-            'rate_pps': 2019974.0282738095,
-            'duration_sec': 30
-        },
-        'ndr': {
-            'l2frame_size': '64',
-            'initial_rate_type': 'rate_percent',
-            'stats': {
-                'overall': {
-                    'drop_rate_percent': 0.0,
-                    'min_delay_usec': 0.0,
-                    'avg_delay_usec': 0.0,
-                    'max_delay_usec': 0.0
-                }
-            },
-            'load_percent_per_direction': 13.0859375,
-            'rate_percent': 13.08594422,
-            'rate_bps': 1308594422.0,
-            'rate_pps': 1947313.1279761905,
-            'duration_sec': 30
-        }
-    }
-
-    results = traffic_client.get_ndr_and_pdr()
-    assert len(results) == 2
-    for result in results.values():
-        result.pop('timestamp_sec')
-        result.pop('time_taken_sec')
-    assert results == expected_results

 """
 
 
 """
 
 


@@ -631,7 +389,6 @@ def test_ndr_pdr_search(traffic_client):
 def setup_module(module):
     nfvbench.log.setup(mute_stdout=True)
 
 def setup_module(module):
     nfvbench.log.setup(mute_stdout=True)
 

-

 def test_no_credentials():
     cred = Credentials('/completely/wrong/path/openrc', None, False)
     if cred.rc_auth_url:
 def test_no_credentials():
     cred = Credentials('/completely/wrong/path/openrc', None, False)
     if cred.rc_auth_url:


@@ -667,7 +424,8 @@ except ImportError:
 # pylint: disable=wrong-import-position,ungrouped-imports
 from nfvbench.traffic_client import Device
 from nfvbench.traffic_client import IpBlock
 # pylint: disable=wrong-import-position,ungrouped-imports
 from nfvbench.traffic_client import Device
 from nfvbench.traffic_client import IpBlock

-
+from nfvbench.traffic_client import TrafficClient
+from nfvbench.traffic_client import TrafficGeneratorFactory

 
 # pylint: enable=wrong-import-position,ungrouped-imports
 
 
 # pylint: enable=wrong-import-position,ungrouped-imports
 


@@ -828,3 +586,95 @@ def test_fluentd():
         raise Exception("test")
     except Exception:
         logger.exception("got exception")
         raise Exception("test")
     except Exception:
         logger.exception("got exception")

+
+def assert_ndr_pdr(stats, ndr, ndr_dr, pdr, pdr_dr):
+    assert stats['ndr']['rate_percent'] == ndr
+    assert stats['ndr']['stats']['overall']['drop_percentage'] == ndr_dr
+    assert_equivalence(pdr, stats['pdr']['rate_percent'])
+    assert_equivalence(pdr_dr, stats['pdr']['stats']['overall']['drop_percentage'])
+
+def get_traffic_client():
+    config = AttrDict({
+        'traffic_generator': {'host_name': 'nfvbench_tg',
+                              'default_profile': 'dummy',
+                              'generator_profile': [{'name': 'dummy',
+                                                     'tool': 'dummy',
+                                                     'ip': '127.0.0.1',
+                                                     'intf_speed': '10Gbps',
+                                                     'interfaces': [{'port': 0, 'pci': 0},
+                                                                    {'port': 1, 'pci': 0}]}],
+                              'ip_addrs_step': '0.0.0.1',
+                              'ip_addrs': ['10.0.0.0/8', '20.0.0.0/8'],
+                              'tg_gateway_ip_addrs': ['1.1.0.100', '2.2.0.100'],
+                              'tg_gateway_ip_addrs_step': '0.0.0.1',
+                              'gateway_ip_addrs': ['1.1.0.2', '2.2.0.2'],
+                              'gateway_ip_addrs_step': '0.0.0.1',
+                              'udp_src_port': None,
+                              'udp_dst_port': None},
+        'generator_profile': 'dummy',
+        'service_chain': 'PVP',
+        'service_chain_count': 1,
+        'flow_count': 10,
+        'vlan_tagging': True,
+        'no_arp': False,
+        'duration_sec': 1,
+        'interval_sec': 1,
+        'single_run': False,
+        'ndr_run': True,
+        'pdr_run': True,
+        'rate': 'ndr_pdr',
+        'check_traffic_time_sec': 200,
+        'generic_poll_sec': 2,
+        'measurement': {'NDR': 0.001, 'PDR': 0.1, 'load_epsilon': 0.1},
+    })
+    generator_factory = TrafficGeneratorFactory(config)
+    config.generator_config = generator_factory.get_generator_config(config.generator_profile)
+    traffic_client = TrafficClient(config, skip_sleep=True)
+    traffic_client.start_traffic_generator()
+    traffic_client.set_traffic('64', True)
+    return traffic_client
+
+def test_ndr_at_lr():
+    traffic_client = get_traffic_client()
+    tg = traffic_client.gen
+
+    # this is a perfect sut with no loss at LR
+    tg.set_response_curve(lr_dr=0, ndr=100, max_actual_tx=100, max_11_tx=100)
+    # tx packets should be line rate for 64B and no drops...
+    assert tg.get_tx_pps_dropped_pps(100) == (LR_64B_PPS, 0)
+    # NDR and PDR should be at 100%
+    traffic_client.ensure_end_to_end()
+    results = traffic_client.get_ndr_and_pdr()
+
+    assert_ndr_pdr(results, 200.0, 0.0, 200.0, 0.0)
+
+def test_ndr_at_50():
+    traffic_client = get_traffic_client()
+    tg = traffic_client.gen
+    # this is a sut with an NDR of 50% and linear drop rate after NDR up to 20% drops at LR
+    # (meaning that if you send 100% TX, you will only receive 80% RX)
+    # the tg requested TX/actual TX ratio is 1up to 50%, after 50%
+    # is linear up 80% actuak TX when requesting 100%
+    tg.set_response_curve(lr_dr=20, ndr=50, max_actual_tx=80, max_11_tx=50)
+    # tx packets should be half line rate for 64B and no drops...
+    assert tg.get_tx_pps_dropped_pps(50) == (LR_64B_PPS / 2, 0)
+    # at 100% TX requested, actual TX is 80% where the drop rate is 3/5 of 20% of the actual TX
+    assert tg.get_tx_pps_dropped_pps(100) == (int(LR_64B_PPS * 0.8),
+                                              int(LR_64B_PPS * 0.8 * 0.6 * 0.2))
+    results = traffic_client.get_ndr_and_pdr()
+    assert_ndr_pdr(results, 100.0, 0.0, 100.781, 0.09374)
+
+def test_ndr_pdr_low_cpu():
+    traffic_client = get_traffic_client()
+    tg = traffic_client.gen
+    # This test is for the case where the TG is underpowered and cannot send fast enough for the NDR
+    # true NDR=40%, actual TX at 50% = 30%, actual measured DR is 0%
+    # The ndr/pdr should bail out with a warning and a best effort measured NDR of 30%
+    tg.set_response_curve(lr_dr=50, ndr=40, max_actual_tx=60, max_11_tx=0)
+    # tx packets should be 30% at requested half line rate for 64B and no drops...
+    assert tg.get_tx_pps_dropped_pps(50) == (int(LR_64B_PPS * 0.3), 0)
+    results = traffic_client.get_ndr_and_pdr()
+    assert results
+    # import pprint
+    # pp = pprint.PrettyPrinter(indent=4)
+    # pp.pprint(results)

diff --git a/tox.ini b/tox.ini
index 1dab8a7..5aa8997 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -10,7 +10,7 @@ setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt

-commands = py.test -q -s --basetemp={envtmpdir} {posargs}
+commands = py.test -q --basetemp={envtmpdir} {posargs}

 
 [testenv:pep8]
 commands = flake8 {toxinidir}
 
 [testenv:pep8]
 commands = flake8 {toxinidir}