X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=testcases%2Ftestcase.py;h=7f22c18f5af8f74a45658c397fd79165a71b93f0;hb=refs%2Fchanges%2F45%2F22945%2F1;hp=7d5162e6d4eec861dad4b9a585bcb64d72ebf6c4;hpb=a64311b5ba40d31e438732979eb97cc8e94e7a6e;p=vswitchperf.git diff --git a/testcases/testcase.py b/testcases/testcase.py index 7d5162e6..7f22c18f 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -14,29 +14,33 @@ """TestCase base class """ +from collections import OrderedDict +import copy import csv -import os import logging +import math +import os +import re +import time import subprocess -import copy -from collections import OrderedDict -from core.results.results_constants import ResultsConstants +from conf import settings as S +from conf import get_test_param import core.component_factory as component_factory from core.loader import Loader +from core.results.results_constants import ResultsConstants from tools import tasks from tools import hugepages -from tools.report import report -from conf import settings as S +from tools import functions from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS -from conf import get_test_param + class TestCase(object): """TestCase base class In this basic form runs RFC2544 throughput test """ - def __init__(self, cfg, results_dir): + def __init__(self, cfg): """Pull out fields from test config :param cfg: A dictionary of string-value pairs describing the test @@ -44,38 +48,78 @@ class TestCase(object): values. :param results_dir: Where the csv formatted results are written. """ + self._testcase_start_time = time.time() self._hugepages_mounted = False + self._traffic_ctl = None + self._vnf_ctl = None + self._vswitch_ctl = None + self._collector = None + self._loadgen = None + self._output_file = None + self._tc_results = None + self._settings_original = {} + self._settings_paths_modified = False + self._testcast_run_time = None + + # store all GUEST_ specific settings to keep original values before their expansion + for key in S.__dict__: + if key.startswith('GUEST_'): + self._settings_original[key] = S.getValue(key) + + self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) + self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF'))) + self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) + self._update_settings('TEST_PARAMS', cfg.get('Parameters', S.getValue('TEST_PARAMS'))) + + # update global settings + functions.settings_update_paths() + guest_loopback = get_test_param('guest_loopback', None) + if guest_loopback: + # we can put just one item, it'll be expanded automatically for all VMs + self._update_settings('GUEST_LOOPBACK', [guest_loopback]) + + # set test parameters; CLI options take precedence to testcase settings self._logger = logging.getLogger(__name__) self.name = cfg['Name'] self.desc = cfg.get('Description', 'No description given.') + self.test = cfg.get('TestSteps', None) + + bidirectional = cfg.get('biDirectional', TRAFFIC_DEFAULTS['bidir']) + bidirectional = get_test_param('bidirectional', bidirectional) + if not isinstance(bidirectional, str): + raise TypeError( + 'Bi-dir value must be of type string in testcase configuration') + bidirectional = bidirectional.title() # Keep things consistent + + traffic_type = cfg.get('Traffic Type', TRAFFIC_DEFAULTS['traffic_type']) + traffic_type = get_test_param('traffic_type', traffic_type) + + framerate = cfg.get('iLoad', TRAFFIC_DEFAULTS['frame_rate']) + framerate = get_test_param('iload', framerate) + self.deployment = cfg['Deployment'] self._frame_mod = cfg.get('Frame Modification', None) - framerate = get_test_param('iload', None) - if framerate == None: - framerate = cfg.get('iLoad', 100) - - # identify guest loopback method, so it can be added into reports - self.guest_loopback = [] - if self.deployment in ['pvp', 'pvvp']: - guest_loopback = get_test_param('guest_loopback', None) - if guest_loopback: - self.guest_loopback.append(guest_loopback) - else: - if self.deployment == 'pvp': - self.guest_loopback.append(S.getValue('GUEST_LOOPBACK')[0]) - else: - self.guest_loopback = S.getValue('GUEST_LOOPBACK').copy() + + self._tunnel_type = None + self._tunnel_operation = None + + if self.deployment == 'op2p': + self._tunnel_operation = cfg['Tunnel Operation'] + + if 'Tunnel Type' in cfg: + self._tunnel_type = cfg['Tunnel Type'] + self._tunnel_type = get_test_param('tunnel_type', + self._tunnel_type) # read configuration of streams; CLI parameter takes precedence to # testcase definition - multistream = cfg.get('MultiStream', 0) + multistream = cfg.get('MultiStream', TRAFFIC_DEFAULTS['multistream']) multistream = get_test_param('multistream', multistream) - stream_type = cfg.get('Stream Type', 'L4') + stream_type = cfg.get('Stream Type', TRAFFIC_DEFAULTS['stream_type']) stream_type = get_test_param('stream_type', stream_type) - pre_installed_flows = cfg.get('Pre-installed Flows', 'No') + pre_installed_flows = cfg.get('Pre-installed Flows', TRAFFIC_DEFAULTS['pre_installed_flows']) pre_installed_flows = get_test_param('pre-installed_flows', pre_installed_flows) - # check if test requires background load and which generator it uses self._load_cfg = cfg.get('Load', None) if self._load_cfg and 'tool' in self._load_cfg: @@ -86,95 +130,212 @@ class TestCase(object): if self._frame_mod: self._frame_mod = self._frame_mod.lower() - self._results_dir = results_dir + self._results_dir = S.getValue('RESULTS_PATH') # set traffic details, so they can be passed to vswitch and traffic ctls self._traffic = copy.deepcopy(TRAFFIC_DEFAULTS) - self._traffic.update({'traffic_type': cfg['Traffic Type'], - 'flow_type': cfg.get('Flow Type', 'port'), - 'bidir': cfg['biDirectional'], + self._traffic.update({'traffic_type': traffic_type, + 'flow_type': cfg.get('Flow Type', TRAFFIC_DEFAULTS['flow_type']), + 'bidir': bidirectional, + 'tunnel_type': self._tunnel_type, 'multistream': int(multistream), 'stream_type': stream_type, 'pre_installed_flows' : pre_installed_flows, 'frame_rate': int(framerate)}) - # OVS Vanilla requires guest VM MAC address and IPs to work - if 'linux_bridge' in self.guest_loopback: - self._traffic['l2'].update({'srcmac': S.getValue('GUEST_NET2_MAC')[0], - 'dstmac': S.getValue('GUEST_NET1_MAC')[0]}) - self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'), - 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')}) - # Packet Forwarding mode self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower() - def run(self): - """Run the test + # trafficgen configuration required for tests of tunneling protocols + if self.deployment == "op2p": + self._traffic['l2'].update({'srcmac': + S.getValue('TRAFFICGEN_PORT1_MAC'), + 'dstmac': + S.getValue('TRAFFICGEN_PORT2_MAC')}) + + self._traffic['l3'].update({'srcip': + S.getValue('TRAFFICGEN_PORT1_IP'), + 'dstip': + S.getValue('TRAFFICGEN_PORT2_IP')}) + + if self._tunnel_operation == "decapsulation": + self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2') + self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3') + self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4') + elif S.getValue('NICS')[0]['type'] == 'vf' or S.getValue('NICS')[1]['type'] == 'vf': + mac1 = S.getValue('NICS')[0]['mac'] + mac2 = S.getValue('NICS')[1]['mac'] + if mac1 and mac2: + self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1}) + else: + self._logger.debug("MAC addresses can not be read") - All setup and teardown through controllers is included. + def run_initialize(self): + """ Prepare test execution environment """ self._logger.debug(self.name) # mount hugepages if needed self._mount_hugepages() - # copy sources of l2 forwarding tools into VM shared dir if needed - self._copy_fwd_tools_for_guest() - self._logger.debug("Controllers:") loader = Loader() - traffic_ctl = component_factory.create_traffic( + self._traffic_ctl = component_factory.create_traffic( self._traffic['traffic_type'], loader.get_trafficgen_class()) - vnf_ctl = component_factory.create_vnf( + + self._vnf_ctl = component_factory.create_vnf( self.deployment, loader.get_vnf_class()) + # verify enough hugepages are free to run the testcase + if not self._check_for_enough_hugepages(): + raise RuntimeError('Not enough hugepages free to run test.') + + # perform guest related handling + if self._vnf_ctl.get_vnfs_number(): + # copy sources of l2 forwarding tools into VM shared dir if needed + self._copy_fwd_tools_for_all_guests(self._vnf_ctl.get_vnfs_number()) + + # in case of multi VM in parallel, set the number of streams to the number of VMs + if self.deployment.startswith('pvpv'): + # for each VM NIC pair we need an unique stream + streams = 0 + for vm_nic in S.getValue('GUEST_NICS_NR')[:self._vnf_ctl.get_vnfs_number()]: + streams += int(vm_nic / 2) if vm_nic > 1 else 1 + self._logger.debug("VMs with parallel connection were detected. " + "Thus Number of streams was set to %s", streams) + self._traffic.update({'multistream': streams}) + + # OVS Vanilla requires guest VM MAC address and IPs to work + if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'): + self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'), + 'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')}) + self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'), + 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')}) + if self._vswitch_none: - vswitch_ctl = component_factory.create_pktfwd( + self._vswitch_ctl = component_factory.create_pktfwd( + self.deployment, loader.get_pktfwd_class()) else: - vswitch_ctl = component_factory.create_vswitch( + self._vswitch_ctl = component_factory.create_vswitch( self.deployment, loader.get_vswitch_class(), - self._traffic) + self._traffic, + self._tunnel_operation) - collector = component_factory.create_collector( + self._collector = component_factory.create_collector( loader.get_collector_class(), self._results_dir, self.name) - loadgen = component_factory.create_loadgen( + self._loadgen = component_factory.create_loadgen( self._loadgen, self._load_cfg) - self._logger.debug("Setup:") - with vswitch_ctl, loadgen: - with vnf_ctl, collector: - if not self._vswitch_none: - self._add_flows(vswitch_ctl) - - with traffic_ctl: - traffic_ctl.send_traffic(self._traffic) + self._output_file = os.path.join(self._results_dir, "result_" + self.name + + "_" + self.deployment + ".csv") - # dump vswitch flows before they are affected by VNF termination - if not self._vswitch_none: - vswitch_ctl.dump_vswitch_flows() + self._logger.debug("Setup:") + def run_finalize(self): + """ Tear down test execution environment and record test results + """ # umount hugepages if mounted self._umount_hugepages() - self._logger.debug("Traffic Results:") - traffic_ctl.print_results() + # restore original settings + S.load_from_dict(self._settings_original) + + # cleanup any namespaces created + if os.path.isdir('/tmp/namespaces'): + import tools.namespace + namespace_list = os.listdir('/tmp/namespaces') + if len(namespace_list): + self._logger.info('Cleaning up namespaces') + for name in namespace_list: + tools.namespace.delete_namespace(name) + os.rmdir('/tmp/namespaces') + # cleanup any veth ports created + if os.path.isdir('/tmp/veth'): + import tools.veth + veth_list = os.listdir('/tmp/veth') + if len(veth_list): + self._logger.info('Cleaning up veth ports') + for eth in veth_list: + port1, port2 = eth.split('-') + tools.veth.del_veth_port(port1, port2) + os.rmdir('/tmp/veth') + + def run_report(self): + """ Report test results + """ + self._logger.debug("self._collector Results:") + self._collector.print_results() - self._logger.debug("Collector Results:") - collector.print_results() + if S.getValue('mode') != 'trafficgen-off': + self._logger.debug("Traffic Results:") + self._traffic_ctl.print_results() - output_file = os.path.join(self._results_dir, "result_" + self.name + - "_" + self.deployment + ".csv") + self._tc_results = self._append_results(self._traffic_ctl.get_results()) + TestCase.write_result_to_file(self._tc_results, self._output_file) - tc_results = self._append_results(traffic_ctl.get_results()) - TestCase._write_result_to_file(tc_results, output_file) + def run(self): + """Run the test - report.generate(output_file, tc_results, collector.get_results()) + All setup and teardown through controllers is included. + """ + # prepare test execution environment + self.run_initialize() + + with self._vswitch_ctl, self._loadgen: + with self._vnf_ctl, self._collector: + if not self._vswitch_none: + self._add_flows() + + # run traffic generator if requested, otherwise wait for manual termination + if S.getValue('mode') == 'trafficgen-off': + time.sleep(2) + self._logger.debug("All is set. Please run traffic generator manually.") + input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep) + else: + if S.getValue('mode') == 'trafficgen-pause': + time.sleep(2) + true_vals = ('yes', 'y', 'ye', None) + while True: + choice = input(os.linesep + 'Transmission paused, should' + ' transmission be resumed? ' + os.linesep).lower() + if not choice or choice not in true_vals: + print('Please respond with \'yes\' or \'y\' ', end='') + else: + break + with self._traffic_ctl: + self._traffic_ctl.send_traffic(self._traffic) + + # dump vswitch flows before they are affected by VNF termination + if not self._vswitch_none: + self._vswitch_ctl.dump_vswitch_flows() + + # tear down test execution environment and log results + self.run_finalize() + + self._testcase_run_time = time.strftime("%H:%M:%S", + time.gmtime(time.time() - self._testcase_start_time)) + logging.info("Testcase execution time: " + self._testcase_run_time) + # report test results + self.run_report() + + def _update_settings(self, param, value): + """ Check value of given configuration parameter + In case that new value is different, then testcase + specific settings is updated and original value stored + + :param param: Name of parameter inside settings + :param value: Disired parameter value + """ + orig_value = S.getValue(param) + if orig_value != value: + self._settings_original[param] = orig_value + S.setValue(param, value) def _append_results(self, results): """ @@ -189,53 +350,83 @@ class TestCase(object): item[ResultsConstants.ID] = self.name item[ResultsConstants.DEPLOYMENT] = self.deployment item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto'] + item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time if self._traffic['multistream']: item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream'] item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type'] item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows'] - if len(self.guest_loopback): - item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(self.guest_loopback) - + if self._vnf_ctl.get_vnfs_number(): + item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK')) + if self._tunnel_type: + item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type return results - def _copy_fwd_tools_for_guest(self): - """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] for use by guests. + def _copy_fwd_tools_for_all_guests(self, vm_count): + """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment. + """ + # consider only VNFs involved in the test + for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]): + self._copy_fwd_tools_for_guest(guest_dir) + + def _copy_fwd_tools_for_guest(self, guest_dir): + """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM + + :param index: Index of VM starting from 1 (i.e. 1st VM has index 1) """ - counter = 0 - # method is executed only for pvp and pvvp, so let's count number of 'v' - while counter < self.deployment.count('v'): - guest_dir = S.getValue('GUEST_SHARE_DIR')[counter] - - # create shared dir if it doesn't exist - if not os.path.exists(guest_dir): - os.makedirs(guest_dir) - - # copy sources into shared dir only if neccessary - if 'testpmd' in self.guest_loopback or 'l2fwd' in self.guest_loopback: - try: - tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"', - os.path.join(S.getValue('RTE_SDK'), ''), - os.path.join(guest_dir, 'DPDK')], - self._logger, - 'Copying DPDK to shared directory...', - True) - tasks.run_task(['rsync', '-a', '-r', '-l', - os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'), - os.path.join(guest_dir, 'l2fwd')], - self._logger, - 'Copying l2fwd to shared directory...', - True) - except subprocess.CalledProcessError: - self._logger.error('Unable to copy DPDK and l2fwd to shared directory') - - counter += 1 + # remove shared dir if it exists to avoid issues with file consistency + if os.path.exists(guest_dir): + tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger, + 'Removing content of shared directory...', True) + + # directory to share files between host and guest + os.makedirs(guest_dir) + + # copy sources into shared dir only if neccessary + guest_loopback = set(S.getValue('GUEST_LOOPBACK')) + if 'testpmd' in guest_loopback: + try: + # exclude whole .git/ subdirectory and all o-files; + # It is assumed, that the same RTE_TARGET is used in both host + # and VMs; This simplification significantly speeds up testpmd + # build. If we will need a different RTE_TARGET in VM, + # then we have to build whole DPDK from the scratch in VM. + # In that case we can copy just DPDK sources (e.g. by excluding + # all items obtained by git status -unormal --porcelain). + # NOTE: Excluding RTE_TARGET directory won't help on systems, + # where DPDK is built for multiple targets (e.g. for gcc & icc) + exclude = [] + exclude.append(r'--exclude=.git/') + exclude.append(r'--exclude=*.o') + tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude + + [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''), + os.path.join(guest_dir, 'DPDK')], + self._logger, + 'Copying DPDK to shared directory...', + True) + except subprocess.CalledProcessError: + self._logger.error('Unable to copy DPDK to shared directory') + raise + if 'l2fwd' in guest_loopback: + try: + tasks.run_task(['rsync', '-a', '-r', '-l', + os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'), + os.path.join(guest_dir, 'l2fwd')], + self._logger, + 'Copying l2fwd to shared directory...', + True) + except subprocess.CalledProcessError: + self._logger.error('Unable to copy l2fwd to shared directory') + raise def _mount_hugepages(self): """Mount hugepages if usage of DPDK or Qemu is detected """ # hugepages are needed by DPDK and Qemu if not self._hugepages_mounted and \ - (self.deployment.count('v') or S.getValue('VSWITCH').lower().count('dpdk')): + (self.deployment.count('v') or \ + S.getValue('VSWITCH').lower().count('dpdk') or \ + self._vswitch_none or \ + self.test and 'vnf' in [step[0][0:3] for step in self.test]): hugepages.mount_hugepages() self._hugepages_mounted = True @@ -246,8 +437,82 @@ class TestCase(object): hugepages.umount_hugepages() self._hugepages_mounted = False + def _check_for_enough_hugepages(self): + """Check to make sure enough hugepages are free to satisfy the + test environment. + """ + hugepages_needed = 0 + hugepage_size = hugepages.get_hugepage_size() + # get hugepage amounts per guest involved in the test + for guest in range(self._vnf_ctl.get_vnfs_number()): + hugepages_needed += math.ceil((int(S.getValue( + 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size) + + # get hugepage amounts for each socket on dpdk + sock0_mem, sock1_mem = 0, 0 + if S.getValue('VSWITCH').lower().count('dpdk'): + # the import below needs to remain here and not put into the module + # imports because of an exception due to settings not yet loaded + from vswitches import ovs_dpdk_vhost + if ovs_dpdk_vhost.OvsDpdkVhost.old_dpdk_config(): + match = re.search( + r'-socket-mem\s+(\d+),(\d+)', + ''.join(S.getValue('VSWITCHD_DPDK_ARGS'))) + if match: + sock0_mem, sock1_mem = (int(match.group(1)) * 1024 / hugepage_size, + int(match.group(2)) * 1024 / hugepage_size) + else: + logging.info( + 'Could not parse socket memory config in dpdk params.') + else: + sock0_mem, sock1_mem = ( + S.getValue( + 'VSWITCHD_DPDK_CONFIG')['dpdk-socket-mem'].split(',')) + sock0_mem, sock1_mem = (int(sock0_mem) * 1024 / hugepage_size, + int(sock1_mem) * 1024 / hugepage_size) + + # If hugepages needed, verify the amounts are free + if any([hugepages_needed, sock0_mem, sock1_mem]): + free_hugepages = hugepages.get_free_hugepages() + if hugepages_needed: + logging.info('Need %s hugepages free for guests', + hugepages_needed) + result1 = free_hugepages >= hugepages_needed + free_hugepages -= hugepages_needed + else: + result1 = True + + if sock0_mem: + logging.info('Need %s hugepages free for dpdk socket 0', + sock0_mem) + result2 = hugepages.get_free_hugepages('0') >= sock0_mem + free_hugepages -= sock0_mem + else: + result2 = True + + if sock1_mem: + logging.info('Need %s hugepages free for dpdk socket 1', + sock1_mem) + result3 = hugepages.get_free_hugepages('1') >= sock1_mem + free_hugepages -= sock1_mem + else: + result3 = True + + logging.info('Need a total of {} total hugepages'.format( + hugepages_needed + sock1_mem + sock0_mem)) + + # The only drawback here is sometimes dpdk doesn't release + # its hugepages on a test failure. This could cause a test + # to fail when dpdk would be OK to start because it will just + # use the previously allocated hugepages. + result4 = True if free_hugepages >= 0 else False + + return all([result1, result2, result3, result4]) + else: + return True + @staticmethod - def _write_result_to_file(results, output): + def write_result_to_file(results, output): """Write list of dictionaries to a CSV file. Each element on list will create separate row in output file. @@ -270,7 +535,6 @@ class TestCase(object): for result in results: writer.writerow(result) - @staticmethod def _get_unique_keys(list_of_dicts): """Gets unique key values as ordered list of strings in given dicts @@ -286,13 +550,10 @@ class TestCase(object): return list(result.keys()) - - def _add_flows(vswitch_ctl): + def _add_flows(self): """Add flows to the vswitch - - :param vswitch_ctl vswitch controller """ - vswitch = vswitch_ctl.get_vswitch() + vswitch = self._vswitch_ctl.get_vswitch() # TODO BOM 15-08-07 the frame mod code assumes that the # physical ports are ports 1 & 2. The actual numbers # need to be retrived from the vSwitch and the metadata value