X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=testcases%2Ftestcase.py;h=d74f34ad825aaa7d189d2b1e0b62c9542cf1d2d5;hb=681dd54cd02cc9e89f26f2799680bfe6eac5ef51;hp=5b9ead6994cc9711ef9c8dd1aea00a36d5619f05;hpb=47875a6f01fa9dc1738d4c4a29ad75e12e251077;p=vswitchperf.git diff --git a/testcases/testcase.py b/testcases/testcase.py index 5b9ead69..d74f34ad 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -1,4 +1,4 @@ -# Copyright 2015-2016 Intel Corporation. +# Copyright 2015-2017 Intel Corporation. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,29 +14,37 @@ """TestCase base class """ +from collections import OrderedDict +import copy import csv +import logging +import math import os +import re import time -import logging import subprocess -import copy -from collections import OrderedDict +from conf import settings as S +from conf import get_test_param, merge_spec import core.component_factory as component_factory from core.loader import Loader from core.results.results_constants import ResultsConstants from tools import tasks from tools import hugepages from tools import functions -from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS -from conf import settings as S -from conf import get_test_param +from tools import namespace +from tools import veth +from tools.teststepstools import TestStepsTools + +CHECK_PREFIX = 'validate_' +# pylint: disable=too-many-instance-attributes class TestCase(object): """TestCase base class In this basic form runs RFC2544 throughput test """ + # pylint: disable=too-many-statements def __init__(self, cfg): """Pull out fields from test config @@ -45,6 +53,7 @@ class TestCase(object): values. :param results_dir: Where the csv formatted results are written. """ + self._testcase_start_time = time.time() self._hugepages_mounted = False self._traffic_ctl = None self._vnf_ctl = None @@ -53,26 +62,39 @@ class TestCase(object): self._loadgen = None self._output_file = None self._tc_results = None - self.guest_loopback = [] self._settings_original = {} self._settings_paths_modified = False + self._testcast_run_time = None + # initialization of step driven specific members + self._step_check = False # by default don't check result for step driven testcases + self._step_vnf_list = {} + self._step_result = [] + self._step_status = None + self._testcase_run_time = None + + # store all GUEST_ specific settings to keep original values before their expansion + for key in S.__dict__: + if key.startswith('GUEST_'): + self._settings_original[key] = S.getValue(key) self._update_settings('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH'))) self._update_settings('VNF', cfg.get('VNF', S.getValue('VNF'))) self._update_settings('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN'))) - self._update_settings('TEST_PARAMS', cfg.get('Parameters', S.getValue('TEST_PARAMS'))) + test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) + tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS')) + test_params = merge_spec(test_params, tc_test_params) + self._update_settings('TEST_PARAMS', test_params) + S.check_test_params() + + # override all redefined GUEST_ values to have them expanded correctly + tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS')) + for key in tmp_test_params: + if key.startswith('GUEST_'): + S.setValue(key, S.getValue(key)) + S.getValue('TEST_PARAMS').pop(key) # update global settings - guest_loopback = get_test_param('guest_loopback', None) - if guest_loopback: - self._update_settings('GUEST_LOOPBACK', [guest_loopback for dummy in S.getValue('GUEST_LOOPBACK')]) - - if 'VSWITCH' in self._settings_original or 'VNF' in self._settings_original: - self._settings_original.update({ - 'RTE_SDK' : S.getValue('RTE_SDK'), - 'OVS_DIR' : S.getValue('OVS_DIR'), - }) - functions.settings_update_paths() + functions.settings_update_paths() # set test parameters; CLI options take precedence to testcase settings self._logger = logging.getLogger(__name__) @@ -80,19 +102,12 @@ class TestCase(object): self.desc = cfg.get('Description', 'No description given.') self.test = cfg.get('TestSteps', None) - bidirectional = cfg.get('biDirectional', TRAFFIC_DEFAULTS['bidir']) - bidirectional = get_test_param('bidirectional', bidirectional) - if not isinstance(bidirectional, str): + bidirectional = S.getValue('TRAFFIC')['bidir'] + if not isinstance(S.getValue('TRAFFIC')['bidir'], str): raise TypeError( - 'Bi-dir value must be of type string in testcase configuration') + 'Bi-dir value must be of type string') bidirectional = bidirectional.title() # Keep things consistent - traffic_type = cfg.get('Traffic Type', TRAFFIC_DEFAULTS['traffic_type']) - traffic_type = get_test_param('traffic_type', traffic_type) - - framerate = cfg.get('iLoad', TRAFFIC_DEFAULTS['frame_rate']) - framerate = get_test_param('iload', framerate) - self.deployment = cfg['Deployment'] self._frame_mod = cfg.get('Frame Modification', None) @@ -104,24 +119,9 @@ class TestCase(object): if 'Tunnel Type' in cfg: self._tunnel_type = cfg['Tunnel Type'] - self._tunnel_type = get_test_param('tunnel_type', + self._tunnel_type = get_test_param('TUNNEL_TYPE', self._tunnel_type) - # identify guest loopback method, so it can be added into reports - if self.deployment == 'pvp': - self.guest_loopback.append(S.getValue('GUEST_LOOPBACK')[0]) - else: - self.guest_loopback = S.getValue('GUEST_LOOPBACK').copy() - - # read configuration of streams; CLI parameter takes precedence to - # testcase definition - multistream = cfg.get('MultiStream', TRAFFIC_DEFAULTS['multistream']) - multistream = get_test_param('multistream', multistream) - stream_type = cfg.get('Stream Type', TRAFFIC_DEFAULTS['stream_type']) - stream_type = get_test_param('stream_type', stream_type) - pre_installed_flows = cfg.get('Pre-installed Flows', TRAFFIC_DEFAULTS['pre_installed_flows']) - pre_installed_flows = get_test_param('pre-installed_flows', pre_installed_flows) - # check if test requires background load and which generator it uses self._load_cfg = cfg.get('Load', None) if self._load_cfg and 'tool' in self._load_cfg: @@ -135,25 +135,12 @@ class TestCase(object): self._results_dir = S.getValue('RESULTS_PATH') # set traffic details, so they can be passed to vswitch and traffic ctls - self._traffic = copy.deepcopy(TRAFFIC_DEFAULTS) - self._traffic.update({'traffic_type': traffic_type, - 'flow_type': cfg.get('Flow Type', TRAFFIC_DEFAULTS['flow_type']), - 'bidir': bidirectional, - 'tunnel_type': self._tunnel_type, - 'multistream': int(multistream), - 'stream_type': stream_type, - 'pre_installed_flows' : pre_installed_flows, - 'frame_rate': int(framerate)}) + self._traffic = copy.deepcopy(S.getValue('TRAFFIC')) + self._traffic.update({'bidir': bidirectional, + 'tunnel_type': self._tunnel_type,}) # Packet Forwarding mode - self._vswitch_none = 'none' == S.getValue('VSWITCH').strip().lower() - - # OVS Vanilla requires guest VM MAC address and IPs to work - if 'linux_bridge' in self.guest_loopback: - self._traffic['l2'].update({'srcmac': S.getValue('GUEST_NET2_MAC')[0], - 'dstmac': S.getValue('GUEST_NET1_MAC')[0]}) - self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'), - 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')}) + self._vswitch_none = S.getValue('VSWITCH').strip().lower() == 'none' # trafficgen configuration required for tests of tunneling protocols if self.deployment == "op2p": @@ -171,7 +158,9 @@ class TestCase(object): self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2') self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3') self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4') - elif S.getValue('NICS')[0]['type'] == 'vf' or S.getValue('NICS')[1]['type'] == 'vf': + elif len(S.getValue('NICS')) and \ + (S.getValue('NICS')[0]['type'] == 'vf' or + S.getValue('NICS')[1]['type'] == 'vf'): mac1 = S.getValue('NICS')[0]['mac'] mac2 = S.getValue('NICS')[1]['mac'] if mac1 and mac2: @@ -179,6 +168,12 @@ class TestCase(object): else: self._logger.debug("MAC addresses can not be read") + # count how many VNFs are involved in TestSteps + if self.test: + for step in self.test: + if step[0].startswith('vnf'): + self._step_vnf_list[step[0]] = None + def run_initialize(self): """ Prepare test execution environment """ @@ -187,9 +182,6 @@ class TestCase(object): # mount hugepages if needed self._mount_hugepages() - # copy sources of l2 forwarding tools into VM shared dir if needed - self._copy_fwd_tools_for_all_guests() - self._logger.debug("Controllers:") loader = Loader() self._traffic_ctl = component_factory.create_traffic( @@ -198,7 +190,38 @@ class TestCase(object): self._vnf_ctl = component_factory.create_vnf( self.deployment, - loader.get_vnf_class()) + loader.get_vnf_class(), + len(self._step_vnf_list)) + + # verify enough hugepages are free to run the testcase + if not self._check_for_enough_hugepages(): + raise RuntimeError('Not enough hugepages free to run test.') + + # perform guest related handling + tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list) + if tmp_vm_count: + # copy sources of l2 forwarding tools into VM shared dir if needed + self._copy_fwd_tools_for_all_guests(tmp_vm_count) + + # in case of multi VM in parallel, set the number of streams to the number of VMs + if self.deployment.startswith('pvpv'): + # for each VM NIC pair we need an unique stream + streams = 0 + for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]: + streams += int(vm_nic / 2) if vm_nic > 1 else 1 + self._logger.debug("VMs with parallel connection were detected. " + "Thus Number of streams was set to %s", streams) + # update streams if needed; In case of additional VNFs deployed by TestSteps + # user can define a proper stream count manually + if 'multistream' not in self._traffic or self._traffic['multistream'] < streams: + self._traffic.update({'multistream': streams}) + + # OVS Vanilla requires guest VM MAC address and IPs to work + if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'): + self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'), + 'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')}) + self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'), + 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')}) if self._vswitch_none: self._vswitch_ctl = component_factory.create_pktfwd( @@ -221,29 +244,53 @@ class TestCase(object): self._output_file = os.path.join(self._results_dir, "result_" + self.name + "_" + self.deployment + ".csv") + self._step_status = {'status' : True, 'details' : ''} + self._logger.debug("Setup:") def run_finalize(self): """ Tear down test execution environment and record test results """ + # Stop all VNFs started by TestSteps in case that something went wrong + self.step_stop_vnfs() + # umount hugepages if mounted self._umount_hugepages() # restore original settings S.load_from_dict(self._settings_original) + # cleanup any namespaces created + if os.path.isdir('/tmp/namespaces'): + namespace_list = os.listdir('/tmp/namespaces') + if len(namespace_list): + self._logger.info('Cleaning up namespaces') + for name in namespace_list: + namespace.delete_namespace(name) + os.rmdir('/tmp/namespaces') + # cleanup any veth ports created + if os.path.isdir('/tmp/veth'): + veth_list = os.listdir('/tmp/veth') + if len(veth_list): + self._logger.info('Cleaning up veth ports') + for eth in veth_list: + port1, port2 = eth.split('-') + veth.del_veth_port(port1, port2) + os.rmdir('/tmp/veth') + def run_report(self): """ Report test results """ self._logger.debug("self._collector Results:") self._collector.print_results() - if S.getValue('mode') != 'trafficgen-off': + results = self._traffic_ctl.get_results() + if results: self._logger.debug("Traffic Results:") self._traffic_ctl.print_results() - self._tc_results = self._append_results(self._traffic_ctl.get_results()) - TestCase._write_result_to_file(self._tc_results, self._output_file) + self._tc_results = self._append_results(results) + TestCase.write_result_to_file(self._tc_results, self._output_file) def run(self): """Run the test @@ -253,37 +300,36 @@ class TestCase(object): # prepare test execution environment self.run_initialize() - with self._vswitch_ctl, self._loadgen: - with self._vnf_ctl, self._collector: - if not self._vswitch_none: - self._add_flows() - - # run traffic generator if requested, otherwise wait for manual termination - if S.getValue('mode') == 'trafficgen-off': - time.sleep(2) - self._logger.debug("All is set. Please run traffic generator manually.") - input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep) - else: - if S.getValue('mode') == 'trafficgen-pause': - time.sleep(2) - true_vals = ('yes', 'y', 'ye', None) - while True: - choice = input(os.linesep + 'Transmission paused, should' - ' transmission be resumed? ' + os.linesep).lower() - if not choice or choice not in true_vals: - print('Please respond with \'yes\' or \'y\' ', end='') - else: - break - with self._traffic_ctl: - self._traffic_ctl.send_traffic(self._traffic) - - # dump vswitch flows before they are affected by VNF termination + try: + with self._vswitch_ctl, self._loadgen: + with self._vnf_ctl, self._collector: if not self._vswitch_none: - self._vswitch_ctl.dump_vswitch_flows() - - # tear down test execution environment and log results - self.run_finalize() + self._add_flows() + with self._traffic_ctl: + # execute test based on TestSteps definition if needed... + if self.step_run(): + # ...and continue with traffic generation, but keep + # in mind, that clean deployment does not configure + # OVS nor executes the traffic + if self.deployment != 'clean': + self._traffic_ctl.send_traffic(self._traffic) + + # dump vswitch flows before they are affected by VNF termination + if not self._vswitch_none: + self._vswitch_ctl.dump_vswitch_flows() + + # garbage collection for case that TestSteps modify existing deployment + self.step_stop_vnfs() + + finally: + # tear down test execution environment and log results + self.run_finalize() + + self._testcase_run_time = time.strftime("%H:%M:%S", + time.gmtime(time.time() - + self._testcase_start_time)) + logging.info("Testcase execution time: " + self._testcase_run_time) # report test results self.run_report() @@ -297,7 +343,7 @@ class TestCase(object): """ orig_value = S.getValue(param) if orig_value != value: - self._settings_original[param] = orig_value + self._settings_original[param] = copy.deepcopy(orig_value) S.setValue(param, value) def _append_results(self, results): @@ -313,32 +359,29 @@ class TestCase(object): item[ResultsConstants.ID] = self.name item[ResultsConstants.DEPLOYMENT] = self.deployment item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto'] + item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time if self._traffic['multistream']: item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream'] item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type'] item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows'] - if self.deployment in ['pvp', 'pvvp'] and len(self.guest_loopback): - item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(self.guest_loopback) + if self._vnf_ctl.get_vnfs_number(): + item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK')) if self._tunnel_type: item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type return results - def _copy_fwd_tools_for_all_guests(self): + def _copy_fwd_tools_for_all_guests(self, vm_count): """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment. """ - # data are copied only for pvp and pvvp, so let's count number of 'v' - counter = 1 - while counter <= self.deployment.count('v'): - self._copy_fwd_tools_for_guest(counter) - counter += 1 + # consider only VNFs involved in the test + for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]): + self._copy_fwd_tools_for_guest(guest_dir) - def _copy_fwd_tools_for_guest(self, index): + def _copy_fwd_tools_for_guest(self, guest_dir): """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM :param index: Index of VM starting from 1 (i.e. 1st VM has index 1) """ - guest_dir = S.getValue('GUEST_SHARE_DIR')[index-1] - # remove shared dir if it exists to avoid issues with file consistency if os.path.exists(guest_dir): tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger, @@ -348,14 +391,32 @@ class TestCase(object): os.makedirs(guest_dir) # copy sources into shared dir only if neccessary - if 'testpmd' in self.guest_loopback or 'l2fwd' in self.guest_loopback: + guest_loopback = set(S.getValue('GUEST_LOOPBACK')) + if 'testpmd' in guest_loopback: try: - tasks.run_task(['rsync', '-a', '-r', '-l', r'--exclude="\.git"', - os.path.join(S.getValue('RTE_SDK'), ''), + # exclude whole .git/ subdirectory and all o-files; + # It is assumed, that the same RTE_TARGET is used in both host + # and VMs; This simplification significantly speeds up testpmd + # build. If we will need a different RTE_TARGET in VM, + # then we have to build whole DPDK from the scratch in VM. + # In that case we can copy just DPDK sources (e.g. by excluding + # all items obtained by git status -unormal --porcelain). + # NOTE: Excluding RTE_TARGET directory won't help on systems, + # where DPDK is built for multiple targets (e.g. for gcc & icc) + exclude = [] + exclude.append(r'--exclude=.git/') + exclude.append(r'--exclude=*.o') + tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude + + [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''), os.path.join(guest_dir, 'DPDK')], self._logger, 'Copying DPDK to shared directory...', True) + except subprocess.CalledProcessError: + self._logger.error('Unable to copy DPDK to shared directory') + raise + if 'l2fwd' in guest_loopback: + try: tasks.run_task(['rsync', '-a', '-r', '-l', os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'), os.path.join(guest_dir, 'l2fwd')], @@ -363,12 +424,13 @@ class TestCase(object): 'Copying l2fwd to shared directory...', True) except subprocess.CalledProcessError: - self._logger.error('Unable to copy DPDK and l2fwd to shared directory') - + self._logger.error('Unable to copy l2fwd to shared directory') + raise def _mount_hugepages(self): """Mount hugepages if usage of DPDK or Qemu is detected """ + # pylint: disable=too-many-boolean-expressions # hugepages are needed by DPDK and Qemu if not self._hugepages_mounted and \ (self.deployment.count('v') or \ @@ -385,8 +447,82 @@ class TestCase(object): hugepages.umount_hugepages() self._hugepages_mounted = False + def _check_for_enough_hugepages(self): + """Check to make sure enough hugepages are free to satisfy the + test environment. + """ + hugepages_needed = 0 + hugepage_size = hugepages.get_hugepage_size() + # get hugepage amounts per guest involved in the test + for guest in range(self._vnf_ctl.get_vnfs_number()): + hugepages_needed += math.ceil((int(S.getValue( + 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size) + + # get hugepage amounts for each socket on dpdk + sock0_mem, sock1_mem = 0, 0 + if S.getValue('VSWITCH').lower().count('dpdk'): + # the import below needs to remain here and not put into the module + # imports because of an exception due to settings not yet loaded + from vswitches import ovs_dpdk_vhost + if ovs_dpdk_vhost.OvsDpdkVhost.old_dpdk_config(): + match = re.search( + r'-socket-mem\s+(\d+),(\d+)', + ''.join(S.getValue('VSWITCHD_DPDK_ARGS'))) + if match: + sock0_mem, sock1_mem = (int(match.group(1)) * 1024 / hugepage_size, + int(match.group(2)) * 1024 / hugepage_size) + else: + logging.info( + 'Could not parse socket memory config in dpdk params.') + else: + sock0_mem, sock1_mem = ( + S.getValue( + 'VSWITCHD_DPDK_CONFIG')['dpdk-socket-mem'].split(',')) + sock0_mem, sock1_mem = (int(sock0_mem) * 1024 / hugepage_size, + int(sock1_mem) * 1024 / hugepage_size) + + # If hugepages needed, verify the amounts are free + if any([hugepages_needed, sock0_mem, sock1_mem]): + free_hugepages = hugepages.get_free_hugepages() + if hugepages_needed: + logging.info('Need %s hugepages free for guests', + hugepages_needed) + result1 = free_hugepages >= hugepages_needed + free_hugepages -= hugepages_needed + else: + result1 = True + + if sock0_mem: + logging.info('Need %s hugepages free for dpdk socket 0', + sock0_mem) + result2 = hugepages.get_free_hugepages('0') >= sock0_mem + free_hugepages -= sock0_mem + else: + result2 = True + + if sock1_mem: + logging.info('Need %s hugepages free for dpdk socket 1', + sock1_mem) + result3 = hugepages.get_free_hugepages('1') >= sock1_mem + free_hugepages -= sock1_mem + else: + result3 = True + + logging.info('Need a total of %s total hugepages', + hugepages_needed + sock1_mem + sock0_mem) + + # The only drawback here is sometimes dpdk doesn't release + # its hugepages on a test failure. This could cause a test + # to fail when dpdk would be OK to start because it will just + # use the previously allocated hugepages. + result4 = True if free_hugepages >= 0 else False + + return all([result1, result2, result3, result4]) + else: + return True + @staticmethod - def _write_result_to_file(results, output): + def write_result_to_file(results, output): """Write list of dictionaries to a CSV file. Each element on list will create separate row in output file. @@ -428,7 +564,7 @@ class TestCase(object): """Add flows to the vswitch """ vswitch = self._vswitch_ctl.get_vswitch() - # TODO BOM 15-08-07 the frame mod code assumes that the + # NOTE BOM 15-08-07 the frame mod code assumes that the # physical ports are ports 1 & 2. The actual numbers # need to be retrived from the vSwitch and the metadata value # updated accordingly. @@ -494,7 +630,7 @@ class TestCase(object): 'goto_table:3']} vswitch.add_flow(bridge, flow) elif self._frame_mod == "ip_port": - # TODO BOM 15-08-27 The traffic generated is assumed + # NOTE BOM 15-08-27 The traffic generated is assumed # to be UDP (nw_proto 17d) which is the default case but # we will need to pick up the actual traffic params in use. flow = {'table':'2', 'priority':'1000', 'metadata':'2', @@ -509,3 +645,147 @@ class TestCase(object): vswitch.add_flow(bridge, flow) else: pass + + + # + # TestSteps realted methods + # + def step_report_status(self, label, status): + """ Log status of test step + """ + self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED') + + def step_stop_vnfs(self): + """ Stop all VNFs started by TestSteps + """ + for vnf in self._step_vnf_list: + if self._step_vnf_list[vnf]: + self._step_vnf_list[vnf].stop() + + @staticmethod + def step_eval_param(param, STEP): + # pylint: disable=invalid-name + """ Helper function for #STEP macro evaluation + """ + if isinstance(param, str): + # evaluate every #STEP reference inside parameter itself + macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param) + if macros: + for macro in macros: + # pylint: disable=eval-used + tmp_val = str(eval(macro[1:])) + param = param.replace(macro, tmp_val) + return param + elif isinstance(param, list) or isinstance(param, tuple): + tmp_list = [] + for item in param: + tmp_list.append(TestCase.step_eval_param(item, STEP)) + return tmp_list + elif isinstance(param, dict): + tmp_dict = {} + for (key, value) in param.items(): + tmp_dict[key] = TestCase.step_eval_param(value, STEP) + return tmp_dict + else: + return param + + @staticmethod + def step_eval_params(params, step_result): + """ Evaluates referrences to results from previous steps + """ + eval_params = [] + # evaluate all parameters if needed + for param in params: + eval_params.append(TestCase.step_eval_param(param, step_result)) + return eval_params + + def step_run(self): + """ Execute actions specified by TestSteps list + + :return: False if any error was detected + True otherwise + """ + # anything to do? + if not self.test: + return True + + # required for VNFs initialization + loader = Loader() + # initialize list with results + self._step_result = [None] * len(self.test) + + # We have to suppress pylint report, because test_object has to be set according + # to the test step definition + # pylint: disable=redefined-variable-type + # run test step by step... + for i, step in enumerate(self.test): + step_ok = not self._step_check + if step[0] == 'vswitch': + test_object = self._vswitch_ctl.get_vswitch() + elif step[0] == 'namespace': + test_object = namespace + elif step[0] == 'veth': + test_object = veth + elif step[0] == 'settings': + test_object = S + elif step[0] == 'tools': + test_object = TestStepsTools() + step[1] = step[1].title() + elif step[0] == 'trafficgen': + test_object = self._traffic_ctl + # in case of send_traffic or send_traffic_async methods, ensure + # that specified traffic values are merged with existing self._traffic + if step[1].startswith('send_traffic'): + tmp_traffic = copy.deepcopy(self._traffic) + tmp_traffic.update(step[2]) + step[2] = tmp_traffic + elif step[0].startswith('vnf'): + if not self._step_vnf_list[step[0]]: + # initialize new VM + self._step_vnf_list[step[0]] = loader.get_vnf_class()() + test_object = self._step_vnf_list[step[0]] + elif step[0] == 'wait': + input(os.linesep + "Step {}: Press Enter to continue with " + "the next step...".format(i) + os.linesep + os.linesep) + continue + else: + self._logger.error("Unsupported test object %s", step[0]) + self._step_status = {'status' : False, 'details' : ' '.join(step)} + self.step_report_status("Step '{}'".format(' '.join(step)), + self._step_status['status']) + return False + + test_method = getattr(test_object, step[1]) + if self._step_check: + test_method_check = getattr(test_object, CHECK_PREFIX + step[1]) + else: + test_method_check = None + + step_params = [] + try: + # eval parameters, but use only valid step_results + # to support negative indexes + step_params = TestCase.step_eval_params(step[2:], self._step_result[:i]) + step_log = '{} {}'.format(' '.join(step[:2]), step_params) + self._logger.debug("Step %s '%s' start", i, step_log) + self._step_result[i] = test_method(*step_params) + self._logger.debug("Step %s '%s' results '%s'", i, + step_log, self._step_result[i]) + time.sleep(5) + if self._step_check: + step_ok = test_method_check(self._step_result[i], *step_params) + except (AssertionError, AttributeError, IndexError) as ex: + step_ok = False + self._logger.error("Step %s raised %s", i, type(ex).__name__) + + if self._step_check: + self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok) + + if not step_ok: + self._step_status = {'status' : False, 'details' : step_log} + # Stop all VNFs started by TestSteps + self.step_stop_vnfs() + return False + + # all steps processed without any issue + return True