1 # Copyright 2015-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """TestCase base class
17 from collections import OrderedDict
27 from datetime import datetime as dt
28 from conf import settings as S
29 from conf import get_test_param, merge_spec
30 import core.component_factory as component_factory
31 from core.loader import Loader
32 from core.results.results_constants import ResultsConstants
33 from tools import tasks
34 from tools import hugepages
35 from tools import functions
36 from tools import namespace
37 from tools import veth
38 from tools.teststepstools import TestStepsTools
40 CHECK_PREFIX = 'validate_'
42 # pylint: disable=too-many-instance-attributes
43 class TestCase(object):
44 """TestCase base class
46 In this basic form runs RFC2544 throughput test
48 # pylint: disable=too-many-statements
49 def __init__(self, test_cfg):
50 """Pull out fields from test config
52 :param test_cfg: A dictionary of string-value pairs describing the test
53 configuration. Both the key and values strings use well-known
55 :param results_dir: Where the csv formatted results are written.
57 # make a local copy of test configuration to avoid modification of
58 # original content used in vsperf main script
59 cfg = copy.deepcopy(test_cfg)
61 self._testcase_start_time = time.time()
62 self._testcase_stop_time = self._testcase_start_time
63 self._hugepages_mounted = False
64 self._traffic_ctl = None
66 self._vswitch_ctl = None
67 self._collector = None
69 self._output_file = None
70 self._tc_results = None
71 self._settings_paths_modified = False
72 self._testcast_run_time = None
74 # initialization of step driven specific members
75 self._step_check = False # by default don't check result for step driven testcases
76 self._step_vnf_list = {}
77 self._step_result = []
78 self._step_result_mapping = {}
79 self._step_status = None
80 self._step_send_traffic = False # indication if send_traffic was called within test steps
82 self._testcase_run_time = None
84 S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
85 S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
86 S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
87 test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
88 tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
89 test_params = merge_spec(test_params, tc_test_params)
90 S.setValue('TEST_PARAMS', test_params)
93 # override all redefined GUEST_ values to have them expanded correctly
94 tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
95 for key in tmp_test_params:
96 if key.startswith('GUEST_'):
97 S.setValue(key, S.getValue(key))
98 S.getValue('TEST_PARAMS').pop(key)
100 # update global settings
101 functions.settings_update_paths()
103 # set test parameters; CLI options take precedence to testcase settings
104 self._logger = logging.getLogger(__name__)
105 self.name = cfg['Name']
106 self.desc = cfg.get('Description', 'No description given.')
107 self.test = cfg.get('TestSteps', None)
109 # log testcase name and details
110 tmp_desc = functions.format_description(self.desc, 50)
111 self._logger.info('############################################################')
112 self._logger.info('# Test: %s', self.name)
113 self._logger.info('# Details: %s', tmp_desc[0])
114 for i in range(1, len(tmp_desc)):
115 self._logger.info('# %s', tmp_desc[i])
116 self._logger.info('############################################################')
118 bidirectional = S.getValue('TRAFFIC')['bidir']
119 if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
121 'Bi-dir value must be of type string')
122 bidirectional = bidirectional.title() # Keep things consistent
124 self.deployment = cfg['Deployment']
125 self._frame_mod = cfg.get('Frame Modification', None)
127 self._tunnel_type = None
128 self._tunnel_operation = None
130 if self.deployment == 'op2p':
131 self._tunnel_operation = cfg['Tunnel Operation']
133 if 'Tunnel Type' in cfg:
134 self._tunnel_type = cfg['Tunnel Type']
135 self._tunnel_type = get_test_param('TUNNEL_TYPE',
138 # check if test requires background load and which generator it uses
139 self._load_cfg = cfg.get('Load', None)
142 self._frame_mod = self._frame_mod.lower()
143 self._results_dir = S.getValue('RESULTS_PATH')
145 # set traffic details, so they can be passed to vswitch and traffic ctls
146 self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
147 self._traffic.update({'bidir': bidirectional,
148 'tunnel_type': self._tunnel_type,})
150 self._traffic = functions.check_traffic(self._traffic)
152 # Packet Forwarding mode
153 self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
155 # trafficgen configuration required for tests of tunneling protocols
156 if self.deployment == "op2p":
157 self._traffic['l2'].update({'srcmac':
158 S.getValue('TRAFFICGEN_PORT1_MAC'),
160 S.getValue('TRAFFICGEN_PORT2_MAC')})
162 self._traffic['l3'].update({'srcip':
163 S.getValue('TRAFFICGEN_PORT1_IP'),
165 S.getValue('TRAFFICGEN_PORT2_IP')})
167 if self._tunnel_operation == "decapsulation":
168 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
169 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
170 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
171 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
172 elif len(S.getValue('NICS')) and \
173 (S.getValue('NICS')[0]['type'] == 'vf' or
174 S.getValue('NICS')[1]['type'] == 'vf'):
175 mac1 = S.getValue('NICS')[0]['mac']
176 mac2 = S.getValue('NICS')[1]['mac']
178 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
180 self._logger.debug("MAC addresses can not be read")
182 # count how many VNFs are involved in TestSteps
184 for step in self.test:
185 if step[0].startswith('vnf'):
186 self._step_vnf_list[step[0]] = None
188 def run_initialize(self):
189 """ Prepare test execution environment
191 # mount hugepages if needed
192 self._mount_hugepages()
194 self._logger.debug("Controllers:")
196 self._traffic_ctl = component_factory.create_traffic(
197 self._traffic['traffic_type'],
198 loader.get_trafficgen_class())
200 self._vnf_ctl = component_factory.create_vnf(
202 loader.get_vnf_class(),
203 len(self._step_vnf_list))
205 self._vnf_list = self._vnf_ctl.get_vnfs()
207 # verify enough hugepages are free to run the testcase
208 if not self._check_for_enough_hugepages():
209 raise RuntimeError('Not enough hugepages free to run test.')
211 # perform guest related handling
212 tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
214 # copy sources of l2 forwarding tools into VM shared dir if needed
215 self._copy_fwd_tools_for_all_guests(tmp_vm_count)
217 # in case of multi VM in parallel, set the number of streams to the number of VMs
218 if self.deployment.startswith('pvpv'):
219 # for each VM NIC pair we need an unique stream
221 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
222 streams += int(vm_nic / 2) if vm_nic > 1 else 1
223 self._logger.debug("VMs with parallel connection were detected. "
224 "Thus Number of streams was set to %s", streams)
225 # update streams if needed; In case of additional VNFs deployed by TestSteps
226 # user can define a proper stream count manually
227 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
228 self._traffic.update({'multistream': streams})
230 # OVS Vanilla requires guest VM MAC address and IPs to work
231 if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
232 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
233 'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
234 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
235 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
237 if self._vswitch_none:
238 self._vswitch_ctl = component_factory.create_pktfwd(
240 loader.get_pktfwd_class())
242 self._vswitch_ctl = component_factory.create_vswitch(
244 loader.get_vswitch_class(),
246 self._tunnel_operation)
248 self._collector = component_factory.create_collector(
249 loader.get_collector_class(),
250 self._results_dir, self.name)
251 self._loadgen = component_factory.create_loadgen(
252 loader.get_loadgen_class(),
255 self._output_file = os.path.join(self._results_dir, "result_" + self.name +
256 "_" + self.deployment + ".csv")
258 self._step_status = {'status' : True, 'details' : ''}
260 self._logger.debug("Setup:")
262 def run_finalize(self):
263 """ Tear down test execution environment and record test results
265 # Stop all VNFs started by TestSteps in case that something went wrong
266 self.step_stop_vnfs()
268 # umount hugepages if mounted
269 self._umount_hugepages()
271 # cleanup any namespaces created
272 if os.path.isdir('/tmp/namespaces'):
273 namespace_list = os.listdir('/tmp/namespaces')
274 if len(namespace_list):
275 self._logger.info('Cleaning up namespaces')
276 for name in namespace_list:
277 namespace.delete_namespace(name)
278 os.rmdir('/tmp/namespaces')
279 # cleanup any veth ports created
280 if os.path.isdir('/tmp/veth'):
281 veth_list = os.listdir('/tmp/veth')
283 self._logger.info('Cleaning up veth ports')
284 for eth in veth_list:
285 port1, port2 = eth.split('-')
286 veth.del_veth_port(port1, port2)
287 os.rmdir('/tmp/veth')
289 def run_report(self):
290 """ Report test results
292 self._logger.debug("self._collector Results:")
293 self._collector.print_results()
295 results = self._traffic_ctl.get_results()
297 self._logger.debug("Traffic Results:")
298 self._traffic_ctl.print_results()
300 if self._tc_results is None:
301 self._tc_results = self._append_results(results)
303 # integration step driven tests have their status and possible
304 # failure details stored inside self._tc_results
305 results = self._append_results(results)
306 if len(self._tc_results) < len(results):
307 if len(self._tc_results) > 1:
308 raise RuntimeError('Testcase results do not match:'
310 'trafficgen results: {}\n',
314 tmp_results = copy.deepcopy(self._tc_results[0])
315 self._tc_results = []
317 tmp_res = copy.deepcopy(tmp_results)
319 self._tc_results.append(tmp_res)
321 for i, result in enumerate(results):
322 self._tc_results[i].update(result)
324 TestCase.write_result_to_file(self._tc_results, self._output_file)
329 All setup and teardown through controllers is included.
331 # prepare test execution environment
332 self.run_initialize()
335 with self._vswitch_ctl, self._loadgen:
336 with self._vnf_ctl, self._collector:
337 if not self._vswitch_none:
340 self._versions += self._vswitch_ctl.get_vswitch().get_version()
342 with self._traffic_ctl:
343 # execute test based on TestSteps definition if needed...
345 # ...and continue with traffic generation, but keep
346 # in mind, that clean deployment does not configure
347 # OVS nor executes the traffic
348 if self.deployment != 'clean' and not self._step_send_traffic:
349 self._traffic_ctl.send_traffic(self._traffic)
351 # dump vswitch flows before they are affected by VNF termination
352 if not self._vswitch_none:
353 self._vswitch_ctl.dump_vswitch_flows()
355 # garbage collection for case that TestSteps modify existing deployment
356 self.step_stop_vnfs()
359 # tear down test execution environment and log results
362 self._testcase_stop_time = time.time()
363 self._testcase_run_time = time.strftime("%H:%M:%S",
364 time.gmtime(self._testcase_stop_time -
365 self._testcase_start_time))
366 logging.info("Testcase execution time: " + self._testcase_run_time)
367 # report test results
370 def _append_results(self, results):
372 Method appends mandatory Test Case results to list of dictionaries.
374 :param results: list of dictionaries which contains results from
377 :returns: modified list of dictionaries.
380 item[ResultsConstants.ID] = self.name
381 item[ResultsConstants.DEPLOYMENT] = self.deployment
382 item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
383 item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
384 item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
385 # convert timestamps to human readable format
386 item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
387 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
388 item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
389 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
390 if self._traffic['multistream']:
391 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
392 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
393 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
394 if self._vnf_ctl.get_vnfs_number():
395 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
396 if self._tunnel_type:
397 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
400 def _copy_fwd_tools_for_all_guests(self, vm_count):
401 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
403 # consider only VNFs involved in the test
404 for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
405 self._copy_fwd_tools_for_guest(guest_dir)
407 def _copy_fwd_tools_for_guest(self, guest_dir):
408 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
410 :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
412 # remove shared dir if it exists to avoid issues with file consistency
413 if os.path.exists(guest_dir):
414 tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
415 'Removing content of shared directory...', True)
417 # directory to share files between host and guest
418 os.makedirs(guest_dir)
420 # copy sources into shared dir only if neccessary
421 guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
422 if 'testpmd' in guest_loopback:
424 # exclude whole .git/ subdirectory and all o-files;
425 # It is assumed, that the same RTE_TARGET is used in both host
426 # and VMs; This simplification significantly speeds up testpmd
427 # build. If we will need a different RTE_TARGET in VM,
428 # then we have to build whole DPDK from the scratch in VM.
429 # In that case we can copy just DPDK sources (e.g. by excluding
430 # all items obtained by git status -unormal --porcelain).
431 # NOTE: Excluding RTE_TARGET directory won't help on systems,
432 # where DPDK is built for multiple targets (e.g. for gcc & icc)
434 exclude.append(r'--exclude=.git/')
435 exclude.append(r'--exclude=*.o')
436 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
437 [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
438 os.path.join(guest_dir, 'DPDK')],
440 'Copying DPDK to shared directory...',
442 except subprocess.CalledProcessError:
443 self._logger.error('Unable to copy DPDK to shared directory')
445 if 'l2fwd' in guest_loopback:
447 tasks.run_task(['rsync', '-a', '-r', '-l',
448 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
449 os.path.join(guest_dir, 'l2fwd')],
451 'Copying l2fwd to shared directory...',
453 except subprocess.CalledProcessError:
454 self._logger.error('Unable to copy l2fwd to shared directory')
457 def _mount_hugepages(self):
458 """Mount hugepages if usage of DPDK or Qemu is detected
460 # pylint: disable=too-many-boolean-expressions
461 # hugepages are needed by DPDK and Qemu
462 if not self._hugepages_mounted and \
463 (self.deployment.count('v') or \
464 str(S.getValue('VSWITCH')).lower().count('dpdk') or \
465 self._vswitch_none or \
466 self.test and 'vnf' in [step[0][0:3] for step in self.test]):
467 hugepages.mount_hugepages()
468 self._hugepages_mounted = True
470 def _umount_hugepages(self):
471 """Umount hugepages if they were mounted before
473 if self._hugepages_mounted:
474 hugepages.umount_hugepages()
475 self._hugepages_mounted = False
477 def _check_for_enough_hugepages(self):
478 """Check to make sure enough hugepages are free to satisfy the
482 hugepage_size = hugepages.get_hugepage_size()
483 # get hugepage amounts per guest involved in the test
484 for guest in range(self._vnf_ctl.get_vnfs_number()):
485 hugepages_needed += math.ceil((int(S.getValue(
486 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
488 # get hugepage amounts for each socket on dpdk
489 sock0_mem, sock1_mem = 0, 0
491 if str(S.getValue('VSWITCH')).lower().count('dpdk'):
492 sock_mem = S.getValue('DPDK_SOCKET_MEM')
493 sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
494 int(sock_mem[1]) * 1024 / hugepage_size)
496 # If hugepages needed, verify the amounts are free
497 if any([hugepages_needed, sock0_mem, sock1_mem]):
498 free_hugepages = hugepages.get_free_hugepages()
500 logging.info('Need %s hugepages free for guests',
502 result1 = free_hugepages >= hugepages_needed
503 free_hugepages -= hugepages_needed
508 logging.info('Need %s hugepages free for dpdk socket 0',
510 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
511 free_hugepages -= sock0_mem
516 logging.info('Need %s hugepages free for dpdk socket 1',
518 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
519 free_hugepages -= sock1_mem
523 logging.info('Need a total of %s total hugepages',
524 hugepages_needed + sock1_mem + sock0_mem)
526 # The only drawback here is sometimes dpdk doesn't release
527 # its hugepages on a test failure. This could cause a test
528 # to fail when dpdk would be OK to start because it will just
529 # use the previously allocated hugepages.
530 result4 = True if free_hugepages >= 0 else False
532 return all([result1, result2, result3, result4])
537 def write_result_to_file(results, output):
538 """Write list of dictionaries to a CSV file.
540 Each element on list will create separate row in output file.
541 If output file already exists, data will be appended at the end,
542 otherwise it will be created.
544 :param results: list of dictionaries.
545 :param output: path to output file.
547 with open(output, 'a') as csvfile:
549 logging.info("Write results to file: " + output)
550 fieldnames = TestCase._get_unique_keys(results)
552 writer = csv.DictWriter(csvfile, fieldnames)
554 if not csvfile.tell(): # file is now empty
557 for result in results:
558 writer.writerow(result)
561 def _get_unique_keys(list_of_dicts):
562 """Gets unique key values as ordered list of strings in given dicts
564 :param list_of_dicts: list of dictionaries.
566 :returns: list of unique keys(strings).
568 result = OrderedDict()
569 for item in list_of_dicts:
570 for key in item.keys():
573 return list(result.keys())
575 def _add_flows(self):
576 """Add flows to the vswitch
578 vswitch = self._vswitch_ctl.get_vswitch()
579 # NOTE BOM 15-08-07 the frame mod code assumes that the
580 # physical ports are ports 1 & 2. The actual numbers
581 # need to be retrived from the vSwitch and the metadata value
582 # updated accordingly.
583 bridge = S.getValue('VSWITCH_BRIDGE_NAME')
584 if self._frame_mod == "vlan":
585 # 0x8100 => VLAN ethertype
586 self._logger.debug(" **** VLAN ***** ")
587 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
588 'actions': ['push_vlan:0x8100', 'goto_table:3']}
589 vswitch.add_flow(bridge, flow)
590 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
591 'actions': ['push_vlan:0x8100', 'goto_table:3']}
592 vswitch.add_flow(bridge, flow)
593 elif self._frame_mod == "mpls":
594 # 0x8847 => MPLS unicast ethertype
595 self._logger.debug(" **** MPLS ***** ")
596 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
597 'actions': ['push_mpls:0x8847', 'goto_table:3']}
598 vswitch.add_flow(bridge, flow)
599 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
600 'actions': ['push_mpls:0x8847', 'goto_table:3']}
601 vswitch.add_flow(bridge, flow)
602 elif self._frame_mod == "mac":
603 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
604 'actions': ['mod_dl_src:22:22:22:22:22:22',
606 vswitch.add_flow(bridge, flow)
607 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
608 'actions': ['mod_dl_src:11:11:11:11:11:11',
610 vswitch.add_flow(bridge, flow)
611 elif self._frame_mod == "dscp":
612 # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
613 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
615 'actions': ['mod_nw_tos:184', 'goto_table:3']}
616 vswitch.add_flow(bridge, flow)
617 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
619 'actions': ['mod_nw_tos:184', 'goto_table:3']}
620 vswitch.add_flow(bridge, flow)
621 elif self._frame_mod == "ttl":
622 # 251 and 241 are the highest prime numbers < 255
623 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
625 'actions': ['mod_nw_ttl:251', 'goto_table:3']}
626 vswitch.add_flow(bridge, flow)
627 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
629 'actions': ['mod_nw_ttl:241', 'goto_table:3']}
630 vswitch.add_flow(bridge, flow)
631 elif self._frame_mod == "ip_addr":
632 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
634 'actions': ['mod_nw_src:10.10.10.10',
635 'mod_nw_dst:20.20.20.20',
637 vswitch.add_flow(bridge, flow)
638 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
640 'actions': ['mod_nw_src:20.20.20.20',
641 'mod_nw_dst:10.10.10.10',
643 vswitch.add_flow(bridge, flow)
644 elif self._frame_mod == "ip_port":
645 # NOTE BOM 15-08-27 The traffic generated is assumed
646 # to be UDP (nw_proto 17d) which is the default case but
647 # we will need to pick up the actual traffic params in use.
648 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
649 'dl_type':'0x0800', 'nw_proto':'17',
650 'actions': ['mod_tp_src:44444',
651 'mod_tp_dst:44444', 'goto_table:3']}
652 vswitch.add_flow(bridge, flow)
653 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
654 'dl_type':'0x0800', 'nw_proto':'17',
655 'actions': ['mod_tp_src:44444',
656 'mod_tp_dst:44444', 'goto_table:3']}
657 vswitch.add_flow(bridge, flow)
663 # TestSteps realted methods
665 def step_report_status(self, label, status):
666 """ Log status of test step
668 self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
670 def step_stop_vnfs(self):
671 """ Stop all VNFs started by TestSteps
673 for vnf in self._step_vnf_list:
674 if self._step_vnf_list[vnf]:
675 self._step_vnf_list[vnf].stop()
677 def step_eval_param(self, param, step_result):
678 """ Helper function for #STEP macro evaluation
680 if isinstance(param, str):
681 # evaluate every #STEP reference inside parameter itself
682 macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
686 if macro[1] in self._step_result_mapping:
687 key = self._step_result_mapping[macro[1]]
690 # pylint: disable=eval-used
691 tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
692 param = param.replace(macro[0], tmp_val)
694 # evaluate references to vsperf configuration options
695 macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
698 # pylint: disable=eval-used
700 tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
701 param = param.replace('${}'.format(macro[0]), tmp_val)
702 # ignore that required option can't be evaluated
703 except (IndexError, KeyError, AttributeError):
704 self._logger.debug("Skipping %s as it isn't a configuration "
705 "parameter.", '${}'.format(macro[0]))
707 elif isinstance(param, list) or isinstance(param, tuple):
710 tmp_list.append(self.step_eval_param(item, step_result))
712 elif isinstance(param, dict):
714 for (key, value) in param.items():
715 tmp_dict[key] = self.step_eval_param(value, step_result)
720 def step_eval_params(self, params, step_result):
721 """ Evaluates referrences to results from previous steps
724 # evaluate all parameters if needed
726 eval_params.append(self.step_eval_param(param, step_result))
729 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
731 """ Execute actions specified by TestSteps list
733 :return: False if any error was detected
740 # required for VNFs initialization
742 # initialize list with results
743 self._step_result = [None] * len(self.test)
745 # We have to suppress pylint report, because test_object has to be set according
746 # to the test step definition
747 # pylint: disable=redefined-variable-type
748 # run test step by step...
749 for i, step in enumerate(self.test):
750 step_ok = not self._step_check
751 step_check = self._step_check
753 # configure step result mapping if step alias/label is detected
754 if step[0].startswith('#'):
757 raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
758 if key in self._step_result_mapping:
759 raise RuntimeError('Step alias {} has been used already for step '
760 '{}'.format(key, self._step_result_mapping[key]))
761 self._step_result_mapping[step[0][1:]] = i
764 # store regex filter if it is specified
765 if isinstance(step[-1], str) and step[-1].startswith('|'):
766 # evalute macros and variables used in regex
767 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
770 # check if step verification should be suppressed
771 if step[0].startswith('!'):
774 step[0] = step[0][1:]
775 if step[0] == 'vswitch':
776 test_object = self._vswitch_ctl.get_vswitch()
777 elif step[0] == 'namespace':
778 test_object = namespace
779 elif step[0] == 'veth':
781 elif step[0] == 'settings':
783 elif step[0] == 'tools':
784 test_object = TestStepsTools()
785 step[1] = step[1].title()
786 elif step[0] == 'trafficgen':
787 test_object = self._traffic_ctl
788 # in case of send_traffic or send_traffic_async methods, ensure
789 # that specified traffic values are merged with existing self._traffic
790 if step[1].startswith('send_traffic'):
791 tmp_traffic = copy.deepcopy(self._traffic)
792 tmp_traffic.update(step[2])
793 step[2] = tmp_traffic
794 # store indication that traffic has been sent
795 # so it is not sent again after the execution of teststeps
796 self._step_send_traffic = True
797 elif step[0].startswith('vnf'):
798 # use vnf started within TestSteps
799 if not self._step_vnf_list[step[0]]:
801 self._step_vnf_list[step[0]] = loader.get_vnf_class()()
802 test_object = self._step_vnf_list[step[0]]
803 elif step[0].startswith('VNF'):
804 if step[1] in ('start', 'stop'):
805 raise RuntimeError("Cannot execute start() or stop() method of "
806 "VNF deployed automatically by scenario.")
807 # use vnf started by scenario deployment (e.g. pvp)
808 vnf_index = int(step[0][3:])
810 test_object = self._vnf_list[vnf_index]
812 raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
813 elif step[0] == 'wait':
814 input(os.linesep + "Step {}: Press Enter to continue with "
815 "the next step...".format(i) + os.linesep + os.linesep)
817 elif step[0] == 'sleep':
818 self._logger.debug("Sleep %s seconds", step[1])
819 time.sleep(int(step[1]))
821 elif step[0] == 'log':
822 test_object = self._logger
823 # there isn't a need for validation of log entry
826 elif step[0] == 'pdb':
831 self._logger.error("Unsupported test object %s", step[0])
832 self._step_status = {'status' : False, 'details' : ' '.join(step)}
833 self.step_report_status("Step '{}'".format(' '.join(step)),
834 self._step_status['status'])
837 test_method = getattr(test_object, step[1])
839 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
841 test_method_check = None
845 # eval parameters, but use only valid step_results
846 # to support negative indexes
847 step_params = self.step_eval_params(step[2:], self._step_result[:i])
848 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
849 step_log += ' filter "{}"'.format(regex) if regex else ''
850 self._logger.debug("Step %s '%s' start", i, step_log)
851 self._step_result[i] = test_method(*step_params)
853 # apply regex to step output
854 self._step_result[i] = functions.filter_output(
855 self._step_result[i], regex)
857 self._logger.debug("Step %s '%s' results '%s'", i,
858 step_log, self._step_result[i])
859 time.sleep(S.getValue('TEST_STEP_DELAY'))
861 step_ok = test_method_check(self._step_result[i], *step_params)
862 except (AssertionError, AttributeError, IndexError) as ex:
864 self._logger.error("Step %s raised %s", i, type(ex).__name__)
867 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
870 self._step_status = {'status' : False, 'details' : step_log}
871 # Stop all VNFs started by TestSteps
872 self.step_stop_vnfs()
875 # all steps processed without any issue
879 # get methods for TestCase members, which needs to be publicly available
881 def get_output_file(self):
882 """Return content of self._output_file member
884 return self._output_file
887 """Return content of self.desc member
891 def get_versions(self):
892 """Return content of self.versions member
894 return self._versions
896 def get_traffic(self):
897 """Return content of self._traffic member
901 def get_tc_results(self):
902 """Return content of self._tc_results member
904 return self._tc_results
906 def get_collector(self):
907 """Return content of self._collector member
909 return self._collector