1 # Copyright 2015-2018 Intel Corporation, Tieto and others.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """TestCase base class
17 from collections import OrderedDict
27 from datetime import datetime as dt
28 from conf import settings as S
29 from conf import merge_spec
30 import core.component_factory as component_factory
31 from core.loader import Loader
32 from core.results.results_constants import ResultsConstants
33 from tools import tasks
34 from tools import hugepages
35 from tools import functions
36 from tools import namespace
37 from tools import veth
38 from tools.teststepstools import TestStepsTools
39 from tools.llc_management import rmd
41 CHECK_PREFIX = 'validate_'
43 # pylint: disable=too-many-instance-attributes
44 class TestCase(object):
45 """TestCase base class
47 In this basic form runs RFC2544 throughput test
49 # pylint: disable=too-many-statements
50 def __init__(self, test_cfg):
51 """Pull out fields from test config
53 :param test_cfg: A dictionary of string-value pairs describing the test
54 configuration. Both the key and values strings use well-known
56 :param results_dir: Where the csv formatted results are written.
58 # make a local copy of test configuration to avoid modification of
59 # original content used in vsperf main script
60 cfg = copy.deepcopy(test_cfg)
62 self._testcase_start_time = time.time()
63 self._testcase_stop_time = self._testcase_start_time
64 self._hugepages_mounted = False
65 self._traffic_ctl = None
67 self._vswitch_ctl = None
68 self._collector = None
70 self._output_file = None
71 self._tc_results = None
72 self._settings_paths_modified = False
73 self._testcast_run_time = None
75 # initialization of step driven specific members
76 self._step_check = False # by default don't check result for step driven testcases
77 self._step_vnf_list = {}
78 self._step_result = []
79 self._step_result_mapping = {}
80 self._step_status = None
81 self._step_send_traffic = False # indication if send_traffic was called within test steps
83 self._testcase_run_time = None
85 S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
86 S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
87 S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
88 S.setValue('TUNNEL_TYPE', cfg.get('Tunnel Type', S.getValue('TUNNEL_TYPE')))
89 test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
90 tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
91 test_params = merge_spec(test_params, tc_test_params)
92 S.setValue('TEST_PARAMS', test_params)
95 # override all redefined GUEST_ values to have them expanded correctly
96 tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
97 for key in tmp_test_params:
98 if key.startswith('GUEST_'):
99 S.setValue(key, S.getValue(key))
100 S.getValue('TEST_PARAMS').pop(key)
102 # update global settings
103 functions.settings_update_paths()
105 # set test parameters; CLI options take precedence to testcase settings
106 self._logger = logging.getLogger(__name__)
107 self.name = cfg['Name']
108 self.desc = cfg.get('Description', 'No description given.')
109 self.test = cfg.get('TestSteps', None)
111 # log testcase name and details
112 tmp_desc = functions.format_description(self.desc, 50)
113 self._logger.info('############################################################')
114 self._logger.info('# Test: %s', self.name)
115 self._logger.info('# Details: %s', tmp_desc[0])
116 for i in range(1, len(tmp_desc)):
117 self._logger.info('# %s', tmp_desc[i])
118 self._logger.info('############################################################')
120 bidirectional = S.getValue('TRAFFIC')['bidir']
121 if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
123 'Bi-dir value must be of type string')
124 bidirectional = bidirectional.title() # Keep things consistent
126 self.deployment = cfg['Deployment']
127 self._frame_mod = cfg.get('Frame Modification', None)
129 self._tunnel_operation = cfg.get('Tunnel Operation', None)
131 # check if test requires background load and which generator it uses
132 self._load_cfg = cfg.get('Load', None)
135 self._frame_mod = self._frame_mod.lower()
136 self._results_dir = S.getValue('RESULTS_PATH')
138 # set traffic details, so they can be passed to vswitch and traffic ctls
139 self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
140 self._traffic.update({'bidir': bidirectional})
142 # Packet Forwarding mode
143 self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
145 # trafficgen configuration required for tests of tunneling protocols
146 if self._tunnel_operation:
147 self._traffic.update({'tunnel_type': S.getValue('TUNNEL_TYPE')})
148 self._traffic['l2'].update({'srcmac':
149 S.getValue('TRAFFICGEN_PORT1_MAC'),
151 S.getValue('TRAFFICGEN_PORT2_MAC')})
153 self._traffic['l3'].update({'srcip':
154 S.getValue('TRAFFICGEN_PORT1_IP'),
156 S.getValue('TRAFFICGEN_PORT2_IP')})
158 if self._tunnel_operation == "decapsulation":
159 self._traffic['l2'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L2'))
160 self._traffic['l3'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L3'))
161 self._traffic['l4'].update(S.getValue(S.getValue('TUNNEL_TYPE').upper() + '_FRAME_L4'))
162 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
163 elif len(S.getValue('NICS')) >= 2 and \
164 (S.getValue('NICS')[0]['type'] == 'vf' or
165 S.getValue('NICS')[1]['type'] == 'vf'):
166 mac1 = S.getValue('NICS')[0]['mac']
167 mac2 = S.getValue('NICS')[1]['mac']
169 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
171 self._logger.debug("MAC addresses can not be read")
173 self._traffic = functions.check_traffic(self._traffic)
175 # count how many VNFs are involved in TestSteps
177 for step in self.test:
178 if step[0].startswith('vnf'):
179 self._step_vnf_list[step[0]] = None
181 # if llc allocation is required, initialize it.
182 if S.getValue('LLC_ALLOCATION'):
183 self._rmd = rmd.CacheAllocator()
185 def run_initialize(self):
186 """ Prepare test execution environment
188 # mount hugepages if needed
189 self._mount_hugepages()
191 self._logger.debug("Controllers:")
193 self._traffic_ctl = component_factory.create_traffic(
194 self._traffic['traffic_type'],
195 loader.get_trafficgen_class())
197 self._vnf_ctl = component_factory.create_vnf(
199 loader.get_vnf_class(),
200 len(self._step_vnf_list))
202 self._vnf_list = self._vnf_ctl.get_vnfs()
204 # verify enough hugepages are free to run the testcase
205 if not self._check_for_enough_hugepages():
206 raise RuntimeError('Not enough hugepages free to run test.')
208 # perform guest related handling
209 tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
211 # copy sources of l2 forwarding tools into VM shared dir if needed
212 self._copy_fwd_tools_for_all_guests(tmp_vm_count)
214 # in case of multi VM in parallel, set the number of streams to the number of VMs
215 if self.deployment.startswith('pvpv'):
216 # for each VM NIC pair we need an unique stream
218 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
219 streams += int(vm_nic / 2) if vm_nic > 1 else 1
220 self._logger.debug("VMs with parallel connection were detected. "
221 "Thus Number of streams was set to %s", streams)
222 # update streams if needed; In case of additional VNFs deployed by TestSteps
223 # user can define a proper stream count manually
224 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
225 self._traffic.update({'multistream': streams})
227 # OVS Vanilla requires guest VM MAC address and IPs to work
228 if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
229 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
230 'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
231 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
232 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
234 if self._vswitch_none:
235 self._vswitch_ctl = component_factory.create_pktfwd(
237 loader.get_pktfwd_class())
239 self._vswitch_ctl = component_factory.create_vswitch(
241 loader.get_vswitch_class(),
243 self._tunnel_operation)
245 self._collector = component_factory.create_collector(
246 loader.get_collector_class(),
247 self._results_dir, self.name)
248 self._loadgen = component_factory.create_loadgen(
249 loader.get_loadgen_class(),
252 self._output_file = os.path.join(self._results_dir, "result_{}_{}_{}.csv".format(
253 str(S.getValue('_TEST_INDEX')), self.name, self.deployment))
255 self._step_status = {'status' : True, 'details' : ''}
257 # Perform LLC-allocations
258 if S.getValue('LLC_ALLOCATION'):
259 self._rmd.setup_llc_allocation()
261 self._logger.debug("Setup:")
263 def run_finalize(self):
264 """ Tear down test execution environment and record test results
266 # Stop all VNFs started by TestSteps in case that something went wrong
267 self.step_stop_vnfs()
269 # Cleanup any LLC-allocations
270 if S.getValue('LLC_ALLOCATION'):
271 self._rmd.cleanup_llc_allocation()
273 # Stop all processes executed by testcase
274 tasks.terminate_all_tasks(self._logger)
276 # umount hugepages if mounted
277 self._umount_hugepages()
279 # cleanup any namespaces created
280 if os.path.isdir('/tmp/namespaces'):
281 namespace_list = os.listdir('/tmp/namespaces')
283 self._logger.info('Cleaning up namespaces')
284 for name in namespace_list:
285 namespace.delete_namespace(name)
286 os.rmdir('/tmp/namespaces')
287 # cleanup any veth ports created
288 if os.path.isdir('/tmp/veth'):
289 veth_list = os.listdir('/tmp/veth')
291 self._logger.info('Cleaning up veth ports')
292 for eth in veth_list:
293 port1, port2 = eth.split('-')
294 veth.del_veth_port(port1, port2)
295 os.rmdir('/tmp/veth')
297 def run_report(self):
298 """ Report test results
300 self._logger.debug("self._collector Results:")
301 self._collector.print_results()
303 results = self._traffic_ctl.get_results()
305 self._logger.debug("Traffic Results:")
306 self._traffic_ctl.print_results()
308 if self._tc_results is None:
309 self._tc_results = self._append_results(results)
311 # integration step driven tests have their status and possible
312 # failure details stored inside self._tc_results
313 results = self._append_results(results)
314 if len(self._tc_results) < len(results):
315 if len(self._tc_results) > 1:
316 raise RuntimeError('Testcase results do not match:'
318 'trafficgen results: %s\n' %
322 tmp_results = copy.deepcopy(self._tc_results[0])
323 self._tc_results = []
325 tmp_res = copy.deepcopy(tmp_results)
327 self._tc_results.append(tmp_res)
329 for i, result in enumerate(results):
330 self._tc_results[i].update(result)
332 TestCase.write_result_to_file(self._tc_results, self._output_file)
337 All setup and teardown through controllers is included.
339 # prepare test execution environment
340 self.run_initialize()
343 with self._vswitch_ctl:
344 with self._vnf_ctl, self._collector, self._loadgen:
345 if not self._vswitch_none:
348 self._versions += self._vswitch_ctl.get_vswitch().get_version()
350 with self._traffic_ctl:
351 # execute test based on TestSteps definition if needed...
353 # ...and continue with traffic generation, but keep
354 # in mind, that clean deployment does not configure
355 # OVS nor executes the traffic
356 if self.deployment != 'clean' and not self._step_send_traffic:
357 self._traffic_ctl.send_traffic(self._traffic)
359 # dump vswitch flows before they are affected by VNF termination
360 if not self._vswitch_none:
361 self._vswitch_ctl.dump_vswitch_flows()
363 # garbage collection for case that TestSteps modify existing deployment
364 self.step_stop_vnfs()
367 # tear down test execution environment and log results
370 self._testcase_stop_time = time.time()
371 self._testcase_run_time = time.strftime("%H:%M:%S",
372 time.gmtime(self._testcase_stop_time -
373 self._testcase_start_time))
374 logging.info("Testcase execution time: %s", self._testcase_run_time)
375 # report test results
378 def _append_results(self, results):
380 Method appends mandatory Test Case results to list of dictionaries.
382 :param results: list of dictionaries which contains results from
385 :returns: modified list of dictionaries.
388 item[ResultsConstants.ID] = self.name
389 item[ResultsConstants.DEPLOYMENT] = self.deployment
390 item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
391 item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
392 item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
393 # convert timestamps to human readable format
394 item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
395 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
396 item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
397 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
398 if self._traffic['multistream']:
399 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
400 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
401 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
402 if self._vnf_ctl.get_vnfs_number():
403 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
404 if self._tunnel_operation:
405 item[ResultsConstants.TUNNEL_TYPE] = S.getValue('TUNNEL_TYPE')
408 def _copy_fwd_tools_for_all_guests(self, vm_count):
409 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
411 # consider only VNFs involved in the test
412 for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
413 self._copy_fwd_tools_for_guest(guest_dir)
415 def _copy_fwd_tools_for_guest(self, guest_dir):
416 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
418 :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
420 # remove shared dir if it exists to avoid issues with file consistency
421 if os.path.exists(guest_dir):
422 tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
423 'Removing content of shared directory...', True)
425 # directory to share files between host and guest
426 os.makedirs(guest_dir)
428 # copy sources into shared dir only if neccessary
429 guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
430 if 'testpmd' in guest_loopback:
432 # exclude whole .git/ subdirectory and all o-files;
433 # It is assumed, that the same RTE_TARGET is used in both host
434 # and VMs; This simplification significantly speeds up testpmd
435 # build. If we will need a different RTE_TARGET in VM,
436 # then we have to build whole DPDK from the scratch in VM.
437 # In that case we can copy just DPDK sources (e.g. by excluding
438 # all items obtained by git status -unormal --porcelain).
439 # NOTE: Excluding RTE_TARGET directory won't help on systems,
440 # where DPDK is built for multiple targets (e.g. for gcc & icc)
442 exclude.append(r'--exclude=.git/')
443 exclude.append(r'--exclude=*.o')
444 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
445 [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
446 os.path.join(guest_dir, 'DPDK')],
448 'Copying DPDK to shared directory...',
450 except subprocess.CalledProcessError:
451 self._logger.error('Unable to copy DPDK to shared directory')
453 if 'l2fwd' in guest_loopback:
455 tasks.run_task(['rsync', '-a', '-r', '-l',
456 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
457 os.path.join(guest_dir, 'l2fwd')],
459 'Copying l2fwd to shared directory...',
461 except subprocess.CalledProcessError:
462 self._logger.error('Unable to copy l2fwd to shared directory')
465 def _mount_hugepages(self):
466 """Mount hugepages if usage of DPDK or Qemu is detected
468 # pylint: disable=too-many-boolean-expressions
469 # hugepages are needed by DPDK and Qemu
470 if not self._hugepages_mounted and \
471 (self.deployment.count('v') or \
472 str(S.getValue('VSWITCH')).lower().count('dpdk') or \
473 self._vswitch_none or \
474 self.test and 'vnf' in [step[0][0:3] for step in self.test]):
475 hugepages.mount_hugepages()
476 self._hugepages_mounted = True
478 def _umount_hugepages(self):
479 """Umount hugepages if they were mounted before
481 if self._hugepages_mounted:
482 hugepages.umount_hugepages()
483 self._hugepages_mounted = False
485 def _check_for_enough_hugepages(self):
486 """Check to make sure enough hugepages are free to satisfy the
490 hugepage_size = hugepages.get_hugepage_size()
491 # get hugepage amounts per guest involved in the test
492 for guest in range(self._vnf_ctl.get_vnfs_number()):
493 hugepages_needed += math.ceil((int(S.getValue(
494 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
496 # get hugepage amounts for each socket on dpdk
497 sock0_mem, sock1_mem = 0, 0
499 if str(S.getValue('VSWITCH')).lower().count('dpdk'):
500 sock_mem = S.getValue('DPDK_SOCKET_MEM')
501 sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
502 int(sock_mem[1]) * 1024 / hugepage_size)
504 # If hugepages needed, verify the amounts are free
505 if any([hugepages_needed, sock0_mem, sock1_mem]):
506 free_hugepages = hugepages.get_free_hugepages()
508 logging.info('Need %s hugepages free for guests',
510 result1 = free_hugepages >= hugepages_needed
511 free_hugepages -= hugepages_needed
516 logging.info('Need %s hugepages free for dpdk socket 0',
518 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
519 free_hugepages -= sock0_mem
524 logging.info('Need %s hugepages free for dpdk socket 1',
526 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
527 free_hugepages -= sock1_mem
531 logging.info('Need a total of %s total hugepages',
532 hugepages_needed + sock1_mem + sock0_mem)
534 # The only drawback here is sometimes dpdk doesn't release
535 # its hugepages on a test failure. This could cause a test
536 # to fail when dpdk would be OK to start because it will just
537 # use the previously allocated hugepages.
538 result4 = True if free_hugepages >= 0 else False
540 return all([result1, result2, result3, result4])
545 def write_result_to_file(results, output):
546 """Write list of dictionaries to a CSV file.
548 Each element on list will create separate row in output file.
549 If output file already exists, data will be appended at the end,
550 otherwise it will be created.
552 :param results: list of dictionaries.
553 :param output: path to output file.
555 with open(output, 'a') as csvfile:
557 logging.info("Write results to file: %s", output)
558 fieldnames = TestCase._get_unique_keys(results)
560 writer = csv.DictWriter(csvfile, fieldnames)
562 if not csvfile.tell(): # file is now empty
565 for result in results:
566 writer.writerow(result)
569 def _get_unique_keys(list_of_dicts):
570 """Gets unique key values as ordered list of strings in given dicts
572 :param list_of_dicts: list of dictionaries.
574 :returns: list of unique keys(strings).
576 result = OrderedDict()
577 for item in list_of_dicts:
578 for key in item.keys():
581 return list(result.keys())
583 def _add_flows(self):
584 """Add flows to the vswitch
586 vswitch = self._vswitch_ctl.get_vswitch()
587 # NOTE BOM 15-08-07 the frame mod code assumes that the
588 # physical ports are ports 1 & 2. The actual numbers
589 # need to be retrived from the vSwitch and the metadata value
590 # updated accordingly.
591 bridge = S.getValue('VSWITCH_BRIDGE_NAME')
592 if self._frame_mod == "vlan":
593 # 0x8100 => VLAN ethertype
594 self._logger.debug(" **** VLAN ***** ")
595 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
596 'actions': ['push_vlan:0x8100', 'goto_table:3']}
597 vswitch.add_flow(bridge, flow)
598 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
599 'actions': ['push_vlan:0x8100', 'goto_table:3']}
600 vswitch.add_flow(bridge, flow)
601 elif self._frame_mod == "mpls":
602 # 0x8847 => MPLS unicast ethertype
603 self._logger.debug(" **** MPLS ***** ")
604 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
605 'actions': ['push_mpls:0x8847', 'goto_table:3']}
606 vswitch.add_flow(bridge, flow)
607 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
608 'actions': ['push_mpls:0x8847', 'goto_table:3']}
609 vswitch.add_flow(bridge, flow)
610 elif self._frame_mod == "mac":
611 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
612 'actions': ['mod_dl_src:22:22:22:22:22:22',
614 vswitch.add_flow(bridge, flow)
615 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
616 'actions': ['mod_dl_src:11:11:11:11:11:11',
618 vswitch.add_flow(bridge, flow)
619 elif self._frame_mod == "dscp":
620 # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
621 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
623 'actions': ['mod_nw_tos:184', 'goto_table:3']}
624 vswitch.add_flow(bridge, flow)
625 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
627 'actions': ['mod_nw_tos:184', 'goto_table:3']}
628 vswitch.add_flow(bridge, flow)
629 elif self._frame_mod == "ttl":
630 # 251 and 241 are the highest prime numbers < 255
631 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
633 'actions': ['mod_nw_ttl:251', 'goto_table:3']}
634 vswitch.add_flow(bridge, flow)
635 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
637 'actions': ['mod_nw_ttl:241', 'goto_table:3']}
638 vswitch.add_flow(bridge, flow)
639 elif self._frame_mod == "ip_addr":
640 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
642 'actions': ['mod_nw_src:10.10.10.10',
643 'mod_nw_dst:20.20.20.20',
645 vswitch.add_flow(bridge, flow)
646 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
648 'actions': ['mod_nw_src:20.20.20.20',
649 'mod_nw_dst:10.10.10.10',
651 vswitch.add_flow(bridge, flow)
652 elif self._frame_mod == "ip_port":
653 # NOTE BOM 15-08-27 The traffic generated is assumed
654 # to be UDP (nw_proto 17d) which is the default case but
655 # we will need to pick up the actual traffic params in use.
656 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
657 'dl_type':'0x0800', 'nw_proto':'17',
658 'actions': ['mod_tp_src:44444',
659 'mod_tp_dst:44444', 'goto_table:3']}
660 vswitch.add_flow(bridge, flow)
661 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
662 'dl_type':'0x0800', 'nw_proto':'17',
663 'actions': ['mod_tp_src:44444',
664 'mod_tp_dst:44444', 'goto_table:3']}
665 vswitch.add_flow(bridge, flow)
671 # TestSteps realted methods
673 def step_report_status(self, label, status):
674 """ Log status of test step
676 self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
678 def step_stop_vnfs(self):
679 """ Stop all VNFs started by TestSteps
681 for vnf in self._step_vnf_list:
682 if self._step_vnf_list[vnf]:
683 self._step_vnf_list[vnf].stop()
685 def step_eval_param(self, param, step_result):
686 """ Helper function for #STEP macro evaluation
688 if isinstance(param, str):
689 # evaluate every #STEP reference inside parameter itself
690 macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
694 if macro[1] in self._step_result_mapping:
695 key = self._step_result_mapping[macro[1]]
698 # pylint: disable=eval-used
699 tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
700 param = param.replace(macro[0], tmp_val)
702 # evaluate references to vsperf configuration options
703 macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
706 # pylint: disable=eval-used
708 tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
709 param = param.replace('${}'.format(macro[0]), tmp_val)
710 # ignore that required option can't be evaluated
711 except (IndexError, KeyError, AttributeError):
712 self._logger.debug("Skipping %s as it isn't a configuration "
713 "parameter.", '${}'.format(macro[0]))
715 elif isinstance(param, (list, tuple)):
718 tmp_list.append(self.step_eval_param(item, step_result))
720 elif isinstance(param, dict):
722 for (key, value) in param.items():
723 tmp_dict[key] = self.step_eval_param(value, step_result)
728 def step_eval_params(self, params, step_result):
729 """ Evaluates referrences to results from previous steps
732 # evaluate all parameters if needed
734 eval_params.append(self.step_eval_param(param, step_result))
737 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
739 """ Execute actions specified by TestSteps list
741 :return: False if any error was detected
748 # required for VNFs initialization
750 # initialize list with results
751 self._step_result = [None] * len(self.test)
753 # run test step by step...
754 for i, step in enumerate(self.test):
755 step_ok = not self._step_check
756 step_check = self._step_check
758 # configure step result mapping if step alias/label is detected
759 if step[0].startswith('#'):
762 raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
763 if key in self._step_result_mapping:
764 raise RuntimeError('Step alias {} has been used already for step '
765 '{}'.format(key, self._step_result_mapping[key]))
766 self._step_result_mapping[step[0][1:]] = i
769 # store regex filter if it is specified
770 if isinstance(step[-1], str) and step[-1].startswith('|'):
771 # evalute macros and variables used in regex
772 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
775 # check if step verification should be suppressed
776 if step[0].startswith('!'):
779 step[0] = step[0][1:]
780 if step[0] == 'vswitch':
781 test_object = self._vswitch_ctl.get_vswitch()
782 elif step[0] == 'namespace':
783 test_object = namespace
784 elif step[0] == 'veth':
786 elif step[0] == 'settings':
788 elif step[0] == 'tools':
789 test_object = TestStepsTools()
790 step[1] = step[1].title()
791 elif step[0] == 'trafficgen':
792 test_object = self._traffic_ctl
793 # in case of send_traffic or send_traffic_async methods, ensure
794 # that specified traffic values are merged with existing self._traffic
795 if step[1].startswith('send_traffic'):
796 tmp_traffic = copy.deepcopy(self._traffic)
797 tmp_traffic.update(step[2])
798 step[2] = tmp_traffic
799 # store indication that traffic has been sent
800 # so it is not sent again after the execution of teststeps
801 self._step_send_traffic = True
802 elif step[0].startswith('vnf'):
803 # use vnf started within TestSteps
804 if not self._step_vnf_list[step[0]]:
806 self._step_vnf_list[step[0]] = loader.get_vnf_class()()
807 test_object = self._step_vnf_list[step[0]]
808 elif step[0].startswith('VNF'):
809 if step[1] in ('start', 'stop'):
810 raise RuntimeError("Cannot execute start() or stop() method of "
811 "VNF deployed automatically by scenario.")
812 # use vnf started by scenario deployment (e.g. pvp)
813 vnf_index = int(step[0][3:])
815 test_object = self._vnf_list[vnf_index]
817 raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
818 elif step[0] == 'wait':
819 input(os.linesep + "Step {}: Press Enter to continue with "
820 "the next step...".format(i) + os.linesep + os.linesep)
822 elif step[0] == 'sleep':
823 self._logger.debug("Sleep %s seconds", step[1])
824 time.sleep(int(step[1]))
826 elif step[0] == 'log':
827 test_object = self._logger
828 # there isn't a need for validation of log entry
831 elif step[0] == 'pdb':
836 self._logger.error("Unsupported test object %s", step[0])
837 self._step_status = {'status' : False, 'details' : ' '.join(step)}
838 self.step_report_status("Step '{}'".format(' '.join(step)),
839 self._step_status['status'])
842 test_method = getattr(test_object, step[1])
844 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
846 test_method_check = None
850 # eval parameters, but use only valid step_results
851 # to support negative indexes
852 step_params = self.step_eval_params(step[2:], self._step_result[:i])
853 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
854 step_log += ' filter "{}"'.format(regex) if regex else ''
855 self._logger.debug("Step %s '%s' start", i, step_log)
856 self._step_result[i] = test_method(*step_params)
858 # apply regex to step output
859 self._step_result[i] = functions.filter_output(
860 self._step_result[i], regex)
862 self._logger.debug("Step %s '%s' results '%s'", i,
863 step_log, self._step_result[i])
864 time.sleep(S.getValue('TEST_STEP_DELAY'))
866 step_ok = test_method_check(self._step_result[i], *step_params)
867 except (AssertionError, AttributeError, IndexError) as ex:
869 self._logger.error("Step %s raised %s", i, type(ex).__name__)
872 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
875 self._step_status = {'status' : False, 'details' : step_log}
876 # Stop all VNFs started by TestSteps
877 self.step_stop_vnfs()
880 # all steps processed without any issue
884 # get methods for TestCase members, which needs to be publicly available
886 def get_output_file(self):
887 """Return content of self._output_file member
889 return self._output_file
892 """Return content of self.desc member
896 def get_versions(self):
897 """Return content of self.versions member
899 return self._versions
901 def get_traffic(self):
902 """Return content of self._traffic member
906 def get_tc_results(self):
907 """Return content of self._tc_results member
909 return self._tc_results
911 def get_collector(self):
912 """Return content of self._collector member
914 return self._collector