1 # Copyright 2015-2017 Intel Corporation.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """TestCase base class
17 from collections import OrderedDict
27 from datetime import datetime as dt
28 from conf import settings as S
29 from conf import get_test_param, merge_spec
30 import core.component_factory as component_factory
31 from core.loader import Loader
32 from core.results.results_constants import ResultsConstants
33 from tools import tasks
34 from tools import hugepages
35 from tools import functions
36 from tools import namespace
37 from tools import veth
38 from tools.teststepstools import TestStepsTools
39 from tools.llc_management import rmd
41 CHECK_PREFIX = 'validate_'
43 # pylint: disable=too-many-instance-attributes
44 class TestCase(object):
45 """TestCase base class
47 In this basic form runs RFC2544 throughput test
49 # pylint: disable=too-many-statements
50 def __init__(self, test_cfg):
51 """Pull out fields from test config
53 :param test_cfg: A dictionary of string-value pairs describing the test
54 configuration. Both the key and values strings use well-known
56 :param results_dir: Where the csv formatted results are written.
58 # make a local copy of test configuration to avoid modification of
59 # original content used in vsperf main script
60 cfg = copy.deepcopy(test_cfg)
62 self._testcase_start_time = time.time()
63 self._testcase_stop_time = self._testcase_start_time
64 self._hugepages_mounted = False
65 self._traffic_ctl = None
67 self._vswitch_ctl = None
68 self._collector = None
70 self._output_file = None
71 self._tc_results = None
72 self._settings_paths_modified = False
73 self._testcast_run_time = None
75 # initialization of step driven specific members
76 self._step_check = False # by default don't check result for step driven testcases
77 self._step_vnf_list = {}
78 self._step_result = []
79 self._step_result_mapping = {}
80 self._step_status = None
81 self._step_send_traffic = False # indication if send_traffic was called within test steps
83 self._testcase_run_time = None
85 S.setValue('VSWITCH', cfg.get('vSwitch', S.getValue('VSWITCH')))
86 S.setValue('VNF', cfg.get('VNF', S.getValue('VNF')))
87 S.setValue('TRAFFICGEN', cfg.get('Trafficgen', S.getValue('TRAFFICGEN')))
88 test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
89 tc_test_params = cfg.get('Parameters', S.getValue('TEST_PARAMS'))
90 test_params = merge_spec(test_params, tc_test_params)
91 S.setValue('TEST_PARAMS', test_params)
94 # override all redefined GUEST_ values to have them expanded correctly
95 tmp_test_params = copy.deepcopy(S.getValue('TEST_PARAMS'))
96 for key in tmp_test_params:
97 if key.startswith('GUEST_'):
98 S.setValue(key, S.getValue(key))
99 S.getValue('TEST_PARAMS').pop(key)
101 # update global settings
102 functions.settings_update_paths()
104 # set test parameters; CLI options take precedence to testcase settings
105 self._logger = logging.getLogger(__name__)
106 self.name = cfg['Name']
107 self.desc = cfg.get('Description', 'No description given.')
108 self.test = cfg.get('TestSteps', None)
110 # log testcase name and details
111 tmp_desc = functions.format_description(self.desc, 50)
112 self._logger.info('############################################################')
113 self._logger.info('# Test: %s', self.name)
114 self._logger.info('# Details: %s', tmp_desc[0])
115 for i in range(1, len(tmp_desc)):
116 self._logger.info('# %s', tmp_desc[i])
117 self._logger.info('############################################################')
119 bidirectional = S.getValue('TRAFFIC')['bidir']
120 if not isinstance(S.getValue('TRAFFIC')['bidir'], str):
122 'Bi-dir value must be of type string')
123 bidirectional = bidirectional.title() # Keep things consistent
125 self.deployment = cfg['Deployment']
126 self._frame_mod = cfg.get('Frame Modification', None)
128 self._tunnel_type = None
129 self._tunnel_operation = None
131 if self.deployment == 'op2p':
132 self._tunnel_operation = cfg['Tunnel Operation']
134 if 'Tunnel Type' in cfg:
135 self._tunnel_type = cfg['Tunnel Type']
136 self._tunnel_type = get_test_param('TUNNEL_TYPE',
139 # check if test requires background load and which generator it uses
140 self._load_cfg = cfg.get('Load', None)
143 self._frame_mod = self._frame_mod.lower()
144 self._results_dir = S.getValue('RESULTS_PATH')
146 # set traffic details, so they can be passed to vswitch and traffic ctls
147 self._traffic = copy.deepcopy(S.getValue('TRAFFIC'))
148 self._traffic.update({'bidir': bidirectional,
149 'tunnel_type': self._tunnel_type,})
151 self._traffic = functions.check_traffic(self._traffic)
153 # Packet Forwarding mode
154 self._vswitch_none = str(S.getValue('VSWITCH')).strip().lower() == 'none'
156 # trafficgen configuration required for tests of tunneling protocols
157 if self.deployment == "op2p":
158 self._traffic['l2'].update({'srcmac':
159 S.getValue('TRAFFICGEN_PORT1_MAC'),
161 S.getValue('TRAFFICGEN_PORT2_MAC')})
163 self._traffic['l3'].update({'srcip':
164 S.getValue('TRAFFICGEN_PORT1_IP'),
166 S.getValue('TRAFFICGEN_PORT2_IP')})
168 if self._tunnel_operation == "decapsulation":
169 self._traffic['l2'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L2')
170 self._traffic['l3'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L3')
171 self._traffic['l4'] = S.getValue(self._tunnel_type.upper() + '_FRAME_L4')
172 self._traffic['l2']['dstmac'] = S.getValue('NICS')[1]['mac']
173 elif len(S.getValue('NICS')) >= 2 and \
174 (S.getValue('NICS')[0]['type'] == 'vf' or
175 S.getValue('NICS')[1]['type'] == 'vf'):
176 mac1 = S.getValue('NICS')[0]['mac']
177 mac2 = S.getValue('NICS')[1]['mac']
179 self._traffic['l2'].update({'srcmac': mac2, 'dstmac': mac1})
181 self._logger.debug("MAC addresses can not be read")
183 # count how many VNFs are involved in TestSteps
185 for step in self.test:
186 if step[0].startswith('vnf'):
187 self._step_vnf_list[step[0]] = None
189 # if llc allocation is required, initialize it.
190 if S.getValue('LLC_ALLOCATION'):
191 self._rmd = rmd.CacheAllocator()
193 def run_initialize(self):
194 """ Prepare test execution environment
196 # mount hugepages if needed
197 self._mount_hugepages()
199 self._logger.debug("Controllers:")
201 self._traffic_ctl = component_factory.create_traffic(
202 self._traffic['traffic_type'],
203 loader.get_trafficgen_class())
205 self._vnf_ctl = component_factory.create_vnf(
207 loader.get_vnf_class(),
208 len(self._step_vnf_list))
210 self._vnf_list = self._vnf_ctl.get_vnfs()
212 # verify enough hugepages are free to run the testcase
213 if not self._check_for_enough_hugepages():
214 raise RuntimeError('Not enough hugepages free to run test.')
216 # perform guest related handling
217 tmp_vm_count = self._vnf_ctl.get_vnfs_number() + len(self._step_vnf_list)
219 # copy sources of l2 forwarding tools into VM shared dir if needed
220 self._copy_fwd_tools_for_all_guests(tmp_vm_count)
222 # in case of multi VM in parallel, set the number of streams to the number of VMs
223 if self.deployment.startswith('pvpv'):
224 # for each VM NIC pair we need an unique stream
226 for vm_nic in S.getValue('GUEST_NICS_NR')[:tmp_vm_count]:
227 streams += int(vm_nic / 2) if vm_nic > 1 else 1
228 self._logger.debug("VMs with parallel connection were detected. "
229 "Thus Number of streams was set to %s", streams)
230 # update streams if needed; In case of additional VNFs deployed by TestSteps
231 # user can define a proper stream count manually
232 if 'multistream' not in self._traffic or self._traffic['multistream'] < streams:
233 self._traffic.update({'multistream': streams})
235 # OVS Vanilla requires guest VM MAC address and IPs to work
236 if 'linux_bridge' in S.getValue('GUEST_LOOPBACK'):
237 self._traffic['l2'].update({'srcmac': S.getValue('VANILLA_TGEN_PORT1_MAC'),
238 'dstmac': S.getValue('VANILLA_TGEN_PORT2_MAC')})
239 self._traffic['l3'].update({'srcip': S.getValue('VANILLA_TGEN_PORT1_IP'),
240 'dstip': S.getValue('VANILLA_TGEN_PORT2_IP')})
242 if self._vswitch_none:
243 self._vswitch_ctl = component_factory.create_pktfwd(
245 loader.get_pktfwd_class())
247 self._vswitch_ctl = component_factory.create_vswitch(
249 loader.get_vswitch_class(),
251 self._tunnel_operation)
253 self._collector = component_factory.create_collector(
254 loader.get_collector_class(),
255 self._results_dir, self.name)
256 self._loadgen = component_factory.create_loadgen(
257 loader.get_loadgen_class(),
260 self._output_file = os.path.join(self._results_dir, "result_{}_{}_{}.csv".format(
261 str(S.getValue('_TEST_INDEX')), self.name, self.deployment))
263 self._step_status = {'status' : True, 'details' : ''}
265 # Perform LLC-allocations
266 if S.getValue('LLC_ALLOCATION'):
267 self._rmd.setup_llc_allocation()
269 self._logger.debug("Setup:")
271 def run_finalize(self):
272 """ Tear down test execution environment and record test results
274 # Stop all VNFs started by TestSteps in case that something went wrong
275 self.step_stop_vnfs()
277 # Cleanup any LLC-allocations
278 if S.getValue('LLC_ALLOCATION'):
279 self._rmd.cleanup_llc_allocation()
281 # Stop all processes executed by testcase
282 tasks.terminate_all_tasks(self._logger)
284 # umount hugepages if mounted
285 self._umount_hugepages()
287 # cleanup any namespaces created
288 if os.path.isdir('/tmp/namespaces'):
289 namespace_list = os.listdir('/tmp/namespaces')
290 if len(namespace_list):
291 self._logger.info('Cleaning up namespaces')
292 for name in namespace_list:
293 namespace.delete_namespace(name)
294 os.rmdir('/tmp/namespaces')
295 # cleanup any veth ports created
296 if os.path.isdir('/tmp/veth'):
297 veth_list = os.listdir('/tmp/veth')
299 self._logger.info('Cleaning up veth ports')
300 for eth in veth_list:
301 port1, port2 = eth.split('-')
302 veth.del_veth_port(port1, port2)
303 os.rmdir('/tmp/veth')
305 def run_report(self):
306 """ Report test results
308 self._logger.debug("self._collector Results:")
309 self._collector.print_results()
311 results = self._traffic_ctl.get_results()
313 self._logger.debug("Traffic Results:")
314 self._traffic_ctl.print_results()
316 if self._tc_results is None:
317 self._tc_results = self._append_results(results)
319 # integration step driven tests have their status and possible
320 # failure details stored inside self._tc_results
321 results = self._append_results(results)
322 if len(self._tc_results) < len(results):
323 if len(self._tc_results) > 1:
324 raise RuntimeError('Testcase results do not match:'
326 'trafficgen results: {}\n',
330 tmp_results = copy.deepcopy(self._tc_results[0])
331 self._tc_results = []
333 tmp_res = copy.deepcopy(tmp_results)
335 self._tc_results.append(tmp_res)
337 for i, result in enumerate(results):
338 self._tc_results[i].update(result)
340 TestCase.write_result_to_file(self._tc_results, self._output_file)
345 All setup and teardown through controllers is included.
347 # prepare test execution environment
348 self.run_initialize()
351 with self._vswitch_ctl:
352 with self._vnf_ctl, self._collector, self._loadgen:
353 if not self._vswitch_none:
356 self._versions += self._vswitch_ctl.get_vswitch().get_version()
358 with self._traffic_ctl:
359 # execute test based on TestSteps definition if needed...
361 # ...and continue with traffic generation, but keep
362 # in mind, that clean deployment does not configure
363 # OVS nor executes the traffic
364 if self.deployment != 'clean' and not self._step_send_traffic:
365 self._traffic_ctl.send_traffic(self._traffic)
367 # dump vswitch flows before they are affected by VNF termination
368 if not self._vswitch_none:
369 self._vswitch_ctl.dump_vswitch_flows()
371 # garbage collection for case that TestSteps modify existing deployment
372 self.step_stop_vnfs()
375 # tear down test execution environment and log results
378 self._testcase_stop_time = time.time()
379 self._testcase_run_time = time.strftime("%H:%M:%S",
380 time.gmtime(self._testcase_stop_time -
381 self._testcase_start_time))
382 logging.info("Testcase execution time: " + self._testcase_run_time)
383 # report test results
386 def _append_results(self, results):
388 Method appends mandatory Test Case results to list of dictionaries.
390 :param results: list of dictionaries which contains results from
393 :returns: modified list of dictionaries.
396 item[ResultsConstants.ID] = self.name
397 item[ResultsConstants.DEPLOYMENT] = self.deployment
398 item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
399 item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
400 item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
401 # convert timestamps to human readable format
402 item[ResultsConstants.TEST_START_TIME] = dt.fromtimestamp(
403 self._testcase_start_time).strftime('%Y-%m-%d %H:%M:%S')
404 item[ResultsConstants.TEST_STOP_TIME] = dt.fromtimestamp(
405 self._testcase_stop_time).strftime('%Y-%m-%d %H:%M:%S')
406 if self._traffic['multistream']:
407 item[ResultsConstants.SCAL_STREAM_COUNT] = self._traffic['multistream']
408 item[ResultsConstants.SCAL_STREAM_TYPE] = self._traffic['stream_type']
409 item[ResultsConstants.SCAL_PRE_INSTALLED_FLOWS] = self._traffic['pre_installed_flows']
410 if self._vnf_ctl.get_vnfs_number():
411 item[ResultsConstants.GUEST_LOOPBACK] = ' '.join(S.getValue('GUEST_LOOPBACK'))
412 if self._tunnel_type:
413 item[ResultsConstants.TUNNEL_TYPE] = self._tunnel_type
416 def _copy_fwd_tools_for_all_guests(self, vm_count):
417 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR[s] based on selected deployment.
419 # consider only VNFs involved in the test
420 for guest_dir in set(S.getValue('GUEST_SHARE_DIR')[:vm_count]):
421 self._copy_fwd_tools_for_guest(guest_dir)
423 def _copy_fwd_tools_for_guest(self, guest_dir):
424 """Copy dpdk and l2fwd code to GUEST_SHARE_DIR of VM
426 :param index: Index of VM starting from 1 (i.e. 1st VM has index 1)
428 # remove shared dir if it exists to avoid issues with file consistency
429 if os.path.exists(guest_dir):
430 tasks.run_task(['rm', '-f', '-r', guest_dir], self._logger,
431 'Removing content of shared directory...', True)
433 # directory to share files between host and guest
434 os.makedirs(guest_dir)
436 # copy sources into shared dir only if neccessary
437 guest_loopback = set(S.getValue('GUEST_LOOPBACK'))
438 if 'testpmd' in guest_loopback:
440 # exclude whole .git/ subdirectory and all o-files;
441 # It is assumed, that the same RTE_TARGET is used in both host
442 # and VMs; This simplification significantly speeds up testpmd
443 # build. If we will need a different RTE_TARGET in VM,
444 # then we have to build whole DPDK from the scratch in VM.
445 # In that case we can copy just DPDK sources (e.g. by excluding
446 # all items obtained by git status -unormal --porcelain).
447 # NOTE: Excluding RTE_TARGET directory won't help on systems,
448 # where DPDK is built for multiple targets (e.g. for gcc & icc)
450 exclude.append(r'--exclude=.git/')
451 exclude.append(r'--exclude=*.o')
452 tasks.run_task(['rsync', '-a', '-r', '-l'] + exclude +
453 [os.path.join(S.getValue('TOOLS')['dpdk_src'], ''),
454 os.path.join(guest_dir, 'DPDK')],
456 'Copying DPDK to shared directory...',
458 except subprocess.CalledProcessError:
459 self._logger.error('Unable to copy DPDK to shared directory')
461 if 'l2fwd' in guest_loopback:
463 tasks.run_task(['rsync', '-a', '-r', '-l',
464 os.path.join(S.getValue('ROOT_DIR'), 'src/l2fwd/'),
465 os.path.join(guest_dir, 'l2fwd')],
467 'Copying l2fwd to shared directory...',
469 except subprocess.CalledProcessError:
470 self._logger.error('Unable to copy l2fwd to shared directory')
473 def _mount_hugepages(self):
474 """Mount hugepages if usage of DPDK or Qemu is detected
476 # pylint: disable=too-many-boolean-expressions
477 # hugepages are needed by DPDK and Qemu
478 if not self._hugepages_mounted and \
479 (self.deployment.count('v') or \
480 str(S.getValue('VSWITCH')).lower().count('dpdk') or \
481 self._vswitch_none or \
482 self.test and 'vnf' in [step[0][0:3] for step in self.test]):
483 hugepages.mount_hugepages()
484 self._hugepages_mounted = True
486 def _umount_hugepages(self):
487 """Umount hugepages if they were mounted before
489 if self._hugepages_mounted:
490 hugepages.umount_hugepages()
491 self._hugepages_mounted = False
493 def _check_for_enough_hugepages(self):
494 """Check to make sure enough hugepages are free to satisfy the
498 hugepage_size = hugepages.get_hugepage_size()
499 # get hugepage amounts per guest involved in the test
500 for guest in range(self._vnf_ctl.get_vnfs_number()):
501 hugepages_needed += math.ceil((int(S.getValue(
502 'GUEST_MEMORY')[guest]) * 1000) / hugepage_size)
504 # get hugepage amounts for each socket on dpdk
505 sock0_mem, sock1_mem = 0, 0
507 if str(S.getValue('VSWITCH')).lower().count('dpdk'):
508 sock_mem = S.getValue('DPDK_SOCKET_MEM')
509 sock0_mem, sock1_mem = (int(sock_mem[0]) * 1024 / hugepage_size,
510 int(sock_mem[1]) * 1024 / hugepage_size)
512 # If hugepages needed, verify the amounts are free
513 if any([hugepages_needed, sock0_mem, sock1_mem]):
514 free_hugepages = hugepages.get_free_hugepages()
516 logging.info('Need %s hugepages free for guests',
518 result1 = free_hugepages >= hugepages_needed
519 free_hugepages -= hugepages_needed
524 logging.info('Need %s hugepages free for dpdk socket 0',
526 result2 = hugepages.get_free_hugepages('0') >= sock0_mem
527 free_hugepages -= sock0_mem
532 logging.info('Need %s hugepages free for dpdk socket 1',
534 result3 = hugepages.get_free_hugepages('1') >= sock1_mem
535 free_hugepages -= sock1_mem
539 logging.info('Need a total of %s total hugepages',
540 hugepages_needed + sock1_mem + sock0_mem)
542 # The only drawback here is sometimes dpdk doesn't release
543 # its hugepages on a test failure. This could cause a test
544 # to fail when dpdk would be OK to start because it will just
545 # use the previously allocated hugepages.
546 result4 = True if free_hugepages >= 0 else False
548 return all([result1, result2, result3, result4])
553 def write_result_to_file(results, output):
554 """Write list of dictionaries to a CSV file.
556 Each element on list will create separate row in output file.
557 If output file already exists, data will be appended at the end,
558 otherwise it will be created.
560 :param results: list of dictionaries.
561 :param output: path to output file.
563 with open(output, 'a') as csvfile:
565 logging.info("Write results to file: " + output)
566 fieldnames = TestCase._get_unique_keys(results)
568 writer = csv.DictWriter(csvfile, fieldnames)
570 if not csvfile.tell(): # file is now empty
573 for result in results:
574 writer.writerow(result)
577 def _get_unique_keys(list_of_dicts):
578 """Gets unique key values as ordered list of strings in given dicts
580 :param list_of_dicts: list of dictionaries.
582 :returns: list of unique keys(strings).
584 result = OrderedDict()
585 for item in list_of_dicts:
586 for key in item.keys():
589 return list(result.keys())
591 def _add_flows(self):
592 """Add flows to the vswitch
594 vswitch = self._vswitch_ctl.get_vswitch()
595 # NOTE BOM 15-08-07 the frame mod code assumes that the
596 # physical ports are ports 1 & 2. The actual numbers
597 # need to be retrived from the vSwitch and the metadata value
598 # updated accordingly.
599 bridge = S.getValue('VSWITCH_BRIDGE_NAME')
600 if self._frame_mod == "vlan":
601 # 0x8100 => VLAN ethertype
602 self._logger.debug(" **** VLAN ***** ")
603 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
604 'actions': ['push_vlan:0x8100', 'goto_table:3']}
605 vswitch.add_flow(bridge, flow)
606 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
607 'actions': ['push_vlan:0x8100', 'goto_table:3']}
608 vswitch.add_flow(bridge, flow)
609 elif self._frame_mod == "mpls":
610 # 0x8847 => MPLS unicast ethertype
611 self._logger.debug(" **** MPLS ***** ")
612 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
613 'actions': ['push_mpls:0x8847', 'goto_table:3']}
614 vswitch.add_flow(bridge, flow)
615 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
616 'actions': ['push_mpls:0x8847', 'goto_table:3']}
617 vswitch.add_flow(bridge, flow)
618 elif self._frame_mod == "mac":
619 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
620 'actions': ['mod_dl_src:22:22:22:22:22:22',
622 vswitch.add_flow(bridge, flow)
623 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
624 'actions': ['mod_dl_src:11:11:11:11:11:11',
626 vswitch.add_flow(bridge, flow)
627 elif self._frame_mod == "dscp":
628 # DSCP 184d == 0x4E<<2 => 'Expedited Forwarding'
629 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
631 'actions': ['mod_nw_tos:184', 'goto_table:3']}
632 vswitch.add_flow(bridge, flow)
633 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
635 'actions': ['mod_nw_tos:184', 'goto_table:3']}
636 vswitch.add_flow(bridge, flow)
637 elif self._frame_mod == "ttl":
638 # 251 and 241 are the highest prime numbers < 255
639 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
641 'actions': ['mod_nw_ttl:251', 'goto_table:3']}
642 vswitch.add_flow(bridge, flow)
643 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
645 'actions': ['mod_nw_ttl:241', 'goto_table:3']}
646 vswitch.add_flow(bridge, flow)
647 elif self._frame_mod == "ip_addr":
648 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
650 'actions': ['mod_nw_src:10.10.10.10',
651 'mod_nw_dst:20.20.20.20',
653 vswitch.add_flow(bridge, flow)
654 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
656 'actions': ['mod_nw_src:20.20.20.20',
657 'mod_nw_dst:10.10.10.10',
659 vswitch.add_flow(bridge, flow)
660 elif self._frame_mod == "ip_port":
661 # NOTE BOM 15-08-27 The traffic generated is assumed
662 # to be UDP (nw_proto 17d) which is the default case but
663 # we will need to pick up the actual traffic params in use.
664 flow = {'table':'2', 'priority':'1000', 'metadata':'2',
665 'dl_type':'0x0800', 'nw_proto':'17',
666 'actions': ['mod_tp_src:44444',
667 'mod_tp_dst:44444', 'goto_table:3']}
668 vswitch.add_flow(bridge, flow)
669 flow = {'table':'2', 'priority':'1000', 'metadata':'1',
670 'dl_type':'0x0800', 'nw_proto':'17',
671 'actions': ['mod_tp_src:44444',
672 'mod_tp_dst:44444', 'goto_table:3']}
673 vswitch.add_flow(bridge, flow)
679 # TestSteps realted methods
681 def step_report_status(self, label, status):
682 """ Log status of test step
684 self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
686 def step_stop_vnfs(self):
687 """ Stop all VNFs started by TestSteps
689 for vnf in self._step_vnf_list:
690 if self._step_vnf_list[vnf]:
691 self._step_vnf_list[vnf].stop()
693 def step_eval_param(self, param, step_result):
694 """ Helper function for #STEP macro evaluation
696 if isinstance(param, str):
697 # evaluate every #STEP reference inside parameter itself
698 macros = re.findall(r'(#STEP\[([\w\-:]+)\]((\[[\w\-\'\"]+\])*))', param)
702 if macro[1] in self._step_result_mapping:
703 key = self._step_result_mapping[macro[1]]
706 # pylint: disable=eval-used
707 tmp_val = str(eval('step_result[{}]{}'.format(key, macro[2])))
708 param = param.replace(macro[0], tmp_val)
710 # evaluate references to vsperf configuration options
711 macros = re.findall(r'\$(([\w\-]+)(\[[\w\[\]\-\'\"]+\])*)', param)
714 # pylint: disable=eval-used
716 tmp_val = str(eval("S.getValue('{}'){}".format(macro[1], macro[2])))
717 param = param.replace('${}'.format(macro[0]), tmp_val)
718 # ignore that required option can't be evaluated
719 except (IndexError, KeyError, AttributeError):
720 self._logger.debug("Skipping %s as it isn't a configuration "
721 "parameter.", '${}'.format(macro[0]))
723 elif isinstance(param, list) or isinstance(param, tuple):
726 tmp_list.append(self.step_eval_param(item, step_result))
728 elif isinstance(param, dict):
730 for (key, value) in param.items():
731 tmp_dict[key] = self.step_eval_param(value, step_result)
736 def step_eval_params(self, params, step_result):
737 """ Evaluates referrences to results from previous steps
740 # evaluate all parameters if needed
742 eval_params.append(self.step_eval_param(param, step_result))
745 # pylint: disable=too-many-locals, too-many-branches, too-many-statements
747 """ Execute actions specified by TestSteps list
749 :return: False if any error was detected
756 # required for VNFs initialization
758 # initialize list with results
759 self._step_result = [None] * len(self.test)
761 # We have to suppress pylint report, because test_object has to be set according
762 # to the test step definition
763 # pylint: disable=redefined-variable-type
764 # run test step by step...
765 for i, step in enumerate(self.test):
766 step_ok = not self._step_check
767 step_check = self._step_check
769 # configure step result mapping if step alias/label is detected
770 if step[0].startswith('#'):
773 raise RuntimeError('Step alias can\'t be an integer value {}'.format(key))
774 if key in self._step_result_mapping:
775 raise RuntimeError('Step alias {} has been used already for step '
776 '{}'.format(key, self._step_result_mapping[key]))
777 self._step_result_mapping[step[0][1:]] = i
780 # store regex filter if it is specified
781 if isinstance(step[-1], str) and step[-1].startswith('|'):
782 # evalute macros and variables used in regex
783 regex = self.step_eval_params([step[-1][1:]], self._step_result[:i])[0]
786 # check if step verification should be suppressed
787 if step[0].startswith('!'):
790 step[0] = step[0][1:]
791 if step[0] == 'vswitch':
792 test_object = self._vswitch_ctl.get_vswitch()
793 elif step[0] == 'namespace':
794 test_object = namespace
795 elif step[0] == 'veth':
797 elif step[0] == 'settings':
799 elif step[0] == 'tools':
800 test_object = TestStepsTools()
801 step[1] = step[1].title()
802 elif step[0] == 'trafficgen':
803 test_object = self._traffic_ctl
804 # in case of send_traffic or send_traffic_async methods, ensure
805 # that specified traffic values are merged with existing self._traffic
806 if step[1].startswith('send_traffic'):
807 tmp_traffic = copy.deepcopy(self._traffic)
808 tmp_traffic.update(step[2])
809 step[2] = tmp_traffic
810 # store indication that traffic has been sent
811 # so it is not sent again after the execution of teststeps
812 self._step_send_traffic = True
813 elif step[0].startswith('vnf'):
814 # use vnf started within TestSteps
815 if not self._step_vnf_list[step[0]]:
817 self._step_vnf_list[step[0]] = loader.get_vnf_class()()
818 test_object = self._step_vnf_list[step[0]]
819 elif step[0].startswith('VNF'):
820 if step[1] in ('start', 'stop'):
821 raise RuntimeError("Cannot execute start() or stop() method of "
822 "VNF deployed automatically by scenario.")
823 # use vnf started by scenario deployment (e.g. pvp)
824 vnf_index = int(step[0][3:])
826 test_object = self._vnf_list[vnf_index]
828 raise RuntimeError("VNF with index {} is not running.".format(vnf_index))
829 elif step[0] == 'wait':
830 input(os.linesep + "Step {}: Press Enter to continue with "
831 "the next step...".format(i) + os.linesep + os.linesep)
833 elif step[0] == 'sleep':
834 self._logger.debug("Sleep %s seconds", step[1])
835 time.sleep(int(step[1]))
837 elif step[0] == 'log':
838 test_object = self._logger
839 # there isn't a need for validation of log entry
842 elif step[0] == 'pdb':
847 self._logger.error("Unsupported test object %s", step[0])
848 self._step_status = {'status' : False, 'details' : ' '.join(step)}
849 self.step_report_status("Step '{}'".format(' '.join(step)),
850 self._step_status['status'])
853 test_method = getattr(test_object, step[1])
855 test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
857 test_method_check = None
861 # eval parameters, but use only valid step_results
862 # to support negative indexes
863 step_params = self.step_eval_params(step[2:], self._step_result[:i])
864 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
865 step_log += ' filter "{}"'.format(regex) if regex else ''
866 self._logger.debug("Step %s '%s' start", i, step_log)
867 self._step_result[i] = test_method(*step_params)
869 # apply regex to step output
870 self._step_result[i] = functions.filter_output(
871 self._step_result[i], regex)
873 self._logger.debug("Step %s '%s' results '%s'", i,
874 step_log, self._step_result[i])
875 time.sleep(S.getValue('TEST_STEP_DELAY'))
877 step_ok = test_method_check(self._step_result[i], *step_params)
878 except (AssertionError, AttributeError, IndexError) as ex:
880 self._logger.error("Step %s raised %s", i, type(ex).__name__)
883 self.step_report_status("Step {} - '{}'".format(i, step_log), step_ok)
886 self._step_status = {'status' : False, 'details' : step_log}
887 # Stop all VNFs started by TestSteps
888 self.step_stop_vnfs()
891 # all steps processed without any issue
895 # get methods for TestCase members, which needs to be publicly available
897 def get_output_file(self):
898 """Return content of self._output_file member
900 return self._output_file
903 """Return content of self.desc member
907 def get_versions(self):
908 """Return content of self.versions member
910 return self._versions
912 def get_traffic(self):
913 """Return content of self._traffic member
917 def get_tc_results(self):
918 """Return content of self._tc_results member
920 return self._tc_results
922 def get_collector(self):
923 """Return content of self._collector member
925 return self._collector