testcases/integration.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """IntegrationTestCase class
  15 """
  16
  17 import os
  18 import time
  19 import logging
  20 import copy
  21 import re
  22
  23 from collections import OrderedDict
  24 from testcases import TestCase
  25 from conf import settings as S
  26 from tools import namespace
  27 from tools import veth
  28 from tools.teststepstools import TestStepsTools
  29 from core.loader import Loader
  30
  31 CHECK_PREFIX = 'validate_'
  32
  33
  34 class IntegrationTestCase(TestCase):
  35     """IntegrationTestCase class
  36     """
  37
  38     def __init__(self, cfg):
  39         """ Testcase initialization
  40         """
  41         self._type = 'integration'
  42         super(IntegrationTestCase, self).__init__(cfg)
  43         self._logger = logging.getLogger(__name__)
  44         self._inttest = None
  45
  46     def report_status(self, label, status):
  47         """ Log status of test step
  48         """
  49         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
  50
  51     def run_initialize(self):
  52         """ Prepare test execution environment
  53         """
  54         super(IntegrationTestCase, self).run_initialize()
  55         self._inttest = {'status' : True, 'details' : ''}
  56
  57     def run(self):
  58         """Run the test
  59
  60         All setup and teardown through controllers is included.
  61         """
  62         def eval_step_params(params, step_result):
  63             """ Evaluates referrences to results from previous steps
  64             """
  65             def eval_param(param, STEP):
  66                 # pylint: disable=invalid-name
  67                 """ Helper function
  68                 """
  69                 if isinstance(param, str):
  70                     # evaluate every #STEP reference inside parameter itself
  71                     macros = re.findall(r'#STEP\[[\w\[\]\-\'\"]+\]', param)
  72                     if macros:
  73                         for macro in macros:
  74                             # pylint: disable=eval-used
  75                             tmp_val = str(eval(macro[1:]))
  76                             param = param.replace(macro, tmp_val)
  77                     return param
  78                 elif isinstance(param, list) or isinstance(param, tuple):
  79                     tmp_list = []
  80                     for item in param:
  81                         tmp_list.append(eval_param(item, STEP))
  82                     return tmp_list
  83                 elif isinstance(param, dict):
  84                     tmp_dict = {}
  85                     for (key, value) in param.items():
  86                         tmp_dict[key] = eval_param(value, STEP)
  87                     return tmp_dict
  88                 else:
  89                     return param
  90
  91             eval_params = []
  92             # evaluate all parameters if needed
  93             for param in params:
  94                 eval_params.append(eval_param(param, step_result))
  95             return eval_params
  96
  97         # prepare test execution environment
  98         self.run_initialize()
  99
 100         try:
 101             with self._vswitch_ctl, self._loadgen:
 102                 with self._vnf_ctl, self._collector:
 103                     if not self._vswitch_none:
 104                         self._add_flows()
 105
 106                     # run traffic generator if requested, otherwise wait for manual termination
 107                     if S.getValue('mode') == 'trafficgen-off':
 108                         time.sleep(2)
 109                         self._logger.debug("All is set. Please run traffic generator manually.")
 110                         input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
 111                     else:
 112                         with self._traffic_ctl:
 113                             if not self.test:
 114                                 self._traffic_ctl.send_traffic(self._traffic)
 115                             else:
 116                                 vnf_list = {}
 117                                 loader = Loader()
 118                                 # execute test based on TestSteps definition
 119                                 if self.test:
 120                                     # initialize list with results
 121                                     step_result = [None] * len(self.test)
 122
 123                                     # count how many VNFs are involved in the test
 124                                     for step in self.test:
 125                                         if step[0].startswith('vnf'):
 126                                             vnf_list[step[0]] = None
 127
 128                                     # check/expand GUEST configuration and copy data to shares
 129                                     if len(vnf_list):
 130                                         S.check_vm_settings(len(vnf_list))
 131                                         self._copy_fwd_tools_for_all_guests(len(vnf_list))
 132
 133                                     # run test step by step...
 134                                     for i, step in enumerate(self.test):
 135                                         step_ok = False
 136                                         if step[0] == 'vswitch':
 137                                             test_object = self._vswitch_ctl.get_vswitch()
 138                                         elif step[0] == 'namespace':
 139                                             test_object = namespace
 140                                         elif step[0] == 'veth':
 141                                             test_object = veth
 142                                         elif step[0] == 'settings':
 143                                             test_object = S
 144                                         elif step[0] == 'tools':
 145                                             test_object = TestStepsTools()
 146                                             step[1] = step[1].title()
 147                                         elif step[0] == 'trafficgen':
 148                                             test_object = self._traffic_ctl
 149                                             # in case of send_traffic method, ensure that specified
 150                                             # traffic values are merged with existing self._traffic
 151                                             if step[1] == 'send_traffic':
 152                                                 tmp_traffic = copy.deepcopy(self._traffic)
 153                                                 tmp_traffic.update(step[2])
 154                                                 step[2] = tmp_traffic
 155                                         elif step[0].startswith('vnf'):
 156                                             if not vnf_list[step[0]]:
 157                                                 # initialize new VM
 158                                                 vnf_list[step[0]] = loader.get_vnf_class()()
 159                                             test_object = vnf_list[step[0]]
 160                                         elif step[0] == 'wait':
 161                                             input(os.linesep + "Step {}: Press Enter to continue with "
 162                                                   "the next step...".format(i) + os.linesep + os.linesep)
 163                                             continue
 164                                         else:
 165                                             self._logger.error("Unsupported test object %s", step[0])
 166                                             self._inttest = {'status' : False, 'details' : ' '.join(step)}
 167                                             self.report_status("Step '{}'".format(' '.join(step)),
 168                                                                self._inttest['status'])
 169                                             break
 170
 171                                         test_method = getattr(test_object, step[1])
 172                                         test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 173
 174                                         step_params = []
 175                                         if test_method and test_method_check and \
 176                                             callable(test_method) and callable(test_method_check):
 177
 178                                             try:
 179                                                 # eval parameters, but use only valid step_results
 180                                                 # to support negative indexes
 181                                                 step_params = eval_step_params(step[2:], step_result[:i])
 182                                                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 183                                                 step_result[i] = test_method(*step_params)
 184                                                 self._logger.debug("Step %s '%s' results '%s'", i,
 185                                                                    step_log, step_result[i])
 186                                                 time.sleep(5)
 187                                                 step_ok = test_method_check(step_result[i], *step_params)
 188                                             except AssertionError:
 189                                                 self._inttest = {'status' : False, 'details' : step_log}
 190                                                 self._logger.error("Step %s raised assertion error", i)
 191                                                 # stop vnfs in case of error
 192                                                 for vnf in vnf_list:
 193                                                     vnf_list[vnf].stop()
 194                                                 break
 195                                             except IndexError:
 196                                                 self._inttest = {'status' : False, 'details' : step_log}
 197                                                 self._logger.error("Step %s result index error %s", i,
 198                                                                    ' '.join(step[2:]))
 199                                                 # stop vnfs in case of error
 200                                                 for vnf in vnf_list:
 201                                                     vnf_list[vnf].stop()
 202                                                 break
 203
 204                                         self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
 205                                         if not step_ok:
 206                                             self._inttest = {'status' : False, 'details' : step_log}
 207                                             # stop vnfs in case of error
 208                                             for vnf in vnf_list:
 209                                                 vnf_list[vnf].stop()
 210                                             break
 211
 212                         # dump vswitch flows before they are affected by VNF termination
 213                         if not self._vswitch_none:
 214                             self._vswitch_ctl.dump_vswitch_flows()
 215         finally:
 216             # tear down test execution environment and log results
 217             self.run_finalize()
 218
 219         # report test results
 220         self.run_report()
 221
 222     def run_report(self):
 223         """ Report test results
 224         """
 225         if self.test:
 226             results = OrderedDict()
 227             results['status'] = 'OK' if self._inttest['status'] else 'FAILED'
 228             results['details'] = self._inttest['details']
 229             TestCase.write_result_to_file([results], self._output_file)
 230             self.report_status("Test '{}'".format(self.name), self._inttest['status'])
 231             # inform vsperf about testcase failure
 232             if not self._inttest['status']:
 233                 raise Exception