testcases/integration.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """IntegrationTestCase class
  15 """
  16
  17 import os
  18 import time
  19 import logging
  20 import copy
  21
  22 from testcases import TestCase
  23 from conf import settings as S
  24 from collections import OrderedDict
  25 from tools import namespace
  26 from tools import veth
  27 from core.loader import Loader
  28
  29 CHECK_PREFIX = 'validate_'
  30
  31
  32 class IntegrationTestCase(TestCase):
  33     """IntegrationTestCase class
  34     """
  35
  36     def __init__(self, cfg):
  37         """ Testcase initialization
  38         """
  39         self._type = 'integration'
  40         super(IntegrationTestCase, self).__init__(cfg)
  41         self._logger = logging.getLogger(__name__)
  42         self._inttest = None
  43
  44     def report_status(self, label, status):
  45         """ Log status of test step
  46         """
  47         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
  48
  49     def run_initialize(self):
  50         """ Prepare test execution environment
  51         """
  52         super(IntegrationTestCase, self).run_initialize()
  53         self._inttest = {'status' : True, 'details' : ''}
  54
  55     def run(self):
  56         """Run the test
  57
  58         All setup and teardown through controllers is included.
  59         """
  60         def eval_step_params(params, step_result):
  61             """ Evaluates referrences to results from previous steps
  62             """
  63             def eval_param(param, STEP):
  64                 """ Helper function
  65                 """
  66                 if isinstance(param, str):
  67                     tmp_param = ''
  68                     # evaluate every #STEP reference inside parameter itself
  69                     for chunk in param.split('#'):
  70                         if chunk.startswith('STEP['):
  71                             tmp_param = tmp_param + str(eval(chunk))
  72                         else:
  73                             tmp_param = tmp_param + chunk
  74                     return tmp_param
  75                 elif isinstance(param, list) or isinstance(param, tuple):
  76                     tmp_list = []
  77                     for item in param:
  78                         tmp_list.append(eval_param(item, STEP))
  79                     return tmp_list
  80                 elif isinstance(param, dict):
  81                     tmp_dict = {}
  82                     for (key, value) in param.items():
  83                         tmp_dict[key] = eval_param(value, STEP)
  84                     return tmp_dict
  85                 else:
  86                     return param
  87
  88             eval_params = []
  89             # evaluate all parameters if needed
  90             for param in params:
  91                 eval_params.append(eval_param(param, step_result))
  92             return eval_params
  93
  94         # prepare test execution environment
  95         self.run_initialize()
  96
  97         try:
  98             with self._vswitch_ctl, self._loadgen:
  99                 with self._vnf_ctl, self._collector:
 100                     if not self._vswitch_none:
 101                         self._add_flows()
 102
 103                     # run traffic generator if requested, otherwise wait for manual termination
 104                     if S.getValue('mode') == 'trafficgen-off':
 105                         time.sleep(2)
 106                         self._logger.debug("All is set. Please run traffic generator manually.")
 107                         input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
 108                     else:
 109                         with self._traffic_ctl:
 110                             if not self.test:
 111                                 self._traffic_ctl.send_traffic(self._traffic)
 112                             else:
 113                                 vnf_list = {}
 114                                 loader = Loader()
 115                                 # execute test based on TestSteps definition
 116                                 if self.test:
 117                                     step_result = [None] * len(self.test)
 118                                     for i, step in enumerate(self.test):
 119                                         step_ok = False
 120                                         if step[0] == 'vswitch':
 121                                             test_object = self._vswitch_ctl.get_vswitch()
 122                                         elif step[0] == 'namespace':
 123                                             test_object = namespace
 124                                         elif step[0] == 'veth':
 125                                             test_object = veth
 126                                         elif step[0] == 'trafficgen':
 127                                             test_object = self._traffic_ctl
 128                                             # in case of send_traffic method, ensure that specified
 129                                             # traffic values are merged with existing self._traffic
 130                                             if step[1] == 'send_traffic':
 131                                                 tmp_traffic = copy.deepcopy(self._traffic)
 132                                                 tmp_traffic.update(step[2])
 133                                                 step[2] = tmp_traffic
 134                                         elif step[0].startswith('vnf'):
 135                                             if not step[0] in vnf_list:
 136                                                 # initialize new VM and copy data to its shared dir
 137                                                 vnf_list[step[0]] = loader.get_vnf_class()()
 138                                                 self._copy_fwd_tools_for_guest(len(vnf_list))
 139                                             test_object = vnf_list[step[0]]
 140                                         else:
 141                                             self._logger.error("Unsupported test object %s", step[0])
 142                                             self._inttest = {'status' : False, 'details' : ' '.join(step)}
 143                                             self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status'])
 144                                             break
 145
 146                                         test_method = getattr(test_object, step[1])
 147                                         test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 148
 149                                         step_params = []
 150                                         if test_method and test_method_check and \
 151                                             callable(test_method) and callable(test_method_check):
 152
 153                                             try:
 154                                                 step_params = eval_step_params(step[2:], step_result)
 155                                                 step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 156                                                 step_result[i] = test_method(*step_params)
 157                                                 self._logger.debug("Step %s '%s' results '%s'", i,
 158                                                                    step_log, step_result[i])
 159                                                 time.sleep(5)
 160                                                 step_ok = test_method_check(step_result[i], *step_params)
 161                                             except AssertionError:
 162                                                 self._inttest = {'status' : False, 'details' : step_log}
 163                                                 self._logger.error("Step %s raised assertion error", i)
 164                                                 # stop vnfs in case of error
 165                                                 for vnf in vnf_list:
 166                                                     vnf_list[vnf].stop()
 167                                                 break
 168                                             except IndexError:
 169                                                 self._inttest = {'status' : False, 'details' : step_log}
 170                                                 self._logger.error("Step %s result index error %s", i,
 171                                                                    ' '.join(step[2:]))
 172                                                 # stop vnfs in case of error
 173                                                 for vnf in vnf_list:
 174                                                     vnf_list[vnf].stop()
 175                                                 break
 176
 177                                         self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
 178                                         if not step_ok:
 179                                             self._inttest = {'status' : False, 'details' : step_log}
 180                                             # stop vnfs in case of error
 181                                             for vnf in vnf_list:
 182                                                 vnf_list[vnf].stop()
 183                                             break
 184
 185                         # dump vswitch flows before they are affected by VNF termination
 186                         if not self._vswitch_none:
 187                             self._vswitch_ctl.dump_vswitch_flows()
 188         finally:
 189             # tear down test execution environment and log results
 190             self.run_finalize()
 191
 192         # report test results
 193         self.run_report()
 194
 195     def run_report(self):
 196         """ Report test results
 197         """
 198         if self.test:
 199             results = OrderedDict()
 200             results['status'] = 'OK' if self._inttest['status'] else 'FAILED'
 201             results['details'] = self._inttest['details']
 202             TestCase.write_result_to_file([results], self._output_file)
 203             self.report_status("Test '{}'".format(self.name), self._inttest['status'])
 204             # inform vsperf about testcase failure
 205             if not self._inttest['status']:
 206                 raise Exception