testcases/integration.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """IntegrationTestCase class
  15 """
  16
  17 import os
  18 import time
  19 import logging
  20 import copy
  21
  22 from testcases import TestCase
  23 from conf import settings as S
  24 from collections import OrderedDict
  25 from core.loader import Loader
  26
  27 CHECK_PREFIX = 'validate_'
  28
  29 class IntegrationTestCase(TestCase):
  30     """IntegrationTestCase class
  31     """
  32
  33     def __init__(self, cfg):
  34         """ Testcase initialization
  35         """
  36         self._type = 'integration'
  37         super(IntegrationTestCase, self).__init__(cfg)
  38         self._logger = logging.getLogger(__name__)
  39         self._inttest = None
  40
  41     def report_status(self, label, status):
  42         """ Log status of test step
  43         """
  44         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
  45
  46     def run_initialize(self):
  47         """ Prepare test execution environment
  48         """
  49         super(IntegrationTestCase, self).run_initialize()
  50         self._inttest = {'status' : True, 'details' : ''}
  51
  52     def run(self):
  53         """Run the test
  54
  55         All setup and teardown through controllers is included.
  56         """
  57         def eval_step_params(params, step_result):
  58             """ Evaluates referrences to results from previous steps
  59             """
  60             def eval_param(param, STEP):
  61                 """ Helper function
  62                 """
  63                 if isinstance(param, str):
  64                     tmp_param = ''
  65                     # evaluate every #STEP reference inside parameter itself
  66                     for chunk in param.split('#'):
  67                         if chunk.startswith('STEP['):
  68                             tmp_param = tmp_param + str(eval(chunk))
  69                         else:
  70                             tmp_param = tmp_param + chunk
  71                     return tmp_param
  72                 elif isinstance(param, list) or isinstance(param, tuple):
  73                     tmp_list = []
  74                     for item in param:
  75                         tmp_list.append(eval_param(item, STEP))
  76                     return tmp_list
  77                 elif isinstance(param, dict):
  78                     tmp_dict = {}
  79                     for (key, value) in param.items():
  80                         tmp_dict[key] = eval_param(value, STEP)
  81                     return tmp_dict
  82                 else:
  83                     return param
  84
  85             eval_params = []
  86             # evaluate all parameters if needed
  87             for param in params:
  88                 eval_params.append(eval_param(param, step_result))
  89             return eval_params
  90
  91         # prepare test execution environment
  92         self.run_initialize()
  93
  94         with self._vswitch_ctl, self._loadgen:
  95             with self._vnf_ctl, self._collector:
  96                 if not self._vswitch_none:
  97                     self._add_flows()
  98
  99                 # run traffic generator if requested, otherwise wait for manual termination
 100                 if S.getValue('mode') == 'trafficgen-off':
 101                     time.sleep(2)
 102                     self._logger.debug("All is set. Please run traffic generator manually.")
 103                     input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
 104                 else:
 105                     with self._traffic_ctl:
 106                         if not self.test:
 107                             self._traffic_ctl.send_traffic(self._traffic)
 108                         else:
 109                             vnf_list = {}
 110                             loader = Loader()
 111                             # execute test based on TestSteps definition
 112                             if self.test:
 113                                 step_result = [None] * len(self.test)
 114                                 for i, step in enumerate(self.test):
 115                                     step_ok = False
 116                                     if step[0] == 'vswitch':
 117                                         test_object = self._vswitch_ctl.get_vswitch()
 118                                     elif step[0] == 'trafficgen':
 119                                         test_object = self._traffic_ctl
 120                                         # in case of send_traffic method, ensure that specified
 121                                         # traffic values are merged with existing self._traffic
 122                                         if step[1] == 'send_traffic':
 123                                             tmp_traffic = copy.deepcopy(self._traffic)
 124                                             tmp_traffic.update(step[2])
 125                                             step[2] = tmp_traffic
 126                                     elif step[0].startswith('vnf'):
 127                                         if not step[0] in vnf_list:
 128                                             # initialize new VM and copy data to its shared dir
 129                                             vnf_list[step[0]] = loader.get_vnf_class()()
 130                                             self._copy_fwd_tools_for_guest(len(vnf_list))
 131                                         test_object = vnf_list[step[0]]
 132                                     else:
 133                                         self._logger.error("Unsupported test object %s", step[0])
 134                                         self._inttest = {'status' : False, 'details' : ' '.join(step)}
 135                                         self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status'])
 136                                         break
 137
 138                                     test_method = getattr(test_object, step[1])
 139                                     test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 140
 141                                     step_params = []
 142                                     if test_method and test_method_check and \
 143                                         callable(test_method) and callable(test_method_check):
 144
 145                                         try:
 146                                             step_params = eval_step_params(step[2:], step_result)
 147                                             step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 148                                             step_result[i] = test_method(*step_params)
 149                                             self._logger.debug("Step %s '%s' results '%s'", i,
 150                                                                step_log, step_result[i])
 151                                             time.sleep(5)
 152                                             step_ok = test_method_check(step_result[i], *step_params)
 153                                         except AssertionError:
 154                                             self._inttest = {'status' : False, 'details' : step_log}
 155                                             self._logger.error("Step %s raised assertion error", i)
 156                                             # stop vnfs in case of error
 157                                             for vnf in vnf_list:
 158                                                 vnf_list[vnf].stop()
 159                                             break
 160                                         except IndexError:
 161                                             self._inttest = {'status' : False, 'details' : step_log}
 162                                             self._logger.error("Step %s result index error %s", i,
 163                                                                ' '.join(step[2:]))
 164                                             # stop vnfs in case of error
 165                                             for vnf in vnf_list:
 166                                                 vnf_list[vnf].stop()
 167                                             break
 168
 169                                     self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
 170                                     if not step_ok:
 171                                         self._inttest = {'status' : False, 'details' : step_log}
 172                                         # stop vnfs in case of error
 173                                         for vnf in vnf_list:
 174                                             vnf_list[vnf].stop()
 175                                         break
 176
 177                     # dump vswitch flows before they are affected by VNF termination
 178                     if not self._vswitch_none:
 179                         self._vswitch_ctl.dump_vswitch_flows()
 180
 181         # tear down test execution environment and log results
 182         self.run_finalize()
 183
 184         # report test results
 185         self.run_report()
 186
 187     def run_report(self):
 188         """ Report test results
 189         """
 190         if self.test:
 191             results = OrderedDict()
 192             results['status'] = 'OK' if self._inttest['status'] else 'FAILED'
 193             results['details'] = self._inttest['details']
 194             TestCase.write_result_to_file([results], self._output_file)
 195             self.report_status("Test '{}'".format(self.name), self._inttest['status'])
 196             # inform vsperf about testcase failure
 197             if not self._inttest['status']:
 198                 raise Exception