testcases/integration.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14 """IntegrationTestCase class
  15 """
  16
  17 import os
  18 import time
  19 import logging
  20 import copy
  21
  22 from testcases import TestCase
  23 from conf import settings as S
  24 from collections import OrderedDict
  25 from tools import namespace
  26 from tools import veth
  27 from core.loader import Loader
  28
  29 CHECK_PREFIX = 'validate_'
  30
  31
  32 class IntegrationTestCase(TestCase):
  33     """IntegrationTestCase class
  34     """
  35
  36     def __init__(self, cfg):
  37         """ Testcase initialization
  38         """
  39         self._type = 'integration'
  40         super(IntegrationTestCase, self).__init__(cfg)
  41         self._logger = logging.getLogger(__name__)
  42         self._inttest = None
  43
  44     def report_status(self, label, status):
  45         """ Log status of test step
  46         """
  47         self._logger.info("%s ... %s", label, 'OK' if status else 'FAILED')
  48
  49     def run_initialize(self):
  50         """ Prepare test execution environment
  51         """
  52         super(IntegrationTestCase, self).run_initialize()
  53         self._inttest = {'status' : True, 'details' : ''}
  54
  55     def run(self):
  56         """Run the test
  57
  58         All setup and teardown through controllers is included.
  59         """
  60         def eval_step_params(params, step_result):
  61             """ Evaluates referrences to results from previous steps
  62             """
  63             def eval_param(param, STEP):
  64                 """ Helper function
  65                 """
  66                 if isinstance(param, str):
  67                     tmp_param = ''
  68                     # evaluate every #STEP reference inside parameter itself
  69                     for chunk in param.split('#'):
  70                         if chunk.startswith('STEP['):
  71                             tmp_param = tmp_param + str(eval(chunk))
  72                         else:
  73                             tmp_param = tmp_param + chunk
  74                     return tmp_param
  75                 elif isinstance(param, list) or isinstance(param, tuple):
  76                     tmp_list = []
  77                     for item in param:
  78                         tmp_list.append(eval_param(item, STEP))
  79                     return tmp_list
  80                 elif isinstance(param, dict):
  81                     tmp_dict = {}
  82                     for (key, value) in param.items():
  83                         tmp_dict[key] = eval_param(value, STEP)
  84                     return tmp_dict
  85                 else:
  86                     return param
  87
  88             eval_params = []
  89             # evaluate all parameters if needed
  90             for param in params:
  91                 eval_params.append(eval_param(param, step_result))
  92             return eval_params
  93
  94         # prepare test execution environment
  95         self.run_initialize()
  96
  97         with self._vswitch_ctl, self._loadgen:
  98             with self._vnf_ctl, self._collector:
  99                 if not self._vswitch_none:
 100                     self._add_flows()
 101
 102                 # run traffic generator if requested, otherwise wait for manual termination
 103                 if S.getValue('mode') == 'trafficgen-off':
 104                     time.sleep(2)
 105                     self._logger.debug("All is set. Please run traffic generator manually.")
 106                     input(os.linesep + "Press Enter to terminate vswitchperf..." + os.linesep + os.linesep)
 107                 else:
 108                     with self._traffic_ctl:
 109                         if not self.test:
 110                             self._traffic_ctl.send_traffic(self._traffic)
 111                         else:
 112                             vnf_list = {}
 113                             loader = Loader()
 114                             # execute test based on TestSteps definition
 115                             if self.test:
 116                                 step_result = [None] * len(self.test)
 117                                 for i, step in enumerate(self.test):
 118                                     step_ok = False
 119                                     if step[0] == 'vswitch':
 120                                         test_object = self._vswitch_ctl.get_vswitch()
 121                                     elif step[0] == 'namespace':
 122                                         test_object = namespace
 123                                     elif step[0] == 'veth':
 124                                         test_object = veth
 125                                     elif step[0] == 'trafficgen':
 126                                         test_object = self._traffic_ctl
 127                                         # in case of send_traffic method, ensure that specified
 128                                         # traffic values are merged with existing self._traffic
 129                                         if step[1] == 'send_traffic':
 130                                             tmp_traffic = copy.deepcopy(self._traffic)
 131                                             tmp_traffic.update(step[2])
 132                                             step[2] = tmp_traffic
 133                                     elif step[0].startswith('vnf'):
 134                                         if not step[0] in vnf_list:
 135                                             # initialize new VM and copy data to its shared dir
 136                                             vnf_list[step[0]] = loader.get_vnf_class()()
 137                                             self._copy_fwd_tools_for_guest(len(vnf_list))
 138                                         test_object = vnf_list[step[0]]
 139                                     else:
 140                                         self._logger.error("Unsupported test object %s", step[0])
 141                                         self._inttest = {'status' : False, 'details' : ' '.join(step)}
 142                                         self.report_status("Step '{}'".format(' '.join(step)), self._inttest['status'])
 143                                         break
 144
 145                                     test_method = getattr(test_object, step[1])
 146                                     test_method_check = getattr(test_object, CHECK_PREFIX + step[1])
 147
 148                                     step_params = []
 149                                     if test_method and test_method_check and \
 150                                         callable(test_method) and callable(test_method_check):
 151
 152                                         try:
 153                                             step_params = eval_step_params(step[2:], step_result)
 154                                             step_log = '{} {}'.format(' '.join(step[:2]), step_params)
 155                                             step_result[i] = test_method(*step_params)
 156                                             self._logger.debug("Step %s '%s' results '%s'", i,
 157                                                                step_log, step_result[i])
 158                                             time.sleep(5)
 159                                             step_ok = test_method_check(step_result[i], *step_params)
 160                                         except AssertionError:
 161                                             self._inttest = {'status' : False, 'details' : step_log}
 162                                             self._logger.error("Step %s raised assertion error", i)
 163                                             # stop vnfs in case of error
 164                                             for vnf in vnf_list:
 165                                                 vnf_list[vnf].stop()
 166                                             break
 167                                         except IndexError:
 168                                             self._inttest = {'status' : False, 'details' : step_log}
 169                                             self._logger.error("Step %s result index error %s", i,
 170                                                                ' '.join(step[2:]))
 171                                             # stop vnfs in case of error
 172                                             for vnf in vnf_list:
 173                                                 vnf_list[vnf].stop()
 174                                             break
 175
 176                                     self.report_status("Step {} - '{}'".format(i, step_log), step_ok)
 177                                     if not step_ok:
 178                                         self._inttest = {'status' : False, 'details' : step_log}
 179                                         # stop vnfs in case of error
 180                                         for vnf in vnf_list:
 181                                             vnf_list[vnf].stop()
 182                                         break
 183
 184                     # dump vswitch flows before they are affected by VNF termination
 185                     if not self._vswitch_none:
 186                         self._vswitch_ctl.dump_vswitch_flows()
 187
 188         # tear down test execution environment and log results
 189         self.run_finalize()
 190
 191         # report test results
 192         self.run_report()
 193
 194     def run_report(self):
 195         """ Report test results
 196         """
 197         if self.test:
 198             results = OrderedDict()
 199             results['status'] = 'OK' if self._inttest['status'] else 'FAILED'
 200             results['details'] = self._inttest['details']
 201             TestCase.write_result_to_file([results], self._output_file)
 202             self.report_status("Test '{}'".format(self.name), self._inttest['status'])
 203             # inform vsperf about testcase failure
 204             if not self._inttest['status']:
 205                 raise Exception