functest/ci/run_tests.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Ericsson AB and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """ The entry of running tests:
  11 1) Parses functest/ci/testcases.yaml to check which testcase(s) to be run
  12 2) Execute the common operations on every testcase (run, push results to db...)
  13 3) Return the right status code
  14 """
  15
  16 import argparse
  17 import importlib
  18 import logging
  19 import logging.config
  20 import os
  21 import re
  22 import sys
  23 import textwrap
  24 import pkg_resources
  25
  26 import enum
  27 import prettytable
  28 import yaml
  29
  30 import functest.ci.tier_builder as tb
  31 import functest.core.testcase as testcase
  32
  33 # __name__ cannot be used here
  34 LOGGER = logging.getLogger('functest.ci.run_tests')
  35
  36 CONFIG_FUNCTEST_PATH = pkg_resources.resource_filename(
  37     'functest', 'ci/config_functest.yaml')
  38
  39 ENV_FILE = "/home/opnfv/functest/conf/env_file"
  40
  41
  42 class Result(enum.Enum):
  43     """The overall result in enumerated type"""
  44     # pylint: disable=too-few-public-methods
  45     EX_OK = os.EX_OK
  46     EX_ERROR = -1
  47
  48
  49 class BlockingTestFailed(Exception):
  50     """Exception when the blocking test fails"""
  51     pass
  52
  53
  54 class TestNotEnabled(Exception):
  55     """Exception when the test is not enabled"""
  56     pass
  57
  58
  59 class RunTestsParser(object):
  60     """Parser to run tests"""
  61     # pylint: disable=too-few-public-methods
  62
  63     def __init__(self):
  64         self.parser = argparse.ArgumentParser()
  65         self.parser.add_argument("-t", "--test", dest="test", action='store',
  66                                  help="Test case or tier (group of tests) "
  67                                  "to be executed. It will run all the test "
  68                                  "if not specified.")
  69         self.parser.add_argument("-n", "--noclean", help="Do not clean "
  70                                  "OpenStack resources after running each "
  71                                  "test (default=false).",
  72                                  action="store_true")
  73         self.parser.add_argument("-r", "--report", help="Push results to "
  74                                  "database (default=false).",
  75                                  action="store_true")
  76
  77     def parse_args(self, argv=None):
  78         """Parse arguments.
  79
  80         It can call sys.exit if arguments are incorrect.
  81
  82         Returns:
  83             the arguments from cmdline
  84         """
  85         return vars(self.parser.parse_args(argv))
  86
  87
  88 class Runner(object):
  89     """Runner class"""
  90
  91     def __init__(self):
  92         self.executed_test_cases = {}
  93         self.overall_result = Result.EX_OK
  94         self.clean_flag = True
  95         self.report_flag = False
  96         self.tiers = tb.TierBuilder(
  97             os.environ.get('INSTALLER_TYPE', None),
  98             os.environ.get('DEPLOY_SCENARIO', None),
  99             pkg_resources.resource_filename('functest', 'ci/testcases.yaml'))
 100
 101     @staticmethod
 102     def source_envfile(rc_file=ENV_FILE):
 103         """Source the env file passed as arg"""
 104         with open(rc_file, "r") as rcfd:
 105             for line in rcfd:
 106                 var = (line.rstrip('"\n').replace('export ', '').split(
 107                     "=") if re.search(r'(.*)=(.*)', line) else None)
 108                 # The two next lines should be modified as soon as rc_file
 109                 # conforms with common rules. Be aware that it could induce
 110                 # issues if value starts with '
 111                 if var:
 112                     key = re.sub(r'^["\' ]*|[ \'"]*$', '', var[0])
 113                     value = re.sub(r'^["\' ]*|[ \'"]*$', '', "".join(var[1:]))
 114                     os.environ[key] = value
 115
 116     @staticmethod
 117     def get_dict_by_test(testname):
 118         # pylint: disable=bad-continuation,missing-docstring
 119         with open(pkg_resources.resource_filename(
 120                 'functest', 'ci/testcases.yaml')) as tyaml:
 121             testcases_yaml = yaml.safe_load(tyaml)
 122         for dic_tier in testcases_yaml.get("tiers"):
 123             for dic_testcase in dic_tier['testcases']:
 124                 if dic_testcase['case_name'] == testname:
 125                     return dic_testcase
 126         LOGGER.error('Project %s is not defined in testcases.yaml', testname)
 127         return None
 128
 129     @staticmethod
 130     def get_run_dict(testname):
 131         """Obtain the 'run' block of the testcase from testcases.yaml"""
 132         try:
 133             dic_testcase = Runner.get_dict_by_test(testname)
 134             if not dic_testcase:
 135                 LOGGER.error("Cannot get %s's config options", testname)
 136             elif 'run' in dic_testcase:
 137                 return dic_testcase['run']
 138             return None
 139         except Exception:  # pylint: disable=broad-except
 140             LOGGER.exception("Cannot get %s's config options", testname)
 141             return None
 142
 143     def run_test(self, test):
 144         """Run one test case"""
 145         if not test.is_enabled():
 146             raise TestNotEnabled(
 147                 "The test case {} is not enabled".format(test.get_name()))
 148         LOGGER.info("Running test case '%s'...", test.get_name())
 149         result = testcase.TestCase.EX_RUN_ERROR
 150         run_dict = self.get_run_dict(test.get_name())
 151         if run_dict:
 152             try:
 153                 module = importlib.import_module(run_dict['module'])
 154                 cls = getattr(module, run_dict['class'])
 155                 test_dict = Runner.get_dict_by_test(test.get_name())
 156                 test_case = cls(**test_dict)
 157                 self.executed_test_cases[test.get_name()] = test_case
 158                 try:
 159                     kwargs = run_dict['args']
 160                     test_case.run(**kwargs)
 161                 except KeyError:
 162                     test_case.run()
 163                 if self.report_flag:
 164                     test_case.push_to_db()
 165                 if test.get_project() == "functest":
 166                     result = test_case.is_successful()
 167                 else:
 168                     result = testcase.TestCase.EX_OK
 169                 LOGGER.info("Test result:\n\n%s\n", test_case)
 170                 if self.clean_flag:
 171                     test_case.clean()
 172             except ImportError:
 173                 LOGGER.exception("Cannot import module %s", run_dict['module'])
 174             except AttributeError:
 175                 LOGGER.exception("Cannot get class %s", run_dict['class'])
 176         else:
 177             raise Exception("Cannot import the class for the test case.")
 178         return result
 179
 180     def run_tier(self, tier):
 181         """Run one tier"""
 182         tier_name = tier.get_name()
 183         tests = tier.get_tests()
 184         if not tests:
 185             LOGGER.info("There are no supported test cases in this tier "
 186                         "for the given scenario")
 187             self.overall_result = Result.EX_ERROR
 188         else:
 189             LOGGER.info("Running tier '%s'", tier_name)
 190             for test in tests:
 191                 self.run_test(test)
 192                 test_case = self.executed_test_cases[test.get_name()]
 193                 if test_case.is_successful() != testcase.TestCase.EX_OK:
 194                     LOGGER.error("The test case '%s' failed.", test.get_name())
 195                     if test.get_project() == "functest":
 196                         self.overall_result = Result.EX_ERROR
 197                     if test.is_blocking():
 198                         raise BlockingTestFailed(
 199                             "The test case {} failed and is blocking".format(
 200                                 test.get_name()))
 201         return self.overall_result
 202
 203     def run_all(self):
 204         """Run all available testcases"""
 205         tiers_to_run = []
 206         msg = prettytable.PrettyTable(
 207             header_style='upper', padding_width=5,
 208             field_names=['tiers', 'order', 'CI Loop', 'description',
 209                          'testcases'])
 210         for tier in self.tiers.get_tiers():
 211             ci_loop = os.environ.get('CI_LOOP', None)
 212             if (tier.get_tests() and ci_loop and
 213                     re.search(ci_loop, tier.get_ci_loop()) is not None):
 214                 tiers_to_run.append(tier)
 215                 msg.add_row([tier.get_name(), tier.get_order(),
 216                              tier.get_ci_loop(),
 217                              textwrap.fill(tier.description, width=40),
 218                              textwrap.fill(' '.join([str(x.get_name(
 219                                  )) for x in tier.get_tests()]), width=40)])
 220         LOGGER.info("TESTS TO BE EXECUTED:\n\n%s\n", msg)
 221         for tier in tiers_to_run:
 222             self.run_tier(tier)
 223
 224     def main(self, **kwargs):
 225         """Entry point of class Runner"""
 226         if 'noclean' in kwargs:
 227             self.clean_flag = not kwargs['noclean']
 228         if 'report' in kwargs:
 229             self.report_flag = kwargs['report']
 230         try:
 231             if 'test' in kwargs:
 232                 LOGGER.debug("Sourcing the credential file...")
 233                 self.source_envfile()
 234
 235                 LOGGER.debug("Test args: %s", kwargs['test'])
 236                 if self.tiers.get_tier(kwargs['test']):
 237                     self.run_tier(self.tiers.get_tier(kwargs['test']))
 238                 elif self.tiers.get_test(kwargs['test']):
 239                     result = self.run_test(
 240                         self.tiers.get_test(kwargs['test']))
 241                     if result != testcase.TestCase.EX_OK:
 242                         LOGGER.error("The test case '%s' failed.",
 243                                      kwargs['test'])
 244                         self.overall_result = Result.EX_ERROR
 245                 elif kwargs['test'] == "all":
 246                     self.run_all()
 247                 else:
 248                     LOGGER.error("Unknown test case or tier '%s', or not "
 249                                  "supported by the given scenario '%s'.",
 250                                  kwargs['test'],
 251                                  os.environ.get('DEPLOY_SCENARIO', ""))
 252                     LOGGER.debug("Available tiers are:\n\n%s",
 253                                  self.tiers)
 254                     return Result.EX_ERROR
 255             else:
 256                 self.run_all()
 257         except BlockingTestFailed:
 258             pass
 259         except Exception:  # pylint: disable=broad-except
 260             LOGGER.exception("Failures when running testcase(s)")
 261             self.overall_result = Result.EX_ERROR
 262         if not self.tiers.get_test(kwargs['test']):
 263             self.summary(self.tiers.get_tier(kwargs['test']))
 264         LOGGER.info("Execution exit value: %s", self.overall_result)
 265         return self.overall_result
 266
 267     def summary(self, tier=None):
 268         """To generate functest report showing the overall results"""
 269         msg = prettytable.PrettyTable(
 270             header_style='upper', padding_width=5,
 271             field_names=['env var', 'value'])
 272         for env_var in ['INSTALLER_TYPE', 'DEPLOY_SCENARIO', 'BUILD_TAG',
 273                         'CI_LOOP']:
 274             msg.add_row([env_var, os.environ.get(env_var, "")])
 275         LOGGER.info("Deployment description:\n\n%s\n", msg)
 276         msg = prettytable.PrettyTable(
 277             header_style='upper', padding_width=5,
 278             field_names=['test case', 'project', 'tier',
 279                          'duration', 'result'])
 280         tiers = [tier] if tier else self.tiers.get_tiers()
 281         for each_tier in tiers:
 282             for test in each_tier.get_tests():
 283                 try:
 284                     test_case = self.executed_test_cases[test.get_name()]
 285                 except KeyError:
 286                     msg.add_row([test.get_name(), test.get_project(),
 287                                  each_tier.get_name(), "00:00", "SKIP"])
 288                 else:
 289                     result = 'PASS' if(test_case.is_successful(
 290                         ) == test_case.EX_OK) else 'FAIL'
 291                     msg.add_row(
 292                         [test_case.case_name, test_case.project_name,
 293                          self.tiers.get_tier_name(test_case.case_name),
 294                          test_case.get_duration(), result])
 295             for test in each_tier.get_skipped_test():
 296                 msg.add_row([test.get_name(), test.get_project(),
 297                              each_tier.get_name(), "00:00", "SKIP"])
 298         LOGGER.info("FUNCTEST REPORT:\n\n%s\n", msg)
 299
 300
 301 def main():
 302     """Entry point"""
 303     logging.config.fileConfig(pkg_resources.resource_filename(
 304         'functest', 'ci/logging.ini'))
 305     logging.captureWarnings(True)
 306     parser = RunTestsParser()
 307     args = parser.parse_args(sys.argv[1:])
 308     runner = Runner()
 309     return runner.main(**args).value