functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import os_client_config
  24 from xtesting.core import testcase
  25 import yaml
  26
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(testcase.TestCase):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     TEMPEST_RESULTS_DIR = os.path.join(
  39         getattr(config.CONF, 'dir_results'), 'tempest')
  40
  41     def __init__(self, **kwargs):
  42         super(TempestCommon, self).__init__(**kwargs)
  43         self.resources = TempestResourcesManager()
  44         self.mode = ""
  45         self.option = []
  46         self.verifier_id = conf_utils.get_verifier_id()
  47         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  48             self.verifier_id)
  49         self.deployment_id = conf_utils.get_verifier_deployment_id()
  50         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  51             self.verifier_id, self.deployment_id)
  52         self.verification_id = None
  53         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  54         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  55         self.list = os.path.join(self.res_dir, 'test_list.txt')
  56         self.conf_file = None
  57
  58     @staticmethod
  59     def read_file(filename):
  60         """Read file and return content as a stripped list."""
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     @staticmethod
  65     def get_verifier_result(verif_id):
  66         """Retrieve verification results."""
  67         result = {
  68             'num_tests': 0,
  69             'num_success': 0,
  70             'num_failures': 0,
  71             'num_skipped': 0
  72         }
  73         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  74         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  75         proc = subprocess.Popen(cmd,
  76                                 stdout=subprocess.PIPE,
  77                                 stderr=subprocess.STDOUT)
  78         for line in proc.stdout:
  79             new_line = line.replace(' ', '').split('|')
  80             if 'Tests' in new_line:
  81                 break
  82             LOGGER.info(line)
  83             if 'Testscount' in new_line:
  84                 result['num_tests'] = int(new_line[2])
  85             elif 'Success' in new_line:
  86                 result['num_success'] = int(new_line[2])
  87             elif 'Skipped' in new_line:
  88                 result['num_skipped'] = int(new_line[2])
  89             elif 'Failures' in new_line:
  90                 result['num_failures'] = int(new_line[2])
  91         return result
  92
  93     @staticmethod
  94     def backup_tempest_config(conf_file, res_dir):
  95         """
  96         Copy config file to tempest results directory
  97         """
  98         if not os.path.exists(res_dir):
  99             os.makedirs(res_dir)
 100         shutil.copyfile(conf_file,
 101                         os.path.join(res_dir, 'tempest.conf'))
 102
 103     def generate_test_list(self):
 104         """Generate test list based on the test mode."""
 105         LOGGER.debug("Generating test case list...")
 106         if self.mode == 'custom':
 107             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 108                 shutil.copyfile(
 109                     conf_utils.TEMPEST_CUSTOM, self.list)
 110             else:
 111                 raise Exception("Tempest test list file %s NOT found."
 112                                 % conf_utils.TEMPEST_CUSTOM)
 113         else:
 114             if self.mode == 'smoke':
 115                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 116             elif self.mode == 'full':
 117                 testr_mode = r"'^tempest\.'"
 118             else:
 119                 testr_mode = self.mode
 120             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 121                 self.verifier_repo_dir, testr_mode, self.list)
 122             output = subprocess.check_output(cmd, shell=True)
 123             LOGGER.info("%s\n%s", cmd, output)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 188                     self.verification_id = re.search(
 189                         r'(?=\(UUID=(.*)\))', line).group(1)
 190                     LOGGER.info('Verification UUID: %s', self.verification_id)
 191                 f_stdout.write(line)
 192         proc.wait()
 193
 194         f_stdout.close()
 195         f_stderr.close()
 196
 197         if self.verification_id is None:
 198             raise Exception('Verification UUID not found')
 199
 200     def parse_verifier_result(self):
 201         """Parse and save test results."""
 202         stat = self.get_verifier_result(self.verification_id)
 203         try:
 204             num_executed = stat['num_tests'] - stat['num_skipped']
 205             try:
 206                 self.result = 100 * stat['num_success'] / num_executed
 207             except ZeroDivisionError:
 208                 self.result = 0
 209                 if stat['num_tests'] > 0:
 210                     LOGGER.info("All tests have been skipped")
 211                 else:
 212                     LOGGER.error("No test has been executed")
 213                     return
 214
 215             with open(os.path.join(self.res_dir,
 216                                    "tempest.log"), 'r') as logfile:
 217                 output = logfile.read()
 218
 219             success_testcases = []
 220             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 221                                     output):
 222                 success_testcases.append(match)
 223             failed_testcases = []
 224             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 225                                     output):
 226                 failed_testcases.append(match)
 227             skipped_testcases = []
 228             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 229                                     output):
 230                 skipped_testcases.append(match)
 231
 232             self.details = {"tests_number": stat['num_tests'],
 233                             "success_number": stat['num_success'],
 234                             "skipped_number": stat['num_skipped'],
 235                             "failures_number": stat['num_failures'],
 236                             "success": success_testcases,
 237                             "skipped": skipped_testcases,
 238                             "failures": failed_testcases}
 239         except Exception:  # pylint: disable=broad-except
 240             self.result = 0
 241
 242         LOGGER.info("Tempest %s success_rate is %s%%",
 243                     self.case_name, self.result)
 244
 245     def generate_report(self):
 246         """Generate verification report."""
 247         html_file = os.path.join(self.res_dir,
 248                                  "tempest-report.html")
 249         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 250                self.verification_id, "--to", html_file]
 251         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 252                          stderr=subprocess.STDOUT)
 253
 254     def configure(self, **kwargs):  # pylint: disable=unused-argument
 255         """
 256         Create all openstack resources for tempest-based testcases and write
 257         tempest.conf.
 258         """
 259         if not os.path.exists(self.res_dir):
 260             os.makedirs(self.res_dir)
 261         self.resources.create()
 262         compute_cnt = len(self.resources.cloud.list_hypervisors())
 263         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 264         conf_utils.configure_tempest_update_params(
 265             self.conf_file, network_name=self.resources.network.id,
 266             image_id=self.resources.image.id,
 267             flavor_id=self.resources.flavor.id,
 268             compute_cnt=compute_cnt,
 269             image_alt_id=self.resources.image_alt.id,
 270             flavor_alt_id=self.resources.flavor_alt.id)
 271         self.backup_tempest_config(self.conf_file, self.res_dir)
 272
 273     def run(self, **kwargs):
 274         self.start_time = time.time()
 275         try:
 276             self.configure(**kwargs)
 277             self.generate_test_list()
 278             self.apply_tempest_blacklist()
 279             self.run_verifier_tests()
 280             self.parse_verifier_result()
 281             self.generate_report()
 282             res = testcase.TestCase.EX_OK
 283         except Exception:  # pylint: disable=broad-except
 284             LOGGER.exception('Error with run')
 285             res = testcase.TestCase.EX_RUN_ERROR
 286         finally:
 287             self.resources.cleanup()
 288         self.stop_time = time.time()
 289         return res
 290
 291
 292 class TempestSmokeSerial(TempestCommon):
 293     """Tempest smoke serial testcase implementation."""
 294     def __init__(self, **kwargs):
 295         if "case_name" not in kwargs:
 296             kwargs["case_name"] = 'tempest_smoke_serial'
 297         TempestCommon.__init__(self, **kwargs)
 298         self.mode = "smoke"
 299         self.option = ["--concurrency", "1"]
 300
 301
 302 class TempestNeutronTrunk(TempestCommon):
 303     """Tempest neutron trunk testcase implementation."""
 304     def __init__(self, **kwargs):
 305         if "case_name" not in kwargs:
 306             kwargs["case_name"] = 'neutron_trunk'
 307         TempestCommon.__init__(self, **kwargs)
 308         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 309         self.res_dir = os.path.join(
 310             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 311         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 312         self.list = os.path.join(self.res_dir, 'test_list.txt')
 313
 314     def configure(self, **kwargs):
 315         super(TempestNeutronTrunk, self).configure(**kwargs)
 316         rconfig = conf_utils.ConfigParser.RawConfigParser()
 317         rconfig.read(self.conf_file)
 318         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 319         with open(self.conf_file, 'wb') as config_file:
 320             rconfig.write(config_file)
 321
 322
 323 class TempestSmokeParallel(TempestCommon):
 324     """Tempest smoke parallel testcase implementation."""
 325     def __init__(self, **kwargs):
 326         if "case_name" not in kwargs:
 327             kwargs["case_name"] = 'tempest_smoke_parallel'
 328         TempestCommon.__init__(self, **kwargs)
 329         self.mode = "smoke"
 330
 331
 332 class TempestFullParallel(TempestCommon):
 333     """Tempest full parallel testcase implementation."""
 334     def __init__(self, **kwargs):
 335         if "case_name" not in kwargs:
 336             kwargs["case_name"] = 'tempest_full_parallel'
 337         TempestCommon.__init__(self, **kwargs)
 338         self.mode = "full"
 339
 340
 341 class TempestCustom(TempestCommon):
 342     """Tempest custom testcase implementation."""
 343     def __init__(self, **kwargs):
 344         if "case_name" not in kwargs:
 345             kwargs["case_name"] = 'tempest_custom'
 346         TempestCommon.__init__(self, **kwargs)
 347         self.mode = "custom"
 348         self.option = ["--concurrency", "1"]
 349
 350
 351 class TempestDefcore(TempestCommon):
 352     """Tempest Defcore testcase implementation."""
 353     def __init__(self, **kwargs):
 354         if "case_name" not in kwargs:
 355             kwargs["case_name"] = 'tempest_defcore'
 356         TempestCommon.__init__(self, **kwargs)
 357         self.mode = "defcore"
 358         self.option = ["--concurrency", "1"]
 359
 360
 361 class TempestResourcesManager(object):
 362     # pylint: disable=too-many-instance-attributes
 363     """Tempest resource manager."""
 364     def __init__(self):
 365         self.guid = '-' + str(uuid.uuid4())
 366         self.cloud = os_client_config.make_shade()
 367         self.domain_id = self.cloud.auth["project_domain_id"]
 368         self.project = None
 369         self.user = None
 370         self.network = None
 371         self.subnet = None
 372         self.image = None
 373         self.image_alt = None
 374         self.flavor = None
 375         self.flavor_alt = None
 376
 377     def _create_project(self):
 378         """Create project for tests."""
 379         self.project = self.cloud.create_project(
 380             getattr(config.CONF, 'tempest_identity_tenant_name') + self.guid,
 381             description=getattr(
 382                 config.CONF, 'tempest_identity_tenant_description'),
 383             domain_id=self.domain_id)
 384         LOGGER.debug("project: %s", self.project)
 385
 386     def _create_user(self):
 387         """Create user for tests."""
 388         self.user = self.cloud.create_user(
 389             name=getattr(
 390                 config.CONF, 'tempest_identity_user_name') + self.guid,
 391             password=getattr(config.CONF, 'tempest_identity_user_password'),
 392             default_project=getattr(
 393                 config.CONF, 'tempest_identity_tenant_name') + self.guid,
 394             domain_id=self.domain_id)
 395         LOGGER.debug("user: %s", self.user)
 396
 397     def _create_network(self):
 398         """Create network for tests."""
 399         tempest_net_name = getattr(
 400             config.CONF, 'tempest_private_net_name') + self.guid
 401         provider = {}
 402         if hasattr(config.CONF, 'tempest_network_type'):
 403             provider["network_type"] = getattr(
 404                 config.CONF, 'tempest_network_type')
 405         if hasattr(config.CONF, 'tempest_physical_network'):
 406             provider["physical_network"] = getattr(
 407                 config.CONF, 'tempest_physical_network')
 408         if hasattr(config.CONF, 'tempest_segmentation_id'):
 409             provider["segmentation_id"] = getattr(
 410                 config.CONF, 'tempest_segmentation_id')
 411         LOGGER.info(
 412             "Creating network with name: '%s'", tempest_net_name)
 413         self.network = self.cloud.create_network(
 414             tempest_net_name, provider=provider)
 415         LOGGER.debug("network: %s", self.network)
 416
 417         self.subnet = self.cloud.create_subnet(
 418             self.network.id,
 419             subnet_name=getattr(
 420                 config.CONF, 'tempest_private_subnet_name') + self.guid,
 421             cidr=getattr(config.CONF, 'tempest_private_subnet_cidr'),
 422             enable_dhcp=True,
 423             dns_nameservers=[env.get('NAMESERVER')])
 424         LOGGER.debug("subnet: %s", self.subnet)
 425
 426     def _create_image(self, name):
 427         """Create image for tests"""
 428         LOGGER.info("Creating image with name: '%s'", name)
 429         image = self.cloud.create_image(
 430             name, filename=getattr(config.CONF, 'openstack_image_url'),
 431             is_public=True)
 432         LOGGER.debug("image: %s", image)
 433         return image
 434
 435     def _create_flavor(self, name):
 436         """Create flavor for tests."""
 437         flavor = self.cloud.create_flavor(
 438             name, getattr(config.CONF, 'openstack_flavor_ram'),
 439             getattr(config.CONF, 'openstack_flavor_vcpus'),
 440             getattr(config.CONF, 'openstack_flavor_disk'))
 441         self.cloud.set_flavor_specs(
 442             flavor.id, getattr(config.CONF, 'flavor_extra_specs', {}))
 443         LOGGER.debug("flavor: %s", flavor)
 444         return flavor
 445
 446     def create(self, create_project=False):
 447         """Create resources for Tempest test suite."""
 448         if create_project:
 449             self._create_project()
 450             self._create_user()
 451         self._create_network()
 452
 453         LOGGER.debug("Creating two images for Tempest suite")
 454         self.image = self._create_image(
 455             getattr(config.CONF, 'openstack_image_name') + self.guid)
 456         self.image_alt = self._create_image(
 457             getattr(config.CONF, 'openstack_image_name_alt') + self.guid)
 458
 459         LOGGER.info("Creating two flavors for Tempest suite")
 460         self.flavor = self._create_flavor(
 461             getattr(config.CONF, 'openstack_flavor_name') + self.guid)
 462         self.flavor_alt = self._create_flavor(
 463             getattr(config.CONF, 'openstack_flavor_name_alt') + self.guid)
 464
 465     def cleanup(self):
 466         """
 467         Cleanup all OpenStack objects. Should be called on completion.
 468         """
 469         self.cloud.delete_image(self.image)
 470         self.cloud.delete_image(self.image_alt)
 471         self.cloud.delete_network(self.network.id)
 472         self.cloud.delete_flavor(self.flavor.id)
 473         self.cloud.delete_flavor(self.flavor_alt.id)
 474         if self.project:
 475             self.cloud.delete_user(self.user.id)
 476             self.cloud.delete_project(self.project.id)