functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.rally import rally
  28 from functest.opnfv_tests.openstack.tempest import conf_utils
  29 from functest.utils import config
  30 from functest.utils import env
  31 from functest.utils import functest_utils
  32
  33 LOGGER = logging.getLogger(__name__)
  34
  35
  36 class TempestCommon(singlevm.VmReady2):
  37     # pylint: disable=too-many-instance-attributes,too-many-public-methods
  38     """TempestCommon testcases implementation class."""
  39
  40     visibility = 'public'
  41     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  42     shared_network = True
  43
  44     def __init__(self, **kwargs):
  45         if "case_name" not in kwargs:
  46             kwargs["case_name"] = 'tempest'
  47         super(TempestCommon, self).__init__(**kwargs)
  48         assert self.orig_cloud
  49         assert self.cloud
  50         assert self.project
  51         if self.orig_cloud.get_role("admin"):
  52             self.role_name = "admin"
  53         elif self.orig_cloud.get_role("Admin"):
  54             self.role_name = "Admin"
  55         else:
  56             raise Exception("Cannot detect neither admin nor Admin")
  57         self.orig_cloud.grant_role(
  58             self.role_name, user=self.project.user.id,
  59             project=self.project.project.id,
  60             domain=self.project.domain.id)
  61         self.orig_cloud.grant_role(
  62             self.role_name, user=self.project.user.id,
  63             domain=self.project.domain.id)
  64         self.deployment_id = None
  65         self.verifier_id = None
  66         self.verifier_repo_dir = None
  67         self.deployment_dir = None
  68         self.verification_id = None
  69         self.res_dir = os.path.join(
  70             getattr(config.CONF, 'dir_results'), self.case_name)
  71         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  72         self.list = os.path.join(self.res_dir, 'test_list.txt')
  73         self.conf_file = None
  74         self.image_alt = None
  75         self.flavor_alt = None
  76         self.services = []
  77         try:
  78             self.services = kwargs['run']['args']['services']
  79         except Exception:  # pylint: disable=broad-except
  80             pass
  81         self.neutron_extensions = []
  82         try:
  83             self.neutron_extensions = kwargs['run']['args'][
  84                 'neutron_extensions']
  85         except Exception:  # pylint: disable=broad-except
  86             pass
  87
  88     def check_services(self):
  89         """Check the mandatory services."""
  90         for service in self.services:
  91             try:
  92                 self.cloud.search_services(service)[0]
  93             except Exception:  # pylint: disable=broad-except
  94                 self.is_skipped = True
  95                 break
  96
  97     def check_extensions(self):
  98         """Check the mandatory network extensions."""
  99         extensions = self.cloud.get_network_extensions()
 100         for network_extension in self.neutron_extensions:
 101             if network_extension not in extensions:
 102                 LOGGER.warning(
 103                     "Cannot find Neutron extension: %s", network_extension)
 104                 self.is_skipped = True
 105                 break
 106
 107     def check_requirements(self):
 108         self.check_services()
 109         self.check_extensions()
 110         if self.is_skipped:
 111             self.project.clean()
 112
 113     @staticmethod
 114     def read_file(filename):
 115         """Read file and return content as a stripped list."""
 116         with open(filename) as src:
 117             return [line.strip() for line in src.readlines()]
 118
 119     @staticmethod
 120     def get_verifier_result(verif_id):
 121         """Retrieve verification results."""
 122         result = {
 123             'num_tests': 0,
 124             'num_success': 0,
 125             'num_failures': 0,
 126             'num_skipped': 0
 127         }
 128         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 129         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 130         proc = subprocess.Popen(cmd,
 131                                 stdout=subprocess.PIPE,
 132                                 stderr=subprocess.STDOUT)
 133         for line in proc.stdout:
 134             LOGGER.info(line.rstrip())
 135             new_line = line.replace(' ', '').split('|')
 136             if 'Tests' in new_line:
 137                 break
 138             if 'Testscount' in new_line:
 139                 result['num_tests'] = int(new_line[2])
 140             elif 'Success' in new_line:
 141                 result['num_success'] = int(new_line[2])
 142             elif 'Skipped' in new_line:
 143                 result['num_skipped'] = int(new_line[2])
 144             elif 'Failures' in new_line:
 145                 result['num_failures'] = int(new_line[2])
 146         return result
 147
 148     @staticmethod
 149     def backup_tempest_config(conf_file, res_dir):
 150         """
 151         Copy config file to tempest results directory
 152         """
 153         if not os.path.exists(res_dir):
 154             os.makedirs(res_dir)
 155         shutil.copyfile(conf_file,
 156                         os.path.join(res_dir, 'tempest.conf'))
 157
 158     def generate_test_list(self, **kwargs):
 159         """Generate test list based on the test mode."""
 160         LOGGER.debug("Generating test case list...")
 161         self.backup_tempest_config(self.conf_file, '/etc')
 162         if kwargs.get('mode') == 'custom':
 163             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 164                 shutil.copyfile(
 165                     conf_utils.TEMPEST_CUSTOM, self.list)
 166             else:
 167                 raise Exception("Tempest test list file %s NOT found."
 168                                 % conf_utils.TEMPEST_CUSTOM)
 169         else:
 170             testr_mode = kwargs.get(
 171                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 172             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 173                 self.verifier_repo_dir, testr_mode, self.list)
 174             output = subprocess.check_output(cmd, shell=True)
 175             LOGGER.info("%s\n%s", cmd, output)
 176         os.remove('/etc/tempest.conf')
 177
 178     def apply_tempest_blacklist(self):
 179         """Exclude blacklisted test cases."""
 180         LOGGER.debug("Applying tempest blacklist...")
 181         if os.path.exists(self.raw_list):
 182             os.remove(self.raw_list)
 183         os.rename(self.list, self.raw_list)
 184         cases_file = self.read_file(self.raw_list)
 185         result_file = open(self.list, 'w')
 186         black_tests = []
 187         try:
 188             deploy_scenario = env.get('DEPLOY_SCENARIO')
 189             if bool(deploy_scenario):
 190                 # if DEPLOY_SCENARIO is set we read the file
 191                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 192                 black_list_yaml = yaml.safe_load(black_list_file)
 193                 black_list_file.close()
 194                 for item in black_list_yaml:
 195                     scenarios = item['scenarios']
 196                     if deploy_scenario in scenarios:
 197                         tests = item['tests']
 198                         for test in tests:
 199                             black_tests.append(test)
 200                         break
 201         except Exception:  # pylint: disable=broad-except
 202             black_tests = []
 203             LOGGER.debug("Tempest blacklist file does not exist.")
 204
 205         for cases_line in cases_file:
 206             for black_tests_line in black_tests:
 207                 if black_tests_line in cases_line:
 208                     break
 209             else:
 210                 result_file.write(str(cases_line) + '\n')
 211         result_file.close()
 212
 213     def run_verifier_tests(self, **kwargs):
 214         """Execute tempest test cases."""
 215         cmd = ["rally", "verify", "start", "--load-list",
 216                self.list]
 217         cmd.extend(kwargs.get('option', []))
 218         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 219
 220         f_stdout = open(
 221             os.path.join(self.res_dir, "tempest.log"), 'w+')
 222
 223         proc = subprocess.Popen(
 224             cmd,
 225             stdout=subprocess.PIPE,
 226             stderr=subprocess.STDOUT,
 227             bufsize=1)
 228
 229         with proc.stdout:
 230             for line in iter(proc.stdout.readline, b''):
 231                 if re.search(r"\} tempest\.", line):
 232                     LOGGER.info(line.rstrip())
 233                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 234                     self.verification_id = re.search(
 235                         r'(?=\(UUID=(.*)\))', line).group(1)
 236                 f_stdout.write(line)
 237         proc.wait()
 238         f_stdout.close()
 239
 240         if self.verification_id is None:
 241             raise Exception('Verification UUID not found')
 242         LOGGER.info('Verification UUID: %s', self.verification_id)
 243
 244         shutil.copy(
 245             "{}/tempest.log".format(self.deployment_dir),
 246             "{}/tempest.debug.log".format(self.res_dir))
 247
 248     def parse_verifier_result(self):
 249         """Parse and save test results."""
 250         stat = self.get_verifier_result(self.verification_id)
 251         try:
 252             num_executed = stat['num_tests'] - stat['num_skipped']
 253             try:
 254                 self.result = 100 * stat['num_success'] / num_executed
 255             except ZeroDivisionError:
 256                 self.result = 0
 257                 if stat['num_tests'] > 0:
 258                     LOGGER.info("All tests have been skipped")
 259                 else:
 260                     LOGGER.error("No test has been executed")
 261                     return
 262
 263             with open(os.path.join(self.res_dir,
 264                                    "rally.log"), 'r') as logfile:
 265                 output = logfile.read()
 266
 267             success_testcases = []
 268             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 269                                     output):
 270                 success_testcases.append(match)
 271             failed_testcases = []
 272             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 273                                     output):
 274                 failed_testcases.append(match)
 275             skipped_testcases = []
 276             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )',
 277                                     output):
 278                 skipped_testcases.append(match)
 279
 280             self.details = {"tests_number": stat['num_tests'],
 281                             "success_number": stat['num_success'],
 282                             "skipped_number": stat['num_skipped'],
 283                             "failures_number": stat['num_failures'],
 284                             "success": success_testcases,
 285                             "skipped": skipped_testcases,
 286                             "failures": failed_testcases}
 287         except Exception:  # pylint: disable=broad-except
 288             self.result = 0
 289
 290         LOGGER.info("Tempest %s success_rate is %s%%",
 291                     self.case_name, self.result)
 292
 293     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 294         """Set image name as tempest img_name_regex"""
 295         rconfig = configparser.RawConfigParser()
 296         rconfig.read(rally_conf)
 297         if not rconfig.has_section('openstack'):
 298             rconfig.add_section('openstack')
 299         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 300             self.image.name))
 301         with open(rally_conf, 'wb') as config_file:
 302             rconfig.write(config_file)
 303
 304     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 305         """Detect and update the default role if required"""
 306         role = self.get_default_role(self.cloud)
 307         if not role:
 308             return
 309         rconfig = configparser.RawConfigParser()
 310         rconfig.read(rally_conf)
 311         if not rconfig.has_section('openstack'):
 312             rconfig.add_section('openstack')
 313         rconfig.set('openstack', 'swift_operator_role', role.name)
 314         with open(rally_conf, 'wb') as config_file:
 315             rconfig.write(config_file)
 316
 317     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 318         """Print rally logs in res dir"""
 319         if not os.path.exists(self.res_dir):
 320             os.makedirs(self.res_dir)
 321         rconfig = configparser.RawConfigParser()
 322         rconfig.read(rally_conf)
 323         rconfig.set('DEFAULT', 'debug', True)
 324         rconfig.set('DEFAULT', 'use_stderr', False)
 325         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 326         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 327         with open(rally_conf, 'wb') as config_file:
 328             rconfig.write(config_file)
 329
 330     @staticmethod
 331     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 332         """Clean Rally config"""
 333         rconfig = configparser.RawConfigParser()
 334         rconfig.read(rally_conf)
 335         if rconfig.has_option('openstack', 'img_name_regex'):
 336             rconfig.remove_option('openstack', 'img_name_regex')
 337         if rconfig.has_option('openstack', 'swift_operator_role'):
 338             rconfig.remove_option('openstack', 'swift_operator_role')
 339         if rconfig.has_option('DEFAULT', 'use_stderr'):
 340             rconfig.remove_option('DEFAULT', 'use_stderr')
 341         if rconfig.has_option('DEFAULT', 'debug'):
 342             rconfig.remove_option('DEFAULT', 'debug')
 343         if rconfig.has_option('DEFAULT', 'log-file'):
 344             rconfig.remove_option('DEFAULT', 'log-file')
 345         if rconfig.has_option('DEFAULT', 'log_dir'):
 346             rconfig.remove_option('DEFAULT', 'log_dir')
 347         with open(rally_conf, 'wb') as config_file:
 348             rconfig.write(config_file)
 349
 350     def update_network_section(self):
 351         """Update network section in tempest.conf"""
 352         rconfig = configparser.RawConfigParser()
 353         rconfig.read(self.conf_file)
 354         if not rconfig.has_section('network'):
 355             rconfig.add_section('network')
 356         rconfig.set('network', 'public_network_id', self.ext_net.id)
 357         rconfig.set('network', 'floating_network_name', self.ext_net.name)
 358         with open(self.conf_file, 'wb') as config_file:
 359             rconfig.write(config_file)
 360
 361     def update_compute_section(self):
 362         """Update compute section in tempest.conf"""
 363         rconfig = configparser.RawConfigParser()
 364         rconfig.read(self.conf_file)
 365         if not rconfig.has_section('compute'):
 366             rconfig.add_section('compute')
 367         rconfig.set('compute', 'fixed_network_name', self.network.name)
 368         with open(self.conf_file, 'wb') as config_file:
 369             rconfig.write(config_file)
 370
 371     def update_scenario_section(self):
 372         """Update scenario section in tempest.conf"""
 373         rconfig = configparser.RawConfigParser()
 374         rconfig.read(self.conf_file)
 375         filename = getattr(
 376             config.CONF, '{}_image'.format(self.case_name), self.filename)
 377         if not rconfig.has_section('scenario'):
 378             rconfig.add_section('scenario')
 379         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 380         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 381         rconfig.set('scenario', 'img_disk_format', getattr(
 382             config.CONF, '{}_image_format'.format(self.case_name),
 383             self.image_format))
 384         extra_properties = self.extra_properties.copy()
 385         if env.get('IMAGE_PROPERTIES'):
 386             extra_properties.update(
 387                 functest_utils.convert_ini_to_dict(
 388                     env.get('IMAGE_PROPERTIES')))
 389         extra_properties.update(
 390             getattr(config.CONF, '{}_extra_properties'.format(
 391                 self.case_name), {}))
 392         rconfig.set(
 393             'scenario', 'img_properties',
 394             functest_utils.convert_dict_to_ini(extra_properties))
 395         with open(self.conf_file, 'wb') as config_file:
 396             rconfig.write(config_file)
 397
 398     def configure(self, **kwargs):  # pylint: disable=unused-argument
 399         """
 400         Create all openstack resources for tempest-based testcases and write
 401         tempest.conf.
 402         """
 403         if not os.path.exists(self.res_dir):
 404             os.makedirs(self.res_dir)
 405         environ = dict(
 406             os.environ,
 407             OS_USERNAME=self.project.user.name,
 408             OS_PROJECT_NAME=self.project.project.name,
 409             OS_PROJECT_ID=self.project.project.id,
 410             OS_PASSWORD=self.project.password)
 411         try:
 412             del environ['OS_TENANT_NAME']
 413             del environ['OS_TENANT_ID']
 414         except Exception:  # pylint: disable=broad-except
 415             pass
 416         self.deployment_id = rally.RallyBase.create_rally_deployment(
 417             environ=environ)
 418         if not self.deployment_id:
 419             raise Exception("Deployment create failed")
 420         self.verifier_id = conf_utils.create_verifier()
 421         if not self.verifier_id:
 422             raise Exception("Verifier create failed")
 423         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
 424             self.verifier_id)
 425         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
 426             self.verifier_id, self.deployment_id)
 427
 428         compute_cnt = len(self.orig_cloud.list_hypervisors())
 429
 430         self.image_alt = self.publish_image_alt()
 431         self.flavor_alt = self.create_flavor_alt()
 432         LOGGER.debug("flavor: %s", self.flavor_alt)
 433
 434         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 435         if not self.conf_file:
 436             raise Exception("Tempest verifier configuring failed")
 437         conf_utils.configure_tempest_update_params(
 438             self.conf_file,
 439             image_id=self.image.id,
 440             flavor_id=self.flavor.id,
 441             compute_cnt=compute_cnt,
 442             image_alt_id=self.image_alt.id,
 443             flavor_alt_id=self.flavor_alt.id,
 444             admin_role_name=self.role_name, cidr=self.cidr,
 445             domain_id=self.project.domain.id)
 446         self.update_network_section()
 447         self.update_compute_section()
 448         self.update_scenario_section()
 449         self.backup_tempest_config(self.conf_file, self.res_dir)
 450
 451     def run(self, **kwargs):
 452         self.start_time = time.time()
 453         try:
 454             assert super(TempestCommon, self).run(
 455                 **kwargs) == testcase.TestCase.EX_OK
 456             if not os.path.exists(self.res_dir):
 457                 os.makedirs(self.res_dir)
 458             self.update_rally_regex()
 459             self.update_default_role()
 460             self.update_rally_logs()
 461             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 462             self.configure(**kwargs)
 463             self.generate_test_list(**kwargs)
 464             self.apply_tempest_blacklist()
 465             self.run_verifier_tests(**kwargs)
 466             self.parse_verifier_result()
 467             rally.RallyBase.verify_report(
 468                 os.path.join(self.res_dir, "tempest-report.html"),
 469                 self.verification_id)
 470             rally.RallyBase.verify_report(
 471                 os.path.join(self.res_dir, "tempest-report.xml"),
 472                 self.verification_id, "junit-xml")
 473             res = testcase.TestCase.EX_OK
 474         except Exception:  # pylint: disable=broad-except
 475             LOGGER.exception('Error with run')
 476             self.result = 0
 477             res = testcase.TestCase.EX_RUN_ERROR
 478         self.stop_time = time.time()
 479         return res
 480
 481     def clean(self):
 482         """
 483         Cleanup all OpenStack objects. Should be called on completion.
 484         """
 485         self.clean_rally_conf()
 486         if self.image_alt:
 487             self.cloud.delete_image(self.image_alt)
 488         if self.flavor_alt:
 489             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 490         super(TempestCommon, self).clean()