functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(singlevm.VmReady2):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     visibility = 'public'
  39     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  40
  41     def __init__(self, **kwargs):
  42         if "case_name" not in kwargs:
  43             kwargs["case_name"] = 'tempest'
  44         super(TempestCommon, self).__init__(**kwargs)
  45         assert self.orig_cloud
  46         assert self.cloud
  47         assert self.project
  48         if self.orig_cloud.get_role("admin"):
  49             self.role_name = "admin"
  50         elif self.orig_cloud.get_role("Admin"):
  51             self.role_name = "Admin"
  52         else:
  53             raise Exception("Cannot detect neither admin nor Admin")
  54         self.orig_cloud.grant_role(
  55             self.role_name, user=self.project.user.id,
  56             project=self.project.project.id,
  57             domain=self.project.domain.id)
  58         self.orig_cloud.grant_role(
  59             self.role_name, user=self.project.user.id,
  60             domain=self.project.domain.id)
  61         environ = dict(
  62             os.environ,
  63             OS_USERNAME=self.project.user.name,
  64             OS_PROJECT_NAME=self.project.project.name,
  65             OS_PROJECT_ID=self.project.project.id,
  66             OS_PASSWORD=self.project.password)
  67         try:
  68             del environ['OS_TENANT_NAME']
  69             del environ['OS_TENANT_ID']
  70         except Exception:  # pylint: disable=broad-except
  71             pass
  72         self.deployment_id = conf_utils.create_rally_deployment(
  73             environ=environ)
  74         if not self.deployment_id:
  75             raise Exception("Deployment create failed")
  76         self.verifier_id = conf_utils.create_verifier()
  77         if not self.verifier_id:
  78             raise Exception("Verifier create failed")
  79         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  80             self.verifier_id)
  81         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  82             self.verifier_id, self.deployment_id)
  83         self.verification_id = None
  84         self.res_dir = os.path.join(
  85             getattr(config.CONF, 'dir_results'), self.case_name)
  86         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  87         self.list = os.path.join(self.res_dir, 'test_list.txt')
  88         self.conf_file = None
  89         self.image_alt = None
  90         self.flavor_alt = None
  91         self.services = []
  92         try:
  93             self.services = kwargs['run']['args']['services']
  94         except Exception:  # pylint: disable=broad-except
  95             pass
  96         self.neutron_extensions = []
  97         try:
  98             self.neutron_extensions = kwargs['run']['args'][
  99                 'neutron_extensions']
 100         except Exception:  # pylint: disable=broad-except
 101             pass
 102
 103     def create_network_resources(self):
 104         pass
 105
 106     def check_services(self):
 107         """Check the mandatory services."""
 108         for service in self.services:
 109             try:
 110                 self.cloud.search_services(service)[0]
 111             except Exception:  # pylint: disable=broad-except
 112                 self.is_skipped = True
 113                 break
 114
 115     def check_extensions(self):
 116         """Check the mandatory network extensions."""
 117         extensions = self.cloud.get_network_extensions()
 118         for network_extension in self.neutron_extensions:
 119             if network_extension not in extensions:
 120                 LOGGER.warning(
 121                     "Cannot find Neutron extension: %s", network_extension)
 122                 self.is_skipped = True
 123                 break
 124
 125     def check_requirements(self):
 126         self.check_services()
 127         self.check_extensions()
 128         if self.is_skipped:
 129             self.project.clean()
 130
 131     @staticmethod
 132     def read_file(filename):
 133         """Read file and return content as a stripped list."""
 134         with open(filename) as src:
 135             return [line.strip() for line in src.readlines()]
 136
 137     @staticmethod
 138     def get_verifier_result(verif_id):
 139         """Retrieve verification results."""
 140         result = {
 141             'num_tests': 0,
 142             'num_success': 0,
 143             'num_failures': 0,
 144             'num_skipped': 0
 145         }
 146         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 147         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 148         proc = subprocess.Popen(cmd,
 149                                 stdout=subprocess.PIPE,
 150                                 stderr=subprocess.STDOUT)
 151         for line in proc.stdout:
 152             LOGGER.info(line.rstrip())
 153             new_line = line.replace(' ', '').split('|')
 154             if 'Tests' in new_line:
 155                 break
 156             if 'Testscount' in new_line:
 157                 result['num_tests'] = int(new_line[2])
 158             elif 'Success' in new_line:
 159                 result['num_success'] = int(new_line[2])
 160             elif 'Skipped' in new_line:
 161                 result['num_skipped'] = int(new_line[2])
 162             elif 'Failures' in new_line:
 163                 result['num_failures'] = int(new_line[2])
 164         return result
 165
 166     @staticmethod
 167     def backup_tempest_config(conf_file, res_dir):
 168         """
 169         Copy config file to tempest results directory
 170         """
 171         if not os.path.exists(res_dir):
 172             os.makedirs(res_dir)
 173         shutil.copyfile(conf_file,
 174                         os.path.join(res_dir, 'tempest.conf'))
 175
 176     def generate_test_list(self, **kwargs):
 177         """Generate test list based on the test mode."""
 178         LOGGER.debug("Generating test case list...")
 179         self.backup_tempest_config(self.conf_file, '/etc')
 180         if kwargs.get('mode') == 'custom':
 181             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 182                 shutil.copyfile(
 183                     conf_utils.TEMPEST_CUSTOM, self.list)
 184             else:
 185                 raise Exception("Tempest test list file %s NOT found."
 186                                 % conf_utils.TEMPEST_CUSTOM)
 187         else:
 188             testr_mode = kwargs.get(
 189                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 190             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 191                 self.verifier_repo_dir, testr_mode, self.list)
 192             output = subprocess.check_output(cmd, shell=True)
 193             LOGGER.info("%s\n%s", cmd, output)
 194         os.remove('/etc/tempest.conf')
 195
 196     def apply_tempest_blacklist(self):
 197         """Exclude blacklisted test cases."""
 198         LOGGER.debug("Applying tempest blacklist...")
 199         if os.path.exists(self.raw_list):
 200             os.remove(self.raw_list)
 201         os.rename(self.list, self.raw_list)
 202         cases_file = self.read_file(self.raw_list)
 203         result_file = open(self.list, 'w')
 204         black_tests = []
 205         try:
 206             deploy_scenario = env.get('DEPLOY_SCENARIO')
 207             if bool(deploy_scenario):
 208                 # if DEPLOY_SCENARIO is set we read the file
 209                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 210                 black_list_yaml = yaml.safe_load(black_list_file)
 211                 black_list_file.close()
 212                 for item in black_list_yaml:
 213                     scenarios = item['scenarios']
 214                     if deploy_scenario in scenarios:
 215                         tests = item['tests']
 216                         for test in tests:
 217                             black_tests.append(test)
 218                         break
 219         except Exception:  # pylint: disable=broad-except
 220             black_tests = []
 221             LOGGER.debug("Tempest blacklist file does not exist.")
 222
 223         for cases_line in cases_file:
 224             for black_tests_line in black_tests:
 225                 if black_tests_line in cases_line:
 226                     break
 227             else:
 228                 result_file.write(str(cases_line) + '\n')
 229         result_file.close()
 230
 231     def run_verifier_tests(self, **kwargs):
 232         """Execute tempest test cases."""
 233         cmd = ["rally", "verify", "start", "--load-list",
 234                self.list]
 235         cmd.extend(kwargs.get('option', []))
 236         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 237
 238         f_stdout = open(
 239             os.path.join(self.res_dir, "tempest.log"), 'w+')
 240
 241         proc = subprocess.Popen(
 242             cmd,
 243             stdout=subprocess.PIPE,
 244             stderr=subprocess.STDOUT,
 245             bufsize=1)
 246
 247         with proc.stdout:
 248             for line in iter(proc.stdout.readline, b''):
 249                 if re.search(r"\} tempest\.", line):
 250                     LOGGER.info(line.rstrip())
 251                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 252                     self.verification_id = re.search(
 253                         r'(?=\(UUID=(.*)\))', line).group(1)
 254                 f_stdout.write(line)
 255         proc.wait()
 256         f_stdout.close()
 257
 258         if self.verification_id is None:
 259             raise Exception('Verification UUID not found')
 260         LOGGER.info('Verification UUID: %s', self.verification_id)
 261
 262         shutil.copy(
 263             "{}/tempest.log".format(self.deployment_dir),
 264             "{}/tempest.debug.log".format(self.res_dir))
 265
 266     def parse_verifier_result(self):
 267         """Parse and save test results."""
 268         stat = self.get_verifier_result(self.verification_id)
 269         try:
 270             num_executed = stat['num_tests'] - stat['num_skipped']
 271             try:
 272                 self.result = 100 * stat['num_success'] / num_executed
 273             except ZeroDivisionError:
 274                 self.result = 0
 275                 if stat['num_tests'] > 0:
 276                     LOGGER.info("All tests have been skipped")
 277                 else:
 278                     LOGGER.error("No test has been executed")
 279                     return
 280
 281             with open(os.path.join(self.res_dir,
 282                                    "tempest.log"), 'r') as logfile:
 283                 output = logfile.read()
 284
 285             success_testcases = []
 286             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 287                                     output):
 288                 success_testcases.append(match)
 289             failed_testcases = []
 290             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 291                                     output):
 292                 failed_testcases.append(match)
 293             skipped_testcases = []
 294             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 295                                     output):
 296                 skipped_testcases.append(match)
 297
 298             self.details = {"tests_number": stat['num_tests'],
 299                             "success_number": stat['num_success'],
 300                             "skipped_number": stat['num_skipped'],
 301                             "failures_number": stat['num_failures'],
 302                             "success": success_testcases,
 303                             "skipped": skipped_testcases,
 304                             "failures": failed_testcases}
 305         except Exception:  # pylint: disable=broad-except
 306             self.result = 0
 307
 308         LOGGER.info("Tempest %s success_rate is %s%%",
 309                     self.case_name, self.result)
 310
 311     def generate_report(self):
 312         """Generate verification report."""
 313         html_file = os.path.join(self.res_dir,
 314                                  "tempest-report.html")
 315         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 316                self.verification_id, "--to", html_file]
 317         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 318                          stderr=subprocess.STDOUT)
 319
 320     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 321         """Set image name as tempest img_name_regex"""
 322         rconfig = configparser.RawConfigParser()
 323         rconfig.read(rally_conf)
 324         if not rconfig.has_section('openstack'):
 325             rconfig.add_section('openstack')
 326         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 327             self.image.name))
 328         with open(rally_conf, 'wb') as config_file:
 329             rconfig.write(config_file)
 330
 331     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 332         """Detect and update the default role if required"""
 333         role = self.get_default_role(self.cloud)
 334         if not role:
 335             return
 336         rconfig = configparser.RawConfigParser()
 337         rconfig.read(rally_conf)
 338         if not rconfig.has_section('openstack'):
 339             rconfig.add_section('openstack')
 340         rconfig.set('openstack', 'swift_operator_role', role.name)
 341         with open(rally_conf, 'wb') as config_file:
 342             rconfig.write(config_file)
 343
 344     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 345         """Print rally logs in res dir"""
 346         if not os.path.exists(self.res_dir):
 347             os.makedirs(self.res_dir)
 348         rconfig = configparser.RawConfigParser()
 349         rconfig.read(rally_conf)
 350         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 351         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 352         with open(rally_conf, 'wb') as config_file:
 353             rconfig.write(config_file)
 354
 355     @staticmethod
 356     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 357         """Clean Rally config"""
 358         rconfig = configparser.RawConfigParser()
 359         rconfig.read(rally_conf)
 360         if rconfig.has_option('openstack', 'img_name_regex'):
 361             rconfig.remove_option('openstack', 'img_name_regex')
 362         if rconfig.has_option('openstack', 'swift_operator_role'):
 363             rconfig.remove_option('openstack', 'swift_operator_role')
 364         if rconfig.has_option('DEFAULT', 'log-file'):
 365             rconfig.remove_option('DEFAULT', 'log-file')
 366         if rconfig.has_option('DEFAULT', 'log_dir'):
 367             rconfig.remove_option('DEFAULT', 'log_dir')
 368         with open(rally_conf, 'wb') as config_file:
 369             rconfig.write(config_file)
 370
 371     def update_scenario_section(self):
 372         """Update scenario section in tempest.conf"""
 373         rconfig = configparser.RawConfigParser()
 374         rconfig.read(self.conf_file)
 375         filename = getattr(
 376             config.CONF, '{}_image'.format(self.case_name), self.filename)
 377         if not rconfig.has_section('scenario'):
 378             rconfig.add_section('scenario')
 379         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 380         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 381         rconfig.set('scenario', 'img_disk_format', getattr(
 382             config.CONF, '{}_image_format'.format(self.case_name),
 383             self.image_format))
 384         extra_properties = self.extra_properties.copy()
 385         if env.get('IMG_PROP'):
 386             extra_properties.update(dict((k.strip(), v.strip()) for k, v in
 387                                          (item.split(': ') for item in
 388                                           env.get('IMG_PROP').split(','))))
 389         extra_properties.update(
 390             getattr(config.CONF, '{}_extra_properties'.format(
 391                 self.case_name), {}))
 392         rconfig.set(
 393             'scenario', 'img_properties',
 394             conf_utils.convert_dict_to_ini(extra_properties))
 395         with open(self.conf_file, 'wb') as config_file:
 396             rconfig.write(config_file)
 397
 398     def configure(self, **kwargs):  # pylint: disable=unused-argument
 399         """
 400         Create all openstack resources for tempest-based testcases and write
 401         tempest.conf.
 402         """
 403         if not os.path.exists(self.res_dir):
 404             os.makedirs(self.res_dir)
 405         compute_cnt = len(self.cloud.list_hypervisors())
 406
 407         self.image_alt = self.publish_image_alt()
 408         self.flavor_alt = self.create_flavor_alt()
 409         LOGGER.debug("flavor: %s", self.flavor_alt)
 410
 411         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 412         if not self.conf_file:
 413             raise Exception("Tempest verifier configuring failed")
 414         conf_utils.configure_tempest_update_params(
 415             self.conf_file,
 416             image_id=self.image.id,
 417             flavor_id=self.flavor.id,
 418             compute_cnt=compute_cnt,
 419             image_alt_id=self.image_alt.id,
 420             flavor_alt_id=self.flavor_alt.id,
 421             admin_role_name=self.role_name, cidr=self.cidr,
 422             domain_id=self.project.domain.id)
 423         self.update_scenario_section()
 424         self.backup_tempest_config(self.conf_file, self.res_dir)
 425
 426     def run(self, **kwargs):
 427         self.start_time = time.time()
 428         try:
 429             assert super(TempestCommon, self).run(
 430                 **kwargs) == testcase.TestCase.EX_OK
 431             if not os.path.exists(self.res_dir):
 432                 os.makedirs(self.res_dir)
 433             self.update_rally_regex()
 434             self.update_default_role()
 435             self.update_rally_logs()
 436             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 437             self.configure(**kwargs)
 438             self.generate_test_list(**kwargs)
 439             self.apply_tempest_blacklist()
 440             self.run_verifier_tests(**kwargs)
 441             self.parse_verifier_result()
 442             self.generate_report()
 443             res = testcase.TestCase.EX_OK
 444         except Exception:  # pylint: disable=broad-except
 445             LOGGER.exception('Error with run')
 446             self.result = 0
 447             res = testcase.TestCase.EX_RUN_ERROR
 448         self.stop_time = time.time()
 449         return res
 450
 451     def clean(self):
 452         """
 453         Cleanup all OpenStack objects. Should be called on completion.
 454         """
 455         self.clean_rally_conf()
 456         if self.image_alt:
 457             self.cloud.delete_image(self.image_alt)
 458         if self.flavor_alt:
 459             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 460         super(TempestCommon, self).clean()