functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21
  22 from six.moves import configparser
  23 from xtesting.core import testcase
  24 import yaml
  25
  26 from functest.core import singlevm
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30 from functest.utils import functest_utils
  31
  32 LOGGER = logging.getLogger(__name__)
  33
  34
  35 class TempestCommon(singlevm.VmReady2):
  36     # pylint: disable=too-many-instance-attributes
  37     """TempestCommon testcases implementation class."""
  38
  39     visibility = 'public'
  40     filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img'
  41
  42     def __init__(self, **kwargs):
  43         if "case_name" not in kwargs:
  44             kwargs["case_name"] = 'tempest'
  45         super(TempestCommon, self).__init__(**kwargs)
  46         assert self.orig_cloud
  47         assert self.cloud
  48         assert self.project
  49         if self.orig_cloud.get_role("admin"):
  50             self.role_name = "admin"
  51         elif self.orig_cloud.get_role("Admin"):
  52             self.role_name = "Admin"
  53         else:
  54             raise Exception("Cannot detect neither admin nor Admin")
  55         self.orig_cloud.grant_role(
  56             self.role_name, user=self.project.user.id,
  57             project=self.project.project.id,
  58             domain=self.project.domain.id)
  59         self.orig_cloud.grant_role(
  60             self.role_name, user=self.project.user.id,
  61             domain=self.project.domain.id)
  62         environ = dict(
  63             os.environ,
  64             OS_USERNAME=self.project.user.name,
  65             OS_PROJECT_NAME=self.project.project.name,
  66             OS_PROJECT_ID=self.project.project.id,
  67             OS_PASSWORD=self.project.password)
  68         try:
  69             del environ['OS_TENANT_NAME']
  70             del environ['OS_TENANT_ID']
  71         except Exception:  # pylint: disable=broad-except
  72             pass
  73         self.deployment_id = conf_utils.create_rally_deployment(
  74             environ=environ)
  75         if not self.deployment_id:
  76             raise Exception("Deployment create failed")
  77         self.verifier_id = conf_utils.create_verifier()
  78         if not self.verifier_id:
  79             raise Exception("Verifier create failed")
  80         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  81             self.verifier_id)
  82         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  83             self.verifier_id, self.deployment_id)
  84         self.verification_id = None
  85         self.res_dir = os.path.join(
  86             getattr(config.CONF, 'dir_results'), self.case_name)
  87         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  88         self.list = os.path.join(self.res_dir, 'test_list.txt')
  89         self.conf_file = None
  90         self.image_alt = None
  91         self.flavor_alt = None
  92         self.services = []
  93         try:
  94             self.services = kwargs['run']['args']['services']
  95         except Exception:  # pylint: disable=broad-except
  96             pass
  97         self.neutron_extensions = []
  98         try:
  99             self.neutron_extensions = kwargs['run']['args'][
 100                 'neutron_extensions']
 101         except Exception:  # pylint: disable=broad-except
 102             pass
 103
 104     def create_network_resources(self):
 105         pass
 106
 107     def check_services(self):
 108         """Check the mandatory services."""
 109         for service in self.services:
 110             try:
 111                 self.cloud.search_services(service)[0]
 112             except Exception:  # pylint: disable=broad-except
 113                 self.is_skipped = True
 114                 break
 115
 116     def check_extensions(self):
 117         """Check the mandatory network extensions."""
 118         extensions = self.cloud.get_network_extensions()
 119         for network_extension in self.neutron_extensions:
 120             if network_extension not in extensions:
 121                 LOGGER.warning(
 122                     "Cannot find Neutron extension: %s", network_extension)
 123                 self.is_skipped = True
 124                 break
 125
 126     def check_requirements(self):
 127         self.check_services()
 128         self.check_extensions()
 129         if self.is_skipped:
 130             self.project.clean()
 131
 132     @staticmethod
 133     def read_file(filename):
 134         """Read file and return content as a stripped list."""
 135         with open(filename) as src:
 136             return [line.strip() for line in src.readlines()]
 137
 138     @staticmethod
 139     def get_verifier_result(verif_id):
 140         """Retrieve verification results."""
 141         result = {
 142             'num_tests': 0,
 143             'num_success': 0,
 144             'num_failures': 0,
 145             'num_skipped': 0
 146         }
 147         cmd = ["rally", "verify", "show", "--uuid", verif_id]
 148         LOGGER.info("Showing result for a verification: '%s'.", cmd)
 149         proc = subprocess.Popen(cmd,
 150                                 stdout=subprocess.PIPE,
 151                                 stderr=subprocess.STDOUT)
 152         for line in proc.stdout:
 153             LOGGER.info(line.rstrip())
 154             new_line = line.replace(' ', '').split('|')
 155             if 'Tests' in new_line:
 156                 break
 157             if 'Testscount' in new_line:
 158                 result['num_tests'] = int(new_line[2])
 159             elif 'Success' in new_line:
 160                 result['num_success'] = int(new_line[2])
 161             elif 'Skipped' in new_line:
 162                 result['num_skipped'] = int(new_line[2])
 163             elif 'Failures' in new_line:
 164                 result['num_failures'] = int(new_line[2])
 165         return result
 166
 167     @staticmethod
 168     def backup_tempest_config(conf_file, res_dir):
 169         """
 170         Copy config file to tempest results directory
 171         """
 172         if not os.path.exists(res_dir):
 173             os.makedirs(res_dir)
 174         shutil.copyfile(conf_file,
 175                         os.path.join(res_dir, 'tempest.conf'))
 176
 177     def generate_test_list(self, **kwargs):
 178         """Generate test list based on the test mode."""
 179         LOGGER.debug("Generating test case list...")
 180         self.backup_tempest_config(self.conf_file, '/etc')
 181         if kwargs.get('mode') == 'custom':
 182             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 183                 shutil.copyfile(
 184                     conf_utils.TEMPEST_CUSTOM, self.list)
 185             else:
 186                 raise Exception("Tempest test list file %s NOT found."
 187                                 % conf_utils.TEMPEST_CUSTOM)
 188         else:
 189             testr_mode = kwargs.get(
 190                 'mode', r'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$')
 191             cmd = "(cd {0}; stestr list '{1}' >{2} 2>/dev/null)".format(
 192                 self.verifier_repo_dir, testr_mode, self.list)
 193             output = subprocess.check_output(cmd, shell=True)
 194             LOGGER.info("%s\n%s", cmd, output)
 195         os.remove('/etc/tempest.conf')
 196
 197     def apply_tempest_blacklist(self):
 198         """Exclude blacklisted test cases."""
 199         LOGGER.debug("Applying tempest blacklist...")
 200         if os.path.exists(self.raw_list):
 201             os.remove(self.raw_list)
 202         os.rename(self.list, self.raw_list)
 203         cases_file = self.read_file(self.raw_list)
 204         result_file = open(self.list, 'w')
 205         black_tests = []
 206         try:
 207             deploy_scenario = env.get('DEPLOY_SCENARIO')
 208             if bool(deploy_scenario):
 209                 # if DEPLOY_SCENARIO is set we read the file
 210                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 211                 black_list_yaml = yaml.safe_load(black_list_file)
 212                 black_list_file.close()
 213                 for item in black_list_yaml:
 214                     scenarios = item['scenarios']
 215                     if deploy_scenario in scenarios:
 216                         tests = item['tests']
 217                         for test in tests:
 218                             black_tests.append(test)
 219                         break
 220         except Exception:  # pylint: disable=broad-except
 221             black_tests = []
 222             LOGGER.debug("Tempest blacklist file does not exist.")
 223
 224         for cases_line in cases_file:
 225             for black_tests_line in black_tests:
 226                 if black_tests_line in cases_line:
 227                     break
 228             else:
 229                 result_file.write(str(cases_line) + '\n')
 230         result_file.close()
 231
 232     def run_verifier_tests(self, **kwargs):
 233         """Execute tempest test cases."""
 234         cmd = ["rally", "verify", "start", "--load-list",
 235                self.list]
 236         cmd.extend(kwargs.get('option', []))
 237         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 238
 239         f_stdout = open(
 240             os.path.join(self.res_dir, "tempest.log"), 'w+')
 241
 242         proc = subprocess.Popen(
 243             cmd,
 244             stdout=subprocess.PIPE,
 245             stderr=subprocess.STDOUT,
 246             bufsize=1)
 247
 248         with proc.stdout:
 249             for line in iter(proc.stdout.readline, b''):
 250                 if re.search(r"\} tempest\.", line):
 251                     LOGGER.info(line.rstrip())
 252                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 253                     self.verification_id = re.search(
 254                         r'(?=\(UUID=(.*)\))', line).group(1)
 255                 f_stdout.write(line)
 256         proc.wait()
 257         f_stdout.close()
 258
 259         if self.verification_id is None:
 260             raise Exception('Verification UUID not found')
 261         LOGGER.info('Verification UUID: %s', self.verification_id)
 262
 263         shutil.copy(
 264             "{}/tempest.log".format(self.deployment_dir),
 265             "{}/tempest.debug.log".format(self.res_dir))
 266
 267     def parse_verifier_result(self):
 268         """Parse and save test results."""
 269         stat = self.get_verifier_result(self.verification_id)
 270         try:
 271             num_executed = stat['num_tests'] - stat['num_skipped']
 272             try:
 273                 self.result = 100 * stat['num_success'] / num_executed
 274             except ZeroDivisionError:
 275                 self.result = 0
 276                 if stat['num_tests'] > 0:
 277                     LOGGER.info("All tests have been skipped")
 278                 else:
 279                     LOGGER.error("No test has been executed")
 280                     return
 281
 282             with open(os.path.join(self.res_dir,
 283                                    "tempest.log"), 'r') as logfile:
 284                 output = logfile.read()
 285
 286             success_testcases = []
 287             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 288                                     output):
 289                 success_testcases.append(match)
 290             failed_testcases = []
 291             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 292                                     output):
 293                 failed_testcases.append(match)
 294             skipped_testcases = []
 295             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 296                                     output):
 297                 skipped_testcases.append(match)
 298
 299             self.details = {"tests_number": stat['num_tests'],
 300                             "success_number": stat['num_success'],
 301                             "skipped_number": stat['num_skipped'],
 302                             "failures_number": stat['num_failures'],
 303                             "success": success_testcases,
 304                             "skipped": skipped_testcases,
 305                             "failures": failed_testcases}
 306         except Exception:  # pylint: disable=broad-except
 307             self.result = 0
 308
 309         LOGGER.info("Tempest %s success_rate is %s%%",
 310                     self.case_name, self.result)
 311
 312     def generate_report(self):
 313         """Generate verification report."""
 314         html_file = os.path.join(self.res_dir,
 315                                  "tempest-report.html")
 316         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 317                self.verification_id, "--to", html_file]
 318         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 319                          stderr=subprocess.STDOUT)
 320
 321     def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
 322         """Set image name as tempest img_name_regex"""
 323         rconfig = configparser.RawConfigParser()
 324         rconfig.read(rally_conf)
 325         if not rconfig.has_section('openstack'):
 326             rconfig.add_section('openstack')
 327         rconfig.set('openstack', 'img_name_regex', '^{}$'.format(
 328             self.image.name))
 329         with open(rally_conf, 'wb') as config_file:
 330             rconfig.write(config_file)
 331
 332     def update_default_role(self, rally_conf='/etc/rally/rally.conf'):
 333         """Detect and update the default role if required"""
 334         role = self.get_default_role(self.cloud)
 335         if not role:
 336             return
 337         rconfig = configparser.RawConfigParser()
 338         rconfig.read(rally_conf)
 339         if not rconfig.has_section('openstack'):
 340             rconfig.add_section('openstack')
 341         rconfig.set('openstack', 'swift_operator_role', role.name)
 342         with open(rally_conf, 'wb') as config_file:
 343             rconfig.write(config_file)
 344
 345     def update_rally_logs(self, rally_conf='/etc/rally/rally.conf'):
 346         """Print rally logs in res dir"""
 347         if not os.path.exists(self.res_dir):
 348             os.makedirs(self.res_dir)
 349         rconfig = configparser.RawConfigParser()
 350         rconfig.read(rally_conf)
 351         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 352         rconfig.set('DEFAULT', 'log_dir', self.res_dir)
 353         with open(rally_conf, 'wb') as config_file:
 354             rconfig.write(config_file)
 355
 356     @staticmethod
 357     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 358         """Clean Rally config"""
 359         rconfig = configparser.RawConfigParser()
 360         rconfig.read(rally_conf)
 361         if rconfig.has_option('openstack', 'img_name_regex'):
 362             rconfig.remove_option('openstack', 'img_name_regex')
 363         if rconfig.has_option('openstack', 'swift_operator_role'):
 364             rconfig.remove_option('openstack', 'swift_operator_role')
 365         if rconfig.has_option('DEFAULT', 'log-file'):
 366             rconfig.remove_option('DEFAULT', 'log-file')
 367         if rconfig.has_option('DEFAULT', 'log_dir'):
 368             rconfig.remove_option('DEFAULT', 'log_dir')
 369         with open(rally_conf, 'wb') as config_file:
 370             rconfig.write(config_file)
 371
 372     def update_scenario_section(self):
 373         """Update scenario section in tempest.conf"""
 374         rconfig = configparser.RawConfigParser()
 375         rconfig.read(self.conf_file)
 376         filename = getattr(
 377             config.CONF, '{}_image'.format(self.case_name), self.filename)
 378         if not rconfig.has_section('scenario'):
 379             rconfig.add_section('scenario')
 380         rconfig.set('scenario', 'img_file', os.path.basename(filename))
 381         rconfig.set('scenario', 'img_dir', os.path.dirname(filename))
 382         rconfig.set('scenario', 'img_disk_format', getattr(
 383             config.CONF, '{}_image_format'.format(self.case_name),
 384             self.image_format))
 385         extra_properties = self.extra_properties.copy()
 386         if env.get('IMAGE_PROPERTIES'):
 387             extra_properties.update(
 388                 functest_utils.convert_ini_to_dict(
 389                     env.get('IMAGE_PROPERTIES')))
 390         extra_properties.update(
 391             getattr(config.CONF, '{}_extra_properties'.format(
 392                 self.case_name), {}))
 393         rconfig.set(
 394             'scenario', 'img_properties',
 395             functest_utils.convert_dict_to_ini(extra_properties))
 396         with open(self.conf_file, 'wb') as config_file:
 397             rconfig.write(config_file)
 398
 399     def configure(self, **kwargs):  # pylint: disable=unused-argument
 400         """
 401         Create all openstack resources for tempest-based testcases and write
 402         tempest.conf.
 403         """
 404         if not os.path.exists(self.res_dir):
 405             os.makedirs(self.res_dir)
 406         compute_cnt = len(self.cloud.list_hypervisors())
 407
 408         self.image_alt = self.publish_image_alt()
 409         self.flavor_alt = self.create_flavor_alt()
 410         LOGGER.debug("flavor: %s", self.flavor_alt)
 411
 412         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 413         if not self.conf_file:
 414             raise Exception("Tempest verifier configuring failed")
 415         conf_utils.configure_tempest_update_params(
 416             self.conf_file,
 417             image_id=self.image.id,
 418             flavor_id=self.flavor.id,
 419             compute_cnt=compute_cnt,
 420             image_alt_id=self.image_alt.id,
 421             flavor_alt_id=self.flavor_alt.id,
 422             admin_role_name=self.role_name, cidr=self.cidr,
 423             domain_id=self.project.domain.id)
 424         self.update_scenario_section()
 425         self.backup_tempest_config(self.conf_file, self.res_dir)
 426
 427     def run(self, **kwargs):
 428         self.start_time = time.time()
 429         try:
 430             assert super(TempestCommon, self).run(
 431                 **kwargs) == testcase.TestCase.EX_OK
 432             if not os.path.exists(self.res_dir):
 433                 os.makedirs(self.res_dir)
 434             self.update_rally_regex()
 435             self.update_default_role()
 436             self.update_rally_logs()
 437             shutil.copy("/etc/rally/rally.conf", self.res_dir)
 438             self.configure(**kwargs)
 439             self.generate_test_list(**kwargs)
 440             self.apply_tempest_blacklist()
 441             self.run_verifier_tests(**kwargs)
 442             self.parse_verifier_result()
 443             self.generate_report()
 444             res = testcase.TestCase.EX_OK
 445         except Exception:  # pylint: disable=broad-except
 446             LOGGER.exception('Error with run')
 447             self.result = 0
 448             res = testcase.TestCase.EX_RUN_ERROR
 449         self.stop_time = time.time()
 450         return res
 451
 452     def clean(self):
 453         """
 454         Cleanup all OpenStack objects. Should be called on completion.
 455         """
 456         self.clean_rally_conf()
 457         if self.image_alt:
 458             self.cloud.delete_image(self.image_alt)
 459         if self.flavor_alt:
 460             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 461         super(TempestCommon, self).clean()