functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import os_client_config
  24 from xtesting.core import testcase
  25 import yaml
  26
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30
  31 LOGGER = logging.getLogger(__name__)
  32
  33
  34 class TempestCommon(testcase.TestCase):
  35     # pylint: disable=too-many-instance-attributes
  36     """TempestCommon testcases implementation class."""
  37
  38     TEMPEST_RESULTS_DIR = os.path.join(
  39         getattr(config.CONF, 'dir_results'), 'tempest')
  40
  41     def __init__(self, **kwargs):
  42         super(TempestCommon, self).__init__(**kwargs)
  43         self.resources = TempestResourcesManager()
  44         self.mode = ""
  45         self.option = []
  46         self.verifier_id = conf_utils.get_verifier_id()
  47         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  48             self.verifier_id)
  49         self.deployment_id = conf_utils.get_verifier_deployment_id()
  50         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  51             self.verifier_id, self.deployment_id)
  52         self.verification_id = None
  53         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  54         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  55         self.list = os.path.join(self.res_dir, 'test_list.txt')
  56         self.conf_file = None
  57
  58     @staticmethod
  59     def read_file(filename):
  60         """Read file and return content as a stripped list."""
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     @staticmethod
  65     def get_verifier_result(verif_id):
  66         """Retrieve verification results."""
  67         result = {
  68             'num_tests': 0,
  69             'num_success': 0,
  70             'num_failures': 0,
  71             'num_skipped': 0
  72         }
  73         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  74         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  75         proc = subprocess.Popen(cmd,
  76                                 stdout=subprocess.PIPE,
  77                                 stderr=subprocess.STDOUT)
  78         for line in proc.stdout:
  79             new_line = line.replace(' ', '').split('|')
  80             if 'Tests' in new_line:
  81                 break
  82             LOGGER.info(line)
  83             if 'Testscount' in new_line:
  84                 result['num_tests'] = int(new_line[2])
  85             elif 'Success' in new_line:
  86                 result['num_success'] = int(new_line[2])
  87             elif 'Skipped' in new_line:
  88                 result['num_skipped'] = int(new_line[2])
  89             elif 'Failures' in new_line:
  90                 result['num_failures'] = int(new_line[2])
  91         return result
  92
  93     @staticmethod
  94     def backup_tempest_config(conf_file, res_dir):
  95         """
  96         Copy config file to tempest results directory
  97         """
  98         if not os.path.exists(res_dir):
  99             os.makedirs(res_dir)
 100         shutil.copyfile(conf_file,
 101                         os.path.join(res_dir, 'tempest.conf'))
 102
 103     def generate_test_list(self):
 104         """Generate test list based on the test mode."""
 105         LOGGER.debug("Generating test case list...")
 106         if self.mode == 'custom':
 107             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 108                 shutil.copyfile(
 109                     conf_utils.TEMPEST_CUSTOM, self.list)
 110             else:
 111                 raise Exception("Tempest test list file %s NOT found."
 112                                 % conf_utils.TEMPEST_CUSTOM)
 113         else:
 114             if self.mode == 'smoke':
 115                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 116             elif self.mode == 'full':
 117                 testr_mode = r"'^tempest\.'"
 118             else:
 119                 testr_mode = self.mode
 120             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 121                 self.verifier_repo_dir, testr_mode, self.list)
 122             output = subprocess.check_output(cmd, shell=True)
 123             LOGGER.info("%s\n%s", cmd, output)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 188                     self.verification_id = re.search(
 189                         r'(?=\(UUID=(.*)\))', line).group(1)
 190                     LOGGER.info('Verification UUID: %s', self.verification_id)
 191                 f_stdout.write(line)
 192         proc.wait()
 193
 194         f_stdout.close()
 195         f_stderr.close()
 196
 197         if self.verification_id is None:
 198             raise Exception('Verification UUID not found')
 199
 200     def parse_verifier_result(self):
 201         """Parse and save test results."""
 202         stat = self.get_verifier_result(self.verification_id)
 203         try:
 204             num_executed = stat['num_tests'] - stat['num_skipped']
 205             try:
 206                 self.result = 100 * stat['num_success'] / num_executed
 207             except ZeroDivisionError:
 208                 self.result = 0
 209                 if stat['num_tests'] > 0:
 210                     LOGGER.info("All tests have been skipped")
 211                 else:
 212                     LOGGER.error("No test has been executed")
 213                     return
 214
 215             with open(os.path.join(self.res_dir,
 216                                    "tempest.log"), 'r') as logfile:
 217                 output = logfile.read()
 218
 219             success_testcases = []
 220             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 221                 success_testcases.append(match)
 222             failed_testcases = []
 223             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail', output):
 224                 failed_testcases.append(match)
 225             skipped_testcases = []
 226             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 227                 skipped_testcases.append(match)
 228
 229             self.details = {"tests_number": stat['num_tests'],
 230                             "success_number": stat['num_success'],
 231                             "skipped_number": stat['num_skipped'],
 232                             "failures_number": stat['num_failures'],
 233                             "success": success_testcases,
 234                             "skipped": skipped_testcases,
 235                             "failures": failed_testcases}
 236         except Exception:  # pylint: disable=broad-except
 237             self.result = 0
 238
 239         LOGGER.info("Tempest %s success_rate is %s%%",
 240                     self.case_name, self.result)
 241
 242     def generate_report(self):
 243         """Generate verification report."""
 244         html_file = os.path.join(self.res_dir,
 245                                  "tempest-report.html")
 246         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 247                self.verification_id, "--to", html_file]
 248         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 249                          stderr=subprocess.STDOUT)
 250
 251     def configure(self, **kwargs):  # pylint: disable=unused-argument
 252         """
 253         Create all openstack resources for tempest-based testcases and write
 254         tempest.conf.
 255         """
 256         if not os.path.exists(self.res_dir):
 257             os.makedirs(self.res_dir)
 258         self.resources.create()
 259         compute_cnt = len(self.resources.cloud.list_hypervisors())
 260         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 261         conf_utils.configure_tempest_update_params(
 262             self.conf_file, network_name=self.resources.network.id,
 263             image_id=self.resources.image.id,
 264             flavor_id=self.resources.flavor.id,
 265             compute_cnt=compute_cnt,
 266             image_alt_id=self.resources.image_alt.id,
 267             flavor_alt_id=self.resources.flavor_alt.id)
 268         self.backup_tempest_config(self.conf_file, self.res_dir)
 269
 270     def run(self, **kwargs):
 271         self.start_time = time.time()
 272         try:
 273             self.configure(**kwargs)
 274             self.generate_test_list()
 275             self.apply_tempest_blacklist()
 276             self.run_verifier_tests()
 277             self.parse_verifier_result()
 278             self.generate_report()
 279             res = testcase.TestCase.EX_OK
 280         except Exception:  # pylint: disable=broad-except
 281             LOGGER.exception('Error with run')
 282             res = testcase.TestCase.EX_RUN_ERROR
 283         finally:
 284             self.resources.cleanup()
 285         self.stop_time = time.time()
 286         return res
 287
 288
 289 class TempestSmokeSerial(TempestCommon):
 290     """Tempest smoke serial testcase implementation."""
 291     def __init__(self, **kwargs):
 292         if "case_name" not in kwargs:
 293             kwargs["case_name"] = 'tempest_smoke_serial'
 294         TempestCommon.__init__(self, **kwargs)
 295         self.mode = "smoke"
 296         self.option = ["--concurrency", "1"]
 297
 298
 299 class TempestNeutronTrunk(TempestCommon):
 300     """Tempest neutron trunk testcase implementation."""
 301     def __init__(self, **kwargs):
 302         if "case_name" not in kwargs:
 303             kwargs["case_name"] = 'neutron_trunk'
 304         TempestCommon.__init__(self, **kwargs)
 305         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 306         self.res_dir = os.path.join(
 307             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 308         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 309         self.list = os.path.join(self.res_dir, 'test_list.txt')
 310
 311     def configure(self, **kwargs):
 312         super(TempestNeutronTrunk, self).configure(**kwargs)
 313         rconfig = conf_utils.ConfigParser.RawConfigParser()
 314         rconfig.read(self.conf_file)
 315         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 316         with open(self.conf_file, 'wb') as config_file:
 317             rconfig.write(config_file)
 318
 319
 320 class TempestSmokeParallel(TempestCommon):
 321     """Tempest smoke parallel testcase implementation."""
 322     def __init__(self, **kwargs):
 323         if "case_name" not in kwargs:
 324             kwargs["case_name"] = 'tempest_smoke_parallel'
 325         TempestCommon.__init__(self, **kwargs)
 326         self.mode = "smoke"
 327
 328
 329 class TempestFullParallel(TempestCommon):
 330     """Tempest full parallel testcase implementation."""
 331     def __init__(self, **kwargs):
 332         if "case_name" not in kwargs:
 333             kwargs["case_name"] = 'tempest_full_parallel'
 334         TempestCommon.__init__(self, **kwargs)
 335         self.mode = "full"
 336
 337
 338 class TempestCustom(TempestCommon):
 339     """Tempest custom testcase implementation."""
 340     def __init__(self, **kwargs):
 341         if "case_name" not in kwargs:
 342             kwargs["case_name"] = 'tempest_custom'
 343         TempestCommon.__init__(self, **kwargs)
 344         self.mode = "custom"
 345         self.option = ["--concurrency", "1"]
 346
 347
 348 class TempestDefcore(TempestCommon):
 349     """Tempest Defcore testcase implementation."""
 350     def __init__(self, **kwargs):
 351         if "case_name" not in kwargs:
 352             kwargs["case_name"] = 'tempest_defcore'
 353         TempestCommon.__init__(self, **kwargs)
 354         self.mode = "defcore"
 355         self.option = ["--concurrency", "1"]
 356
 357
 358 class TempestResourcesManager(object):
 359     # pylint: disable=too-many-instance-attributes
 360     """Tempest resource manager."""
 361     def __init__(self):
 362         self.guid = '-' + str(uuid.uuid4())
 363         self.cloud = os_client_config.make_shade()
 364         self.domain_id = self.cloud.auth["project_domain_id"]
 365         self.project = None
 366         self.user = None
 367         self.network = None
 368         self.subnet = None
 369         self.image = None
 370         self.image_alt = None
 371         self.flavor = None
 372         self.flavor_alt = None
 373
 374     def _create_project(self):
 375         """Create project for tests."""
 376         self.project = self.cloud.create_project(
 377             getattr(config.CONF, 'tempest_identity_tenant_name') + self.guid,
 378             description=getattr(
 379                 config.CONF, 'tempest_identity_tenant_description'),
 380             domain_id=self.domain_id)
 381         LOGGER.debug("project: %s", self.project)
 382
 383     def _create_user(self):
 384         """Create user for tests."""
 385         self.user = self.cloud.create_user(
 386             name=getattr(
 387                 config.CONF, 'tempest_identity_user_name') + self.guid,
 388             password=getattr(config.CONF, 'tempest_identity_user_password'),
 389             default_project=getattr(
 390                 config.CONF, 'tempest_identity_tenant_name') + self.guid,
 391             domain_id=self.domain_id)
 392         LOGGER.debug("user: %s", self.user)
 393
 394     def _create_network(self):
 395         """Create network for tests."""
 396         tempest_net_name = getattr(
 397             config.CONF, 'tempest_private_net_name') + self.guid
 398         provider = {}
 399         if hasattr(config.CONF, 'tempest_network_type'):
 400             provider["network_type"] = getattr(
 401                 config.CONF, 'tempest_network_type')
 402         if hasattr(config.CONF, 'tempest_physical_network'):
 403             provider["physical_network"] = getattr(
 404                 config.CONF, 'tempest_physical_network')
 405         if hasattr(config.CONF, 'tempest_segmentation_id'):
 406             provider["segmentation_id"] = getattr(
 407                 config.CONF, 'tempest_segmentation_id')
 408         LOGGER.info(
 409             "Creating network with name: '%s'", tempest_net_name)
 410         self.network = self.cloud.create_network(
 411             tempest_net_name, provider=provider)
 412         LOGGER.debug("network: %s", self.network)
 413
 414         self.subnet = self.cloud.create_subnet(
 415             self.network.id,
 416             subnet_name=getattr(
 417                 config.CONF, 'tempest_private_subnet_name') + self.guid,
 418             cidr=getattr(config.CONF, 'tempest_private_subnet_cidr'),
 419             enable_dhcp=True,
 420             dns_nameservers=[env.get('NAMESERVER')])
 421         LOGGER.debug("subnet: %s", self.subnet)
 422
 423     def _create_image(self, name):
 424         """Create image for tests"""
 425         LOGGER.info("Creating image with name: '%s'", name)
 426         image = self.cloud.create_image(
 427             name, filename=getattr(config.CONF, 'openstack_image_url'),
 428             is_public=True)
 429         LOGGER.debug("image: %s", image)
 430         return image
 431
 432     def _create_flavor(self, name):
 433         """Create flavor for tests."""
 434         flavor = self.cloud.create_flavor(
 435             name, getattr(config.CONF, 'openstack_flavor_ram'),
 436             getattr(config.CONF, 'openstack_flavor_vcpus'),
 437             getattr(config.CONF, 'openstack_flavor_disk'))
 438         self.cloud.set_flavor_specs(
 439             flavor.id, getattr(config.CONF, 'flavor_extra_specs', {}))
 440         LOGGER.debug("flavor: %s", flavor)
 441         return flavor
 442
 443     def create(self, create_project=False):
 444         """Create resources for Tempest test suite."""
 445         if create_project:
 446             self._create_project()
 447             self._create_user()
 448         self._create_network()
 449
 450         LOGGER.debug("Creating two images for Tempest suite")
 451         self.image = self._create_image(
 452             getattr(config.CONF, 'openstack_image_name') + self.guid)
 453         self.image_alt = self._create_image(
 454             getattr(config.CONF, 'openstack_image_name_alt') + self.guid)
 455
 456         LOGGER.info("Creating two flavors for Tempest suite")
 457         self.flavor = self._create_flavor(
 458             getattr(config.CONF, 'openstack_flavor_name') + self.guid)
 459         self.flavor_alt = self._create_flavor(
 460             getattr(config.CONF, 'openstack_flavor_name_alt') + self.guid)
 461
 462     def cleanup(self):
 463         """
 464         Cleanup all OpenStack objects. Should be called on completion.
 465         """
 466         self.cloud.delete_image(self.image)
 467         self.cloud.delete_image(self.image_alt)
 468         self.cloud.delete_network(self.network.id)
 469         self.cloud.delete_flavor(self.flavor.id)
 470         self.cloud.delete_flavor(self.flavor_alt.id)
 471         if self.project:
 472             self.cloud.delete_user(self.user.id)
 473             self.cloud.delete_project(self.project.id)