functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import yaml
  24
  25 from functest.core import testcase
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils.constants import CONST
  29 import functest.utils.functest_utils as ft_utils
  30
  31 from snaps.config.flavor import FlavorConfig
  32 from snaps.config.network import NetworkConfig, SubnetConfig
  33 from snaps.config.project import ProjectConfig
  34 from snaps.config.user import UserConfig
  35
  36 from snaps.openstack import create_flavor
  37 from snaps.openstack.create_flavor import OpenStackFlavor
  38 from snaps.openstack.tests import openstack_tests
  39 from snaps.openstack.utils import deploy_utils
  40
  41
  42 LOGGER = logging.getLogger(__name__)
  43
  44
  45 class TempestCommon(testcase.TestCase):
  46     # pylint: disable=too-many-instance-attributes
  47     """TempestCommon testcases implementation class."""
  48
  49     def __init__(self, **kwargs):
  50         super(TempestCommon, self).__init__(**kwargs)
  51         self.resources = TempestResourcesManager(**kwargs)
  52         self.mode = ""
  53         self.option = []
  54         self.verifier_id = conf_utils.get_verifier_id()
  55         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  56             self.verifier_id)
  57         self.deployment_id = conf_utils.get_verifier_deployment_id()
  58         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  59             self.verifier_id, self.deployment_id)
  60         self.verification_id = None
  61
  62     @staticmethod
  63     def read_file(filename):
  64         """Read file and return content as a stripped list."""
  65         with open(filename) as src:
  66             return [line.strip() for line in src.readlines()]
  67
  68     @staticmethod
  69     def get_verifier_result(verif_id):
  70         """Retrieve verification results."""
  71         result = {
  72             'num_tests': 0,
  73             'num_success': 0,
  74             'num_failures': 0,
  75             'num_skipped': 0
  76         }
  77         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  78         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  79         proc = subprocess.Popen(cmd,
  80                                 stdout=subprocess.PIPE,
  81                                 stderr=subprocess.STDOUT)
  82         for line in proc.stdout:
  83             new_line = line.replace(' ', '').split('|')
  84             if 'Tests' in new_line:
  85                 break
  86             LOGGER.info(line)
  87             if 'Testscount' in new_line:
  88                 result['num_tests'] = int(new_line[2])
  89             elif 'Success' in new_line:
  90                 result['num_success'] = int(new_line[2])
  91             elif 'Skipped' in new_line:
  92                 result['num_skipped'] = int(new_line[2])
  93             elif 'Failures' in new_line:
  94                 result['num_failures'] = int(new_line[2])
  95         return result
  96
  97     def generate_test_list(self, verifier_repo_dir):
  98         """Generate test list based on the test mode."""
  99         LOGGER.debug("Generating test case list...")
 100         if self.mode == 'defcore':
 101             shutil.copyfile(
 102                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
 103         elif self.mode == 'custom':
 104             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 105                 shutil.copyfile(
 106                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 107             else:
 108                 raise Exception("Tempest test list file %s NOT found."
 109                                 % conf_utils.TEMPEST_CUSTOM)
 110         else:
 111             if self.mode == 'smoke':
 112                 testr_mode = "smoke"
 113             elif self.mode == 'full':
 114                 testr_mode = ""
 115             else:
 116                 testr_mode = 'tempest.api.' + self.mode
 117             cmd = ("cd {0};"
 118                    "testr list-tests {1} > {2};"
 119                    "cd -;".format(verifier_repo_dir,
 120                                   testr_mode,
 121                                   conf_utils.TEMPEST_RAW_LIST))
 122             ft_utils.execute_command(cmd)
 123
 124     def apply_tempest_blacklist(self):
 125         """Exclude blacklisted test cases."""
 126         LOGGER.debug("Applying tempest blacklist...")
 127         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 128         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 129         black_tests = []
 130         try:
 131             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
 132             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 133             if bool(installer_type) * bool(deploy_scenario):
 134                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 135                 # file
 136                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 137                 black_list_yaml = yaml.safe_load(black_list_file)
 138                 black_list_file.close()
 139                 for item in black_list_yaml:
 140                     scenarios = item['scenarios']
 141                     installers = item['installers']
 142                     if (deploy_scenario in scenarios and
 143                             installer_type in installers):
 144                         tests = item['tests']
 145                         for test in tests:
 146                             black_tests.append(test)
 147                         break
 148         except Exception:  # pylint: disable=broad-except
 149             black_tests = []
 150             LOGGER.debug("Tempest blacklist file does not exist.")
 151
 152         for cases_line in cases_file:
 153             for black_tests_line in black_tests:
 154                 if black_tests_line in cases_line:
 155                     break
 156             else:
 157                 result_file.write(str(cases_line) + '\n')
 158         result_file.close()
 159
 160     def run_verifier_tests(self):
 161         """Execute tempest test cases."""
 162         cmd = ["rally", "verify", "start", "--load-list",
 163                conf_utils.TEMPEST_LIST]
 164         cmd.extend(self.option)
 165         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 166
 167         header = ("Tempest environment:\n"
 168                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 169                   (CONST.__getattribute__('INSTALLER_TYPE'),
 170                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 171                    CONST.__getattribute__('NODE_NAME'),
 172                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 173
 174         f_stdout = open(
 175             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 176         f_stderr = open(
 177             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 178                          "tempest-error.log"), 'w+')
 179         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 180                                   "environment.log"), 'w+')
 181         f_env.write(header)
 182
 183         proc = subprocess.Popen(
 184             cmd,
 185             stdout=subprocess.PIPE,
 186             stderr=f_stderr,
 187             bufsize=1)
 188
 189         with proc.stdout:
 190             for line in iter(proc.stdout.readline, b''):
 191                 if re.search(r"\} tempest\.", line):
 192                     LOGGER.info(line.replace('\n', ''))
 193                 elif re.search('Starting verification', line):
 194                     LOGGER.info(line.replace('\n', ''))
 195                     first_pos = line.index("UUID=") + len("UUID=")
 196                     last_pos = line.index(") for deployment")
 197                     self.verification_id = line[first_pos:last_pos]
 198                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 199                 f_stdout.write(line)
 200         proc.wait()
 201
 202         f_stdout.close()
 203         f_stderr.close()
 204         f_env.close()
 205
 206     def parse_verifier_result(self):
 207         """Parse and save test results."""
 208         if self.verification_id is None:
 209             raise Exception('Verification UUID not found')
 210
 211         stat = self.get_verifier_result(self.verification_id)
 212         try:
 213             num_executed = stat['num_tests'] - stat['num_skipped']
 214             try:
 215                 self.result = 100 * stat['num_success'] / num_executed
 216             except ZeroDivisionError:
 217                 self.result = 0
 218                 if stat['num_tests'] > 0:
 219                     LOGGER.info("All tests have been skipped")
 220                 else:
 221                     LOGGER.error("No test has been executed")
 222                     return
 223
 224             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 225                                    "tempest.log"), 'r') as logfile:
 226                 output = logfile.read()
 227
 228             success_testcases = []
 229             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 230                 success_testcases.append(match)
 231             failed_testcases = []
 232             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 233                 failed_testcases.append(match)
 234             skipped_testcases = []
 235             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 236                 skipped_testcases.append(match)
 237
 238             self.details = {"tests": stat['num_tests'],
 239                             "failures": stat['num_failures'],
 240                             "success": success_testcases,
 241                             "skipped": skipped_testcases,
 242                             "errors": failed_testcases}
 243         except Exception:  # pylint: disable=broad-except
 244             self.result = 0
 245
 246         LOGGER.info("Tempest %s success_rate is %s%%",
 247                     self.case_name, self.result)
 248
 249     def run(self):
 250
 251         self.start_time = time.time()
 252         try:
 253             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 254                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 255             resources = self.resources.create()
 256             compute_cnt = snaps_utils.get_active_compute_cnt(
 257                 self.resources.os_creds)
 258             conf_utils.configure_tempest(
 259                 self.deployment_dir,
 260                 network_name=resources.get("network_name"),
 261                 image_id=resources.get("image_id"),
 262                 flavor_id=resources.get("flavor_id"),
 263                 compute_cnt=compute_cnt)
 264             self.generate_test_list(self.verifier_repo_dir)
 265             self.apply_tempest_blacklist()
 266             self.run_verifier_tests()
 267             self.parse_verifier_result()
 268             res = testcase.TestCase.EX_OK
 269         except Exception as err:  # pylint: disable=broad-except
 270             LOGGER.error('Error with run: %s', err)
 271             res = testcase.TestCase.EX_RUN_ERROR
 272         finally:
 273             self.resources.cleanup()
 274
 275         self.stop_time = time.time()
 276         return res
 277
 278
 279 class TempestSmokeSerial(TempestCommon):
 280     """Tempest smoke serial testcase implementation."""
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_smoke_serial'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.mode = "smoke"
 286         self.option = ["--concurrency", "1"]
 287
 288
 289 class TempestSmokeParallel(TempestCommon):
 290     """Tempest smoke parallel testcase implementation."""
 291     def __init__(self, **kwargs):
 292         if "case_name" not in kwargs:
 293             kwargs["case_name"] = 'tempest_smoke_parallel'
 294         TempestCommon.__init__(self, **kwargs)
 295         self.mode = "smoke"
 296
 297
 298 class TempestFullParallel(TempestCommon):
 299     """Tempest full parallel testcase implementation."""
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'tempest_full_parallel'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.mode = "full"
 305
 306
 307 class TempestCustom(TempestCommon):
 308     """Tempest custom testcase implementation."""
 309     def __init__(self, **kwargs):
 310         if "case_name" not in kwargs:
 311             kwargs["case_name"] = 'tempest_custom'
 312         TempestCommon.__init__(self, **kwargs)
 313         self.mode = "custom"
 314         self.option = ["--concurrency", "1"]
 315
 316
 317 class TempestDefcore(TempestCommon):
 318     """Tempest Defcore testcase implementation."""
 319     def __init__(self, **kwargs):
 320         if "case_name" not in kwargs:
 321             kwargs["case_name"] = 'tempest_defcore'
 322         TempestCommon.__init__(self, **kwargs)
 323         self.mode = "defcore"
 324         self.option = ["--concurrency", "1"]
 325
 326
 327 class TempestResourcesManager(object):
 328     """Tempest resource manager."""
 329     def __init__(self, **kwargs):
 330         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 331         self.guid = '-' + str(uuid.uuid4())
 332         self.creators = list()
 333
 334         if hasattr(CONST, 'snaps_images_cirros'):
 335             self.cirros_image_config = CONST.__getattribute__(
 336                 'snaps_images_cirros')
 337         else:
 338             self.cirros_image_config = None
 339
 340     def _create_project(self):
 341         """Create project for tests."""
 342         project_creator = deploy_utils.create_project(
 343             self.os_creds, ProjectConfig(
 344                 name=CONST.__getattribute__(
 345                     'tempest_identity_tenant_name') + self.guid,
 346                 description=CONST.__getattribute__(
 347                     'tempest_identity_tenant_description')))
 348         if project_creator is None or project_creator.get_project() is None:
 349             raise Exception("Failed to create tenant")
 350         self.creators.append(project_creator)
 351         return project_creator.get_project().id
 352
 353     def _create_user(self):
 354         """Create user for tests."""
 355         user_creator = deploy_utils.create_user(
 356             self.os_creds, UserConfig(
 357                 name=CONST.__getattribute__(
 358                     'tempest_identity_user_name') + self.guid,
 359                 password=CONST.__getattribute__(
 360                     'tempest_identity_user_password'),
 361                 project_name=CONST.__getattribute__(
 362                     'tempest_identity_tenant_name') + self.guid))
 363         if user_creator is None or user_creator.get_user() is None:
 364             raise Exception("Failed to create user")
 365         self.creators.append(user_creator)
 366         return user_creator.get_user().id
 367
 368     def _create_network(self, project_name):
 369         """Create network for tests."""
 370         tempest_network_type = None
 371         tempest_physical_network = None
 372         tempest_segmentation_id = None
 373
 374         if hasattr(CONST, 'tempest_network_type'):
 375             tempest_network_type = CONST.__getattribute__(
 376                 'tempest_network_type')
 377         if hasattr(CONST, 'tempest_physical_network'):
 378             tempest_physical_network = CONST.__getattribute__(
 379                 'tempest_physical_network')
 380         if hasattr(CONST, 'tempest_segmentation_id'):
 381             tempest_segmentation_id = CONST.__getattribute__(
 382                 'tempest_segmentation_id')
 383
 384         tempest_net_name = CONST.__getattribute__(
 385             'tempest_private_net_name') + self.guid
 386
 387         network_creator = deploy_utils.create_network(
 388             self.os_creds, NetworkConfig(
 389                 name=tempest_net_name,
 390                 project_name=project_name,
 391                 network_type=tempest_network_type,
 392                 physical_network=tempest_physical_network,
 393                 segmentation_id=tempest_segmentation_id,
 394                 subnet_settings=[SubnetConfig(
 395                     name=CONST.__getattribute__(
 396                         'tempest_private_subnet_name') + self.guid,
 397                     project_name=project_name,
 398                     cidr=CONST.__getattribute__(
 399                         'tempest_private_subnet_cidr'))]))
 400         if network_creator is None or network_creator.get_network() is None:
 401             raise Exception("Failed to create private network")
 402         self.creators.append(network_creator)
 403         return tempest_net_name
 404
 405     def _create_image(self, name):
 406         """Create image for tests"""
 407         os_image_settings = openstack_tests.cirros_image_settings(
 408             name, public=True,
 409             image_metadata=self.cirros_image_config)
 410         image_creator = deploy_utils.create_image(
 411             self.os_creds, os_image_settings)
 412         if image_creator is None:
 413             raise Exception('Failed to create image')
 414         self.creators.append(image_creator)
 415         return image_creator.get_image().id
 416
 417     def _create_flavor(self, name):
 418         """Create flavor for tests."""
 419         scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 420         flavor_metadata = None
 421         if 'ovs' in scenario or 'fdio' in scenario:
 422             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 423         flavor_creator = OpenStackFlavor(
 424             self.os_creds, FlavorConfig(
 425                 name=name,
 426                 ram=CONST.__getattribute__('openstack_flavor_ram'),
 427                 disk=CONST.__getattribute__('openstack_flavor_disk'),
 428                 vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 429                 metadata=flavor_metadata))
 430         flavor = flavor_creator.create()
 431         if flavor is None:
 432             raise Exception('Failed to create flavor')
 433         self.creators.append(flavor_creator)
 434         return flavor.id
 435
 436     def create(self, use_custom_images=False, use_custom_flavors=False,
 437                create_project=False):
 438         """Create resources for Tempest test suite."""
 439         result = {
 440             'tempest_net_name': None,
 441             'image_id': None,
 442             'image_id_alt': None,
 443             'flavor_id': None,
 444             'flavor_id_alt': None
 445         }
 446         project_name = None
 447
 448         if create_project:
 449             LOGGER.debug("Creating project and user for Tempest suite")
 450             project_name = CONST.__getattribute__(
 451                 'tempest_identity_tenant_name') + self.guid
 452             result['project_id'] = self._create_project()
 453             result['user_id'] = self._create_user()
 454             result['tenant_id'] = result['project_id']  # for compatibility
 455
 456         LOGGER.debug("Creating private network for Tempest suite")
 457         result['tempest_net_name'] = self._create_network(project_name)
 458
 459         LOGGER.debug("Creating image for Tempest suite")
 460         image_name = CONST.__getattribute__('openstack_image_name') + self.guid
 461         result['image_id'] = self._create_image(image_name)
 462
 463         if use_custom_images:
 464             LOGGER.debug("Creating 2nd image for Tempest suite")
 465             image_name = CONST.__getattribute__(
 466                 'openstack_image_name_alt') + self.guid
 467             result['image_id_alt'] = self._create_image(image_name)
 468
 469         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 470                 use_custom_flavors):
 471             LOGGER.info("Creating flavor for Tempest suite")
 472             name = CONST.__getattribute__('openstack_flavor_name') + self.guid
 473             result['flavor_id'] = self._create_flavor(name)
 474
 475         if use_custom_flavors:
 476             LOGGER.info("Creating 2nd flavor for Tempest suite")
 477             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 478             if 'ovs' in scenario or 'fdio' in scenario:
 479                 CONST.__setattr__('openstack_flavor_ram', 1024)
 480             name = CONST.__getattribute__(
 481                 'openstack_flavor_name_alt') + self.guid
 482             result['flavor_id_alt'] = self._create_flavor(name)
 483
 484         return result
 485
 486     def cleanup(self):
 487         """
 488         Cleanup all OpenStack objects. Should be called on completion.
 489         """
 490         for creator in reversed(self.creators):
 491             try:
 492                 creator.clean()
 493             except Exception as err:  # pylint: disable=broad-except
 494                 LOGGER.error('Unexpected error cleaning - %s', err)