functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import yaml
  24
  25 from functest.core import testcase
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils.constants import CONST
  29 import functest.utils.functest_utils as ft_utils
  30
  31 from snaps.config.flavor import FlavorConfig
  32 from snaps.config.network import NetworkConfig, SubnetConfig
  33 from snaps.config.project import ProjectConfig
  34 from snaps.config.user import UserConfig
  35
  36 from snaps.openstack import create_flavor
  37 from snaps.openstack.create_flavor import OpenStackFlavor
  38 from snaps.openstack.tests import openstack_tests
  39 from snaps.openstack.utils import deploy_utils
  40
  41
  42 LOGGER = logging.getLogger(__name__)
  43
  44
  45 class TempestCommon(testcase.TestCase):
  46     # pylint: disable=too-many-instance-attributes
  47     """TempestCommon testcases implementation class."""
  48
  49     def __init__(self, **kwargs):
  50         super(TempestCommon, self).__init__(**kwargs)
  51         self.resources = TempestResourcesManager(**kwargs)
  52         self.mode = ""
  53         self.option = []
  54         self.verifier_id = conf_utils.get_verifier_id()
  55         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  56             self.verifier_id)
  57         self.deployment_id = conf_utils.get_verifier_deployment_id()
  58         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  59             self.verifier_id, self.deployment_id)
  60         self.verification_id = None
  61
  62     @staticmethod
  63     def read_file(filename):
  64         """Read file and return content as a stripped list."""
  65         with open(filename) as src:
  66             return [line.strip() for line in src.readlines()]
  67
  68     @staticmethod
  69     def get_verifier_result(verif_id):
  70         """Retrieve verification results."""
  71         result = {
  72             'num_tests': 0,
  73             'num_success': 0,
  74             'num_failures': 0,
  75             'num_skipped': 0
  76         }
  77         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  78         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  79         proc = subprocess.Popen(cmd,
  80                                 stdout=subprocess.PIPE,
  81                                 stderr=subprocess.STDOUT)
  82         for line in proc.stdout:
  83             new_line = line.replace(' ', '').split('|')
  84             if 'Tests' in new_line:
  85                 break
  86             LOGGER.info(line)
  87             if 'Testscount' in new_line:
  88                 result['num_tests'] = int(new_line[2])
  89             elif 'Success' in new_line:
  90                 result['num_success'] = int(new_line[2])
  91             elif 'Skipped' in new_line:
  92                 result['num_skipped'] = int(new_line[2])
  93             elif 'Failures' in new_line:
  94                 result['num_failures'] = int(new_line[2])
  95         return result
  96
  97     def generate_test_list(self, verifier_repo_dir):
  98         """Generate test list based on the test mode."""
  99         LOGGER.debug("Generating test case list...")
 100         if self.mode == 'defcore':
 101             shutil.copyfile(
 102                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
 103         elif self.mode == 'custom':
 104             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 105                 shutil.copyfile(
 106                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 107             else:
 108                 raise Exception("Tempest test list file %s NOT found."
 109                                 % conf_utils.TEMPEST_CUSTOM)
 110         else:
 111             if self.mode == 'smoke':
 112                 testr_mode = "smoke"
 113             elif self.mode == 'full':
 114                 testr_mode = ""
 115             else:
 116                 testr_mode = 'tempest.api.' + self.mode
 117             cmd = ("cd {0};"
 118                    "testr list-tests {1} > {2};"
 119                    "cd -;".format(verifier_repo_dir,
 120                                   testr_mode,
 121                                   conf_utils.TEMPEST_RAW_LIST))
 122             ft_utils.execute_command(cmd)
 123
 124     def apply_tempest_blacklist(self):
 125         """Exclude blacklisted test cases."""
 126         LOGGER.debug("Applying tempest blacklist...")
 127         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 128         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 129         black_tests = []
 130         try:
 131             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
 132             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 133             if bool(installer_type) * bool(deploy_scenario):
 134                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 135                 # file
 136                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 137                 black_list_yaml = yaml.safe_load(black_list_file)
 138                 black_list_file.close()
 139                 for item in black_list_yaml:
 140                     scenarios = item['scenarios']
 141                     installers = item['installers']
 142                     if (deploy_scenario in scenarios and
 143                             installer_type in installers):
 144                         tests = item['tests']
 145                         for test in tests:
 146                             black_tests.append(test)
 147                         break
 148         except Exception:  # pylint: disable=broad-except
 149             black_tests = []
 150             LOGGER.debug("Tempest blacklist file does not exist.")
 151
 152         for cases_line in cases_file:
 153             for black_tests_line in black_tests:
 154                 if black_tests_line in cases_line:
 155                     break
 156             else:
 157                 result_file.write(str(cases_line) + '\n')
 158         result_file.close()
 159
 160     def run_verifier_tests(self):
 161         """Execute tempest test cases."""
 162         cmd = ["rally", "verify", "start", "--load-list",
 163                conf_utils.TEMPEST_LIST]
 164         cmd.extend(self.option)
 165         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 166
 167         header = ("Tempest environment:\n"
 168                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 169                   (CONST.__getattribute__('INSTALLER_TYPE'),
 170                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 171                    CONST.__getattribute__('NODE_NAME'),
 172                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 173
 174         f_stdout = open(
 175             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 176         f_stderr = open(
 177             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 178                          "tempest-error.log"), 'w+')
 179         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 180                                   "environment.log"), 'w+')
 181         f_env.write(header)
 182
 183         proc = subprocess.Popen(
 184             cmd,
 185             stdout=subprocess.PIPE,
 186             stderr=f_stderr,
 187             bufsize=1)
 188
 189         with proc.stdout:
 190             for line in iter(proc.stdout.readline, b''):
 191                 if re.search(r"\} tempest\.", line):
 192                     LOGGER.info(line.replace('\n', ''))
 193                 elif re.search('Starting verification', line):
 194                     LOGGER.info(line.replace('\n', ''))
 195                     first_pos = line.index("UUID=") + len("UUID=")
 196                     last_pos = line.index(") for deployment")
 197                     self.verification_id = line[first_pos:last_pos]
 198                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 199                 f_stdout.write(line)
 200         proc.wait()
 201
 202         f_stdout.close()
 203         f_stderr.close()
 204         f_env.close()
 205
 206     def parse_verifier_result(self):
 207         """Parse and save test results."""
 208         if self.verification_id is None:
 209             raise Exception('Verification UUID not found')
 210
 211         stat = self.get_verifier_result(self.verification_id)
 212         try:
 213             num_executed = stat['num_tests'] - stat['num_skipped']
 214             try:
 215                 self.result = 100 * stat['num_success'] / num_executed
 216             except ZeroDivisionError:
 217                 self.result = 0
 218                 if stat['num_tests'] > 0:
 219                     LOGGER.info("All tests have been skipped")
 220                 else:
 221                     LOGGER.error("No test has been executed")
 222                     return
 223
 224             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 225                                    "tempest.log"), 'r') as logfile:
 226                 output = logfile.read()
 227
 228             success_testcases = []
 229             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 230                 success_testcases.append(match)
 231             failed_testcases = []
 232             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 233                 failed_testcases.append(match)
 234             skipped_testcases = []
 235             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 236                 skipped_testcases.append(match)
 237
 238             self.details = {"tests": stat['num_tests'],
 239                             "failures": stat['num_failures'],
 240                             "success": success_testcases,
 241                             "skipped": skipped_testcases,
 242                             "errors": failed_testcases}
 243         except Exception:  # pylint: disable=broad-except
 244             self.result = 0
 245
 246         LOGGER.info("Tempest %s success_rate is %s%%",
 247                     self.case_name, self.result)
 248
 249     def run(self):
 250
 251         self.start_time = time.time()
 252         try:
 253             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 254                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 255             resources = self.resources.create()
 256             compute_cnt = snaps_utils.get_active_compute_cnt(
 257                 self.resources.os_creds)
 258             conf_utils.configure_tempest(
 259                 self.deployment_dir,
 260                 image_id=resources.get("image_id"),
 261                 flavor_id=resources.get("flavor_id"),
 262                 compute_cnt=compute_cnt)
 263             self.generate_test_list(self.verifier_repo_dir)
 264             self.apply_tempest_blacklist()
 265             self.run_verifier_tests()
 266             self.parse_verifier_result()
 267             res = testcase.TestCase.EX_OK
 268         except Exception as err:  # pylint: disable=broad-except
 269             LOGGER.error('Error with run: %s', err)
 270             res = testcase.TestCase.EX_RUN_ERROR
 271         finally:
 272             self.resources.cleanup()
 273
 274         self.stop_time = time.time()
 275         return res
 276
 277
 278 class TempestSmokeSerial(TempestCommon):
 279     """Tempest smoke serial testcase implementation."""
 280     def __init__(self, **kwargs):
 281         if "case_name" not in kwargs:
 282             kwargs["case_name"] = 'tempest_smoke_serial'
 283         TempestCommon.__init__(self, **kwargs)
 284         self.mode = "smoke"
 285         self.option = ["--concurrency", "1"]
 286
 287
 288 class TempestSmokeParallel(TempestCommon):
 289     """Tempest smoke parallel testcase implementation."""
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_smoke_parallel'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.mode = "smoke"
 295
 296
 297 class TempestFullParallel(TempestCommon):
 298     """Tempest full parallel testcase implementation."""
 299     def __init__(self, **kwargs):
 300         if "case_name" not in kwargs:
 301             kwargs["case_name"] = 'tempest_full_parallel'
 302         TempestCommon.__init__(self, **kwargs)
 303         self.mode = "full"
 304
 305
 306 class TempestCustom(TempestCommon):
 307     """Tempest custom testcase implementation."""
 308     def __init__(self, **kwargs):
 309         if "case_name" not in kwargs:
 310             kwargs["case_name"] = 'tempest_custom'
 311         TempestCommon.__init__(self, **kwargs)
 312         self.mode = "custom"
 313         self.option = ["--concurrency", "1"]
 314
 315
 316 class TempestDefcore(TempestCommon):
 317     """Tempest Defcore testcase implementation."""
 318     def __init__(self, **kwargs):
 319         if "case_name" not in kwargs:
 320             kwargs["case_name"] = 'tempest_defcore'
 321         TempestCommon.__init__(self, **kwargs)
 322         self.mode = "defcore"
 323         self.option = ["--concurrency", "1"]
 324
 325
 326 class TempestResourcesManager(object):
 327     """Tempest resource manager."""
 328     def __init__(self, **kwargs):
 329         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 330         self.guid = '-' + str(uuid.uuid4())
 331         self.creators = list()
 332
 333         if hasattr(CONST, 'snaps_images_cirros'):
 334             self.cirros_image_config = CONST.__getattribute__(
 335                 'snaps_images_cirros')
 336         else:
 337             self.cirros_image_config = None
 338
 339     def _create_project(self):
 340         """Create project for tests."""
 341         project_creator = deploy_utils.create_project(
 342             self.os_creds, ProjectConfig(
 343                 name=CONST.__getattribute__(
 344                     'tempest_identity_tenant_name') + self.guid,
 345                 description=CONST.__getattribute__(
 346                     'tempest_identity_tenant_description')))
 347         if project_creator is None or project_creator.get_project() is None:
 348             raise Exception("Failed to create tenant")
 349         self.creators.append(project_creator)
 350         return project_creator.get_project().id
 351
 352     def _create_user(self):
 353         """Create user for tests."""
 354         user_creator = deploy_utils.create_user(
 355             self.os_creds, UserConfig(
 356                 name=CONST.__getattribute__(
 357                     'tempest_identity_user_name') + self.guid,
 358                 password=CONST.__getattribute__(
 359                     'tempest_identity_user_password'),
 360                 project_name=CONST.__getattribute__(
 361                     'tempest_identity_tenant_name') + self.guid))
 362         if user_creator is None or user_creator.get_user() is None:
 363             raise Exception("Failed to create user")
 364         self.creators.append(user_creator)
 365         return user_creator.get_user().id
 366
 367     def _create_network(self, project_name):
 368         """Create network for tests."""
 369         tempest_network_type = None
 370         tempest_physical_network = None
 371         tempest_segmentation_id = None
 372
 373         if hasattr(CONST, 'tempest_network_type'):
 374             tempest_network_type = CONST.__getattribute__(
 375                 'tempest_network_type')
 376         if hasattr(CONST, 'tempest_physical_network'):
 377             tempest_physical_network = CONST.__getattribute__(
 378                 'tempest_physical_network')
 379         if hasattr(CONST, 'tempest_segmentation_id'):
 380             tempest_segmentation_id = CONST.__getattribute__(
 381                 'tempest_segmentation_id')
 382
 383         network_creator = deploy_utils.create_network(
 384             self.os_creds, NetworkConfig(
 385                 name=CONST.__getattribute__(
 386                     'tempest_private_net_name') + self.guid,
 387                 project_name=project_name,
 388                 network_type=tempest_network_type,
 389                 physical_network=tempest_physical_network,
 390                 segmentation_id=tempest_segmentation_id,
 391                 subnet_settings=[SubnetConfig(
 392                     name=CONST.__getattribute__(
 393                         'tempest_private_subnet_name') + self.guid,
 394                     project_name=project_name,
 395                     cidr=CONST.__getattribute__(
 396                         'tempest_private_subnet_cidr'))]))
 397         if network_creator is None or network_creator.get_network() is None:
 398             raise Exception("Failed to create private network")
 399         self.creators.append(network_creator)
 400
 401     def _create_image(self, name):
 402         """Create image for tests"""
 403         os_image_settings = openstack_tests.cirros_image_settings(
 404             name, public=True,
 405             image_metadata=self.cirros_image_config)
 406         image_creator = deploy_utils.create_image(
 407             self.os_creds, os_image_settings)
 408         if image_creator is None:
 409             raise Exception('Failed to create image')
 410         self.creators.append(image_creator)
 411         return image_creator.get_image().id
 412
 413     def _create_flavor(self, name):
 414         """Create flavor for tests."""
 415         scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 416         flavor_metadata = None
 417         if 'ovs' in scenario or 'fdio' in scenario:
 418             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 419         flavor_creator = OpenStackFlavor(
 420             self.os_creds, FlavorConfig(
 421                 name=name,
 422                 ram=CONST.__getattribute__('openstack_flavor_ram'),
 423                 disk=CONST.__getattribute__('openstack_flavor_disk'),
 424                 vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 425                 metadata=flavor_metadata))
 426         flavor = flavor_creator.create()
 427         if flavor is None:
 428             raise Exception('Failed to create flavor')
 429         self.creators.append(flavor_creator)
 430         return flavor.id
 431
 432     def create(self, use_custom_images=False, use_custom_flavors=False,
 433                create_project=False):
 434         """Create resources for Tempest test suite."""
 435         result = {
 436             'image_id': None,
 437             'image_id_alt': None,
 438             'flavor_id': None,
 439             'flavor_id_alt': None
 440         }
 441         project_name = None
 442
 443         if create_project:
 444             LOGGER.debug("Creating project and user for Tempest suite")
 445             project_name = CONST.__getattribute__(
 446                 'tempest_identity_tenant_name') + self.guid
 447             result['project_id'] = self._create_project()
 448             result['user_id'] = self._create_user()
 449             result['tenant_id'] = result['project_id']  # for compatibility
 450
 451         LOGGER.debug("Creating private network for Tempest suite")
 452         self._create_network(project_name)
 453
 454         LOGGER.debug("Creating image for Tempest suite")
 455         image_name = CONST.__getattribute__('openstack_image_name') + self.guid
 456         result['image_id'] = self._create_image(image_name)
 457
 458         if use_custom_images:
 459             LOGGER.debug("Creating 2nd image for Tempest suite")
 460             image_name = CONST.__getattribute__(
 461                 'openstack_image_name_alt') + self.guid
 462             result['image_id_alt'] = self._create_image(image_name)
 463
 464         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 465                 use_custom_flavors):
 466             LOGGER.info("Creating flavor for Tempest suite")
 467             name = CONST.__getattribute__('openstack_flavor_name') + self.guid
 468             result['flavor_id'] = self._create_flavor(name)
 469
 470         if use_custom_flavors:
 471             LOGGER.info("Creating 2nd flavor for Tempest suite")
 472             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 473             if 'ovs' in scenario or 'fdio' in scenario:
 474                 CONST.__setattr__('openstack_flavor_ram', 1024)
 475             name = CONST.__getattribute__(
 476                 'openstack_flavor_name_alt') + self.guid
 477             result['flavor_id_alt'] = self._create_flavor(name)
 478
 479         return result
 480
 481     def cleanup(self):
 482         """
 483         Cleanup all OpenStack objects. Should be called on completion.
 484         """
 485         for creator in reversed(self.creators):
 486             try:
 487                 creator.clean()
 488             except Exception as err:  # pylint: disable=broad-except
 489                 LOGGER.error('Unexpected error cleaning - %s', err)