functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19 import uuid
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.opnfv_tests.openstack.tempest import conf_utils
  26 from functest.utils.constants import CONST
  27 import functest.utils.functest_utils as ft_utils
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.network import NetworkConfig, SubnetConfig
  31 from snaps.config.project import ProjectConfig
  32 from snaps.config.user import UserConfig
  33
  34 from snaps.openstack import create_flavor
  35 from snaps.openstack.create_flavor import OpenStackFlavor
  36 from snaps.openstack.tests import openstack_tests
  37 from snaps.openstack.utils import deploy_utils
  38
  39
  40 """ logging configuration """
  41 logger = logging.getLogger(__name__)
  42
  43
  44 class TempestCommon(testcase.TestCase):
  45
  46     def __init__(self, **kwargs):
  47         super(TempestCommon, self).__init__(**kwargs)
  48         self.resources = TempestResourcesManager(**kwargs)
  49         self.MODE = ""
  50         self.OPTION = []
  51         self.VERIFIER_ID = conf_utils.get_verifier_id()
  52         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  53             self.VERIFIER_ID)
  54         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  55         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  56             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  57         self.VERIFICATION_ID = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     def generate_test_list(self, verifier_repo_dir):
  65         logger.debug("Generating test case list...")
  66         if self.MODE == 'defcore':
  67             shutil.copyfile(
  68                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  69         elif self.MODE == 'custom':
  70             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  71                 shutil.copyfile(
  72                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  73             else:
  74                 raise Exception("Tempest test list file %s NOT found."
  75                                 % conf_utils.TEMPEST_CUSTOM)
  76         else:
  77             if self.MODE == 'smoke':
  78                 testr_mode = "smoke"
  79             elif self.MODE == 'full':
  80                 testr_mode = ""
  81             else:
  82                 testr_mode = 'tempest.api.' + self.MODE
  83             cmd = ("cd {0};"
  84                    "testr list-tests {1} > {2};"
  85                    "cd -;".format(verifier_repo_dir,
  86                                   testr_mode,
  87                                   conf_utils.TEMPEST_RAW_LIST))
  88             ft_utils.execute_command(cmd)
  89
  90     def apply_tempest_blacklist(self):
  91         logger.debug("Applying tempest blacklist...")
  92         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  93         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  94         black_tests = []
  95         try:
  96             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  97             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  98             if (bool(installer_type) * bool(deploy_scenario)):
  99                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 100                 # file
 101                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 102                 black_list_yaml = yaml.safe_load(black_list_file)
 103                 black_list_file.close()
 104                 for item in black_list_yaml:
 105                     scenarios = item['scenarios']
 106                     installers = item['installers']
 107                     if (deploy_scenario in scenarios and
 108                             installer_type in installers):
 109                         tests = item['tests']
 110                         for test in tests:
 111                             black_tests.append(test)
 112                         break
 113         except Exception:
 114             black_tests = []
 115             logger.debug("Tempest blacklist file does not exist.")
 116
 117         for cases_line in cases_file:
 118             for black_tests_line in black_tests:
 119                 if black_tests_line in cases_line:
 120                     break
 121             else:
 122                 result_file.write(str(cases_line) + '\n')
 123         result_file.close()
 124
 125     def run_verifier_tests(self):
 126         cmd = ["rally", "verify", "start", "--load-list",
 127                conf_utils.TEMPEST_LIST]
 128         cmd.extend(self.OPTION)
 129         logger.info("Starting Tempest test suite: '%s'." % cmd)
 130
 131         header = ("Tempest environment:\n"
 132                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 133                   (CONST.__getattribute__('INSTALLER_TYPE'),
 134                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 135                    CONST.__getattribute__('NODE_NAME'),
 136                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 137
 138         f_stdout = open(
 139             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 140         f_stderr = open(
 141             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 142                          "tempest-error.log"), 'w+')
 143         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 144                                   "environment.log"), 'w+')
 145         f_env.write(header)
 146
 147         p = subprocess.Popen(
 148             cmd,
 149             stdout=subprocess.PIPE,
 150             stderr=f_stderr,
 151             bufsize=1)
 152
 153         with p.stdout:
 154             for line in iter(p.stdout.readline, b''):
 155                 if re.search("\} tempest\.", line):
 156                     logger.info(line.replace('\n', ''))
 157                 elif re.search('Starting verification', line):
 158                     logger.info(line.replace('\n', ''))
 159                     first_pos = line.index("UUID=") + len("UUID=")
 160                     last_pos = line.index(") for deployment")
 161                     self.VERIFICATION_ID = line[first_pos:last_pos]
 162                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 163                 f_stdout.write(line)
 164         p.wait()
 165
 166         f_stdout.close()
 167         f_stderr.close()
 168         f_env.close()
 169
 170     def parse_verifier_result(self):
 171         if self.VERIFICATION_ID is None:
 172             raise Exception('Verification UUID not found')
 173
 174         cmd = ["rally", "verify", "show", "--uuid", self.VERIFICATION_ID]
 175         logger.info("Showing result for a verification: '%s'." % cmd)
 176         p = subprocess.Popen(cmd,
 177                              stdout=subprocess.PIPE,
 178                              stderr=subprocess.STDOUT)
 179         for line in p.stdout:
 180             new_line = line.replace(' ', '').split('|')
 181             if 'Tests' in new_line:
 182                 break
 183
 184             logger.info(line)
 185             if 'Testscount' in new_line:
 186                 num_tests = new_line[2]
 187             elif 'Success' in new_line:
 188                 num_success = new_line[2]
 189             elif 'Skipped' in new_line:
 190                 num_skipped = new_line[2]
 191             elif 'Failures' in new_line:
 192                 num_failures = new_line[2]
 193
 194         try:
 195             num_executed = int(num_tests) - int(num_skipped)
 196             try:
 197                 self.result = 100 * int(num_success) / int(num_executed)
 198             except ZeroDivisionError:
 199                 self.result = 0
 200                 if int(num_tests) > 0:
 201                     logger.info("All tests have been skipped")
 202                 else:
 203                     logger.error("No test has been executed")
 204                     return
 205
 206             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 207                                    "tempest.log"), 'r') as logfile:
 208                 output = logfile.read()
 209
 210             success_testcases = []
 211             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 212                 success_testcases.append(match)
 213             failed_testcases = []
 214             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 215                 failed_testcases.append(match)
 216             skipped_testcases = []
 217             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 218                 skipped_testcases.append(match)
 219
 220             self.details = {"tests": int(num_tests),
 221                             "failures": int(num_failures),
 222                             "success": success_testcases,
 223                             "errors": failed_testcases,
 224                             "skipped": skipped_testcases}
 225         except Exception:
 226             self.result = 0
 227
 228         logger.info("Tempest %s success_rate is %s%%"
 229                     % (self.case_name, self.result))
 230
 231     def run(self):
 232
 233         self.start_time = time.time()
 234         try:
 235             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 236                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 237             resources = self.resources.create()
 238             compute_cnt = snaps_utils.get_active_compute_cnt(
 239                 self.resources.os_creds)
 240             conf_utils.configure_tempest(
 241                 self.DEPLOYMENT_DIR,
 242                 image_id=resources.get("image_id"),
 243                 flavor_id=resources.get("flavor_id"),
 244                 compute_cnt=compute_cnt)
 245             self.generate_test_list(self.VERIFIER_REPO_DIR)
 246             self.apply_tempest_blacklist()
 247             self.run_verifier_tests()
 248             self.parse_verifier_result()
 249             res = testcase.TestCase.EX_OK
 250         except Exception as e:
 251             logger.error('Error with run: %s' % e)
 252             res = testcase.TestCase.EX_RUN_ERROR
 253         finally:
 254             self.resources.cleanup()
 255
 256         self.stop_time = time.time()
 257         return res
 258
 259
 260 class TempestSmokeSerial(TempestCommon):
 261
 262     def __init__(self, **kwargs):
 263         if "case_name" not in kwargs:
 264             kwargs["case_name"] = 'tempest_smoke_serial'
 265         TempestCommon.__init__(self, **kwargs)
 266         self.MODE = "smoke"
 267         self.OPTION = ["--concurrency", "1"]
 268
 269
 270 class TempestSmokeParallel(TempestCommon):
 271
 272     def __init__(self, **kwargs):
 273         if "case_name" not in kwargs:
 274             kwargs["case_name"] = 'tempest_smoke_parallel'
 275         TempestCommon.__init__(self, **kwargs)
 276         self.MODE = "smoke"
 277
 278
 279 class TempestFullParallel(TempestCommon):
 280
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_full_parallel'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.MODE = "full"
 286
 287
 288 class TempestCustom(TempestCommon):
 289
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_custom'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.MODE = "custom"
 295         self.OPTION = ["--concurrency", "1"]
 296
 297
 298 class TempestDefcore(TempestCommon):
 299
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'tempest_defcore'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.MODE = "defcore"
 305         self.OPTION = ["--concurrency", "1"]
 306
 307
 308 class TempestResourcesManager(object):
 309
 310     def __init__(self, **kwargs):
 311         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 312         self.guid = '-' + str(uuid.uuid4())
 313         self.creators = list()
 314
 315         if hasattr(CONST, 'snaps_images_cirros'):
 316             self.cirros_image_config = CONST.__getattribute__(
 317                 'snaps_images_cirros')
 318         else:
 319             self.cirros_image_config = None
 320
 321     def _create_project(self):
 322         """Create project for tests."""
 323         project_creator = deploy_utils.create_project(
 324             self.os_creds, ProjectConfig(
 325                 name=CONST.__getattribute__(
 326                     'tempest_identity_tenant_name') + self.guid,
 327                 description=CONST.__getattribute__(
 328                     'tempest_identity_tenant_description')))
 329         if project_creator is None or project_creator.get_project() is None:
 330             raise Exception("Failed to create tenant")
 331         self.creators.append(project_creator)
 332         return project_creator.get_project().id
 333
 334     def _create_user(self):
 335         """Create user for tests."""
 336         user_creator = deploy_utils.create_user(
 337             self.os_creds, UserConfig(
 338                 name=CONST.__getattribute__(
 339                     'tempest_identity_user_name') + self.guid,
 340                 password=CONST.__getattribute__(
 341                     'tempest_identity_user_password'),
 342                 project_name=CONST.__getattribute__(
 343                     'tempest_identity_tenant_name') + self.guid))
 344         if user_creator is None or user_creator.get_user() is None:
 345             raise Exception("Failed to create user")
 346         self.creators.append(user_creator)
 347         return user_creator.get_user().id
 348
 349     def _create_network(self, project_name):
 350         """Create network for tests."""
 351         tempest_network_type = None
 352         tempest_physical_network = None
 353         tempest_segmentation_id = None
 354
 355         if hasattr(CONST, 'tempest_network_type'):
 356             tempest_network_type = CONST.__getattribute__(
 357                 'tempest_network_type')
 358         if hasattr(CONST, 'tempest_physical_network'):
 359             tempest_physical_network = CONST.__getattribute__(
 360                 'tempest_physical_network')
 361         if hasattr(CONST, 'tempest_segmentation_id'):
 362             tempest_segmentation_id = CONST.__getattribute__(
 363                 'tempest_segmentation_id')
 364
 365         network_creator = deploy_utils.create_network(
 366             self.os_creds, NetworkConfig(
 367                 name=CONST.__getattribute__(
 368                     'tempest_private_net_name') + self.guid,
 369                 project_name=project_name,
 370                 network_type=tempest_network_type,
 371                 physical_network=tempest_physical_network,
 372                 segmentation_id=tempest_segmentation_id,
 373                 subnet_settings=[SubnetConfig(
 374                     name=CONST.__getattribute__(
 375                         'tempest_private_subnet_name') + self.guid,
 376                     project_name=project_name,
 377                     cidr=CONST.__getattribute__(
 378                         'tempest_private_subnet_cidr'))]))
 379         if network_creator is None or network_creator.get_network() is None:
 380             raise Exception("Failed to create private network")
 381         self.creators.append(network_creator)
 382
 383     def _create_image(self, name):
 384         """Create image for tests"""
 385         os_image_settings = openstack_tests.cirros_image_settings(
 386             name, public=True,
 387             image_metadata=self.cirros_image_config)
 388         image_creator = deploy_utils.create_image(
 389             self.os_creds, os_image_settings)
 390         if image_creator is None:
 391             raise Exception('Failed to create image')
 392         self.creators.append(image_creator)
 393         return image_creator.get_image().id
 394
 395     def _create_flavor(self, name):
 396         """Create flavor for tests."""
 397         scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 398         flavor_metadata = None
 399         if 'ovs' in scenario or 'fdio' in scenario:
 400             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 401         flavor_creator = OpenStackFlavor(
 402             self.os_creds, FlavorConfig(
 403                 name=name,
 404                 ram=CONST.__getattribute__('openstack_flavor_ram'),
 405                 disk=CONST.__getattribute__('openstack_flavor_disk'),
 406                 vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 407                 metadata=flavor_metadata))
 408         flavor = flavor_creator.create()
 409         if flavor is None:
 410             raise Exception('Failed to create flavor')
 411         self.creators.append(flavor_creator)
 412         return flavor.id
 413
 414     def create(self, use_custom_images=False, use_custom_flavors=False,
 415                create_project=False):
 416         """Create resources for Tempest test suite."""
 417         result = {
 418             'image_id': None,
 419             'image_id_alt': None,
 420             'flavor_id': None,
 421             'flavor_id_alt': None
 422         }
 423         project_name = None
 424
 425         if create_project:
 426             logger.debug("Creating project and user for Tempest suite")
 427             project_name = CONST.__getattribute__(
 428                 'tempest_identity_tenant_name') + self.guid
 429             result['project_id'] = self._create_project()
 430             result['user_id'] = self._create_user()
 431             result['tenant_id'] = result['project_id']  # for compatibility
 432
 433         logger.debug("Creating private network for Tempest suite")
 434         self._create_network(project_name)
 435
 436         logger.debug("Creating image for Tempest suite")
 437         image_name = CONST.__getattribute__('openstack_image_name') + self.guid
 438         result['image_id'] = self._create_image(image_name)
 439
 440         if use_custom_images:
 441             logger.debug("Creating 2nd image for Tempest suite")
 442             image_name = CONST.__getattribute__(
 443                 'openstack_image_name_alt') + self.guid
 444             result['image_id_alt'] = self._create_image(image_name)
 445
 446         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 447                 use_custom_flavors):
 448             logger.info("Creating flavor for Tempest suite")
 449             name = CONST.__getattribute__('openstack_flavor_name') + self.guid
 450             result['flavor_id'] = self._create_flavor(name)
 451
 452         if use_custom_flavors:
 453             logger.info("Creating 2nd flavor for Tempest suite")
 454             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 455             if 'ovs' in scenario or 'fdio' in scenario:
 456                 CONST.__setattr__('openstack_flavor_ram', 1024)
 457             name = CONST.__getattribute__(
 458                 'openstack_flavor_name_alt') + self.guid
 459             result['flavor_id_alt'] = self._create_flavor(name)
 460
 461         return result
 462
 463     def cleanup(self):
 464         """
 465         Cleanup all OpenStack objects. Should be called on completion.
 466         """
 467         for creator in reversed(self.creators):
 468             try:
 469                 creator.clean()
 470             except Exception as e:
 471                 logger.error('Unexpected error cleaning - %s', e)