functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.snaps import snaps_utils
  24 from functest.opnfv_tests.openstack.tempest import conf_utils
  25 from functest.utils.constants import CONST
  26 import functest.utils.functest_utils as ft_utils
  27
  28 from snaps.openstack import create_flavor
  29 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  30 from snaps.openstack.create_project import ProjectSettings
  31 from snaps.openstack.create_network import NetworkSettings, SubnetSettings
  32 from snaps.openstack.create_user import UserSettings
  33 from snaps.openstack.tests import openstack_tests
  34 from snaps.openstack.utils import deploy_utils
  35
  36
  37 """ logging configuration """
  38 logger = logging.getLogger(__name__)
  39
  40
  41 class TempestCommon(testcase.TestCase):
  42
  43     def __init__(self, **kwargs):
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.resources = TempestResourcesManager(**kwargs)
  46         self.MODE = ""
  47         self.OPTION = ""
  48         self.VERIFIER_ID = conf_utils.get_verifier_id()
  49         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  50             self.VERIFIER_ID)
  51         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  52         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  53             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  54         self.VERIFICATION_ID = None
  55
  56     @staticmethod
  57     def read_file(filename):
  58         with open(filename) as src:
  59             return [line.strip() for line in src.readlines()]
  60
  61     def generate_test_list(self, verifier_repo_dir):
  62         logger.debug("Generating test case list...")
  63         if self.MODE == 'defcore':
  64             shutil.copyfile(
  65                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  66         elif self.MODE == 'custom':
  67             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  68                 shutil.copyfile(
  69                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  70             else:
  71                 raise Exception("Tempest test list file %s NOT found."
  72                                 % conf_utils.TEMPEST_CUSTOM)
  73         else:
  74             if self.MODE == 'smoke':
  75                 testr_mode = "smoke"
  76             elif self.MODE == 'full':
  77                 testr_mode = ""
  78             else:
  79                 testr_mode = 'tempest.api.' + self.MODE
  80             cmd = ("cd {0};"
  81                    "testr list-tests {1} > {2};"
  82                    "cd -;".format(verifier_repo_dir,
  83                                   testr_mode,
  84                                   conf_utils.TEMPEST_RAW_LIST))
  85             ft_utils.execute_command(cmd)
  86
  87     def apply_tempest_blacklist(self):
  88         logger.debug("Applying tempest blacklist...")
  89         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  90         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  91         black_tests = []
  92         try:
  93             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  94             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  95             if (bool(installer_type) * bool(deploy_scenario)):
  96                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  97                 # file
  98                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  99                 black_list_yaml = yaml.safe_load(black_list_file)
 100                 black_list_file.close()
 101                 for item in black_list_yaml:
 102                     scenarios = item['scenarios']
 103                     installers = item['installers']
 104                     if (deploy_scenario in scenarios and
 105                             installer_type in installers):
 106                         tests = item['tests']
 107                         for test in tests:
 108                             black_tests.append(test)
 109                         break
 110         except Exception:
 111             black_tests = []
 112             logger.debug("Tempest blacklist file does not exist.")
 113
 114         for cases_line in cases_file:
 115             for black_tests_line in black_tests:
 116                 if black_tests_line in cases_line:
 117                     break
 118             else:
 119                 result_file.write(str(cases_line) + '\n')
 120         result_file.close()
 121
 122     def run_verifier_tests(self):
 123         self.OPTION += (" --load-list {} --detailed"
 124                         .format(conf_utils.TEMPEST_LIST))
 125
 126         cmd_line = "rally verify start " + self.OPTION
 127         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 128
 129         header = ("Tempest environment:\n"
 130                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 131                   (CONST.__getattribute__('INSTALLER_TYPE'),
 132                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 133                    CONST.__getattribute__('NODE_NAME'),
 134                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 135
 136         f_stdout = open(
 137             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 138         f_stderr = open(
 139             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 140                          "tempest-error.log"), 'w+')
 141         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 142                                   "environment.log"), 'w+')
 143         f_env.write(header)
 144
 145         p = subprocess.Popen(
 146             cmd_line, shell=True,
 147             stdout=subprocess.PIPE,
 148             stderr=f_stderr,
 149             bufsize=1)
 150
 151         with p.stdout:
 152             for line in iter(p.stdout.readline, b''):
 153                 if re.search("\} tempest\.", line):
 154                     logger.info(line.replace('\n', ''))
 155                 elif re.search('Starting verification', line):
 156                     logger.info(line.replace('\n', ''))
 157                     first_pos = line.index("UUID=") + len("UUID=")
 158                     last_pos = line.index(") for deployment")
 159                     self.VERIFICATION_ID = line[first_pos:last_pos]
 160                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 161                 f_stdout.write(line)
 162         p.wait()
 163
 164         f_stdout.close()
 165         f_stderr.close()
 166         f_env.close()
 167
 168     def parse_verifier_result(self):
 169         if self.VERIFICATION_ID is None:
 170             raise Exception('Verification UUID not found')
 171
 172         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 173         logger.info("Showing result for a verification: '%s'." % cmd_line)
 174         p = subprocess.Popen(cmd_line,
 175                              shell=True,
 176                              stdout=subprocess.PIPE,
 177                              stderr=subprocess.STDOUT)
 178         for line in p.stdout:
 179             new_line = line.replace(' ', '').split('|')
 180             if 'Tests' in new_line:
 181                 break
 182
 183             logger.info(line)
 184             if 'Testscount' in new_line:
 185                 num_tests = new_line[2]
 186             elif 'Success' in new_line:
 187                 num_success = new_line[2]
 188             elif 'Skipped' in new_line:
 189                 num_skipped = new_line[2]
 190             elif 'Failures' in new_line:
 191                 num_failures = new_line[2]
 192
 193         try:
 194             num_executed = int(num_tests) - int(num_skipped)
 195             try:
 196                 self.result = 100 * int(num_success) / int(num_executed)
 197             except ZeroDivisionError:
 198                 self.result = 0
 199                 if int(num_tests) > 0:
 200                     logger.info("All tests have been skipped")
 201                 else:
 202                     logger.error("No test has been executed")
 203                     return
 204
 205             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 206                                    "tempest.log"), 'r') as logfile:
 207                 output = logfile.read()
 208
 209             success_testcases = []
 210             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 211                 success_testcases.append(match)
 212             failed_testcases = []
 213             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 214                 failed_testcases.append(match)
 215             skipped_testcases = []
 216             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 217                 skipped_testcases.append(match)
 218
 219             self.details = {"tests": int(num_tests),
 220                             "failures": int(num_failures),
 221                             "success": success_testcases,
 222                             "errors": failed_testcases,
 223                             "skipped": skipped_testcases}
 224         except Exception:
 225             self.result = 0
 226
 227         logger.info("Tempest %s success_rate is %s%%"
 228                     % (self.case_name, self.result))
 229
 230     def run(self):
 231
 232         self.start_time = time.time()
 233         try:
 234             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 235                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 236             resources = self.resources.create()
 237             compute_cnt = snaps_utils.get_active_compute_cnt(
 238                 self.resources.os_creds)
 239             conf_utils.configure_tempest(
 240                 self.DEPLOYMENT_DIR,
 241                 image_id=resources.get("image_id"),
 242                 flavor_id=resources.get("flavor_id"),
 243                 compute_cnt=compute_cnt)
 244             self.generate_test_list(self.VERIFIER_REPO_DIR)
 245             self.apply_tempest_blacklist()
 246             self.run_verifier_tests()
 247             self.parse_verifier_result()
 248             res = testcase.TestCase.EX_OK
 249         except Exception as e:
 250             logger.error('Error with run: %s' % e)
 251             res = testcase.TestCase.EX_RUN_ERROR
 252         finally:
 253             self.resources.cleanup()
 254
 255         self.stop_time = time.time()
 256         return res
 257
 258
 259 class TempestSmokeSerial(TempestCommon):
 260
 261     def __init__(self, **kwargs):
 262         if "case_name" not in kwargs:
 263             kwargs["case_name"] = 'tempest_smoke_serial'
 264         TempestCommon.__init__(self, **kwargs)
 265         self.MODE = "smoke"
 266         self.OPTION = "--concurrency 1"
 267
 268
 269 class TempestSmokeParallel(TempestCommon):
 270
 271     def __init__(self, **kwargs):
 272         if "case_name" not in kwargs:
 273             kwargs["case_name"] = 'tempest_smoke_parallel'
 274         TempestCommon.__init__(self, **kwargs)
 275         self.MODE = "smoke"
 276         self.OPTION = ""
 277
 278
 279 class TempestFullParallel(TempestCommon):
 280
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_full_parallel'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.MODE = "full"
 286
 287
 288 class TempestCustom(TempestCommon):
 289
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_custom'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.MODE = "custom"
 295         self.OPTION = "--concurrency 1"
 296
 297
 298 class TempestDefcore(TempestCommon):
 299
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'tempest_defcore'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.MODE = "defcore"
 305         self.OPTION = "--concurrency 1"
 306
 307
 308 class TempestResourcesManager(object):
 309
 310     def __init__(self, **kwargs):
 311         self.os_creds = None
 312         if 'os_creds' in kwargs:
 313             self.os_creds = kwargs['os_creds']
 314         else:
 315             self.os_creds = openstack_tests.get_credentials(
 316                 os_env_file=CONST.__getattribute__('openstack_creds'))
 317
 318         self.creators = list()
 319
 320         if hasattr(CONST, 'snaps_images_cirros'):
 321             self.cirros_image_config = CONST.__getattribute__(
 322                 'snaps_images_cirros')
 323         else:
 324             self.cirros_image_config = None
 325
 326     def create(self, use_custom_images=False, use_custom_flavors=False,
 327                create_project=False):
 328         if create_project:
 329             logger.debug("Creating project (tenant) for Tempest suite")
 330             project_name = CONST.__getattribute__(
 331                 'tempest_identity_tenant_name')
 332             project_creator = deploy_utils.create_project(
 333                 self.os_creds, ProjectSettings(
 334                     name=project_name,
 335                     description=CONST.__getattribute__(
 336                         'tempest_identity_tenant_description')))
 337             if (project_creator is None or
 338                     project_creator.get_project() is None):
 339                 raise Exception("Failed to create tenant")
 340             project_id = project_creator.get_project().id
 341             self.creators.append(project_creator)
 342
 343             logger.debug("Creating user for Tempest suite")
 344             user_creator = deploy_utils.create_user(
 345                 self.os_creds, UserSettings(
 346                     name=CONST.__getattribute__('tempest_identity_user_name'),
 347                     password=CONST.__getattribute__(
 348                         'tempest_identity_user_password'),
 349                     project_name=project_name))
 350             if user_creator is None or user_creator.get_user() is None:
 351                 raise Exception("Failed to create user")
 352             user_id = user_creator.get_user().id
 353             self.creators.append(user_creator)
 354         else:
 355             project_name = None
 356             project_id = None
 357             user_id = None
 358
 359         logger.debug("Creating private network for Tempest suite")
 360         network_creator = deploy_utils.create_network(
 361             self.os_creds, NetworkSettings(
 362                 name=CONST.__getattribute__('tempest_private_net_name'),
 363                 project_name=project_name,
 364                 subnet_settings=[SubnetSettings(
 365                     name=CONST.__getattribute__('tempest_private_subnet_name'),
 366                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 367                 ]))
 368         if network_creator is None or network_creator.get_network() is None:
 369             raise Exception("Failed to create private network")
 370         self.creators.append(network_creator)
 371
 372         image_id = None
 373         image_id_alt = None
 374         flavor_id = None
 375         flavor_id_alt = None
 376
 377         if (CONST.__getattribute__('tempest_use_custom_images') or
 378            use_custom_images):
 379             logger.debug("Creating image for Tempest suite")
 380             image_base_name = CONST.__getattribute__('openstack_image_name')
 381             os_image_settings = openstack_tests.cirros_image_settings(
 382                 image_base_name, public=True,
 383                 image_metadata=self.cirros_image_config)
 384             logger.debug("Creating image for Tempest suite")
 385             image_creator = deploy_utils.create_image(
 386                 self.os_creds, os_image_settings)
 387             if image_creator is None:
 388                 raise Exception('Failed to create image')
 389             self.creators.append(image_creator)
 390             image_id = image_creator.get_image().id
 391
 392         if use_custom_images:
 393             logger.debug("Creating 2nd image for Tempest suite")
 394             image_base_name_alt = CONST.__getattribute__(
 395                 'openstack_image_name_alt')
 396             os_image_settings_alt = openstack_tests.cirros_image_settings(
 397                 image_base_name_alt, public=True,
 398                 image_metadata=self.cirros_image_config)
 399             logger.debug("Creating 2nd image for Tempest suite")
 400             image_creator_alt = deploy_utils.create_image(
 401                 self.os_creds, os_image_settings_alt)
 402             if image_creator_alt is None:
 403                 raise Exception('Failed to create image')
 404             self.creators.append(image_creator_alt)
 405             image_id_alt = image_creator_alt.get_image().id
 406
 407         if (CONST.__getattribute__('tempest_use_custom_flavors') or
 408            use_custom_flavors):
 409             logger.info("Creating flavor for Tempest suite")
 410             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 411             flavor_metadata = None
 412             if 'ovs' in scenario or 'fdio' in scenario:
 413                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 414             flavor_creator = OpenStackFlavor(
 415                 self.os_creds, FlavorSettings(
 416                     name=CONST.__getattribute__('openstack_flavor_name'),
 417                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 418                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 419                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 420                     metadata=flavor_metadata))
 421             flavor = flavor_creator.create()
 422             if flavor is None:
 423                 raise Exception('Failed to create flavor')
 424             self.creators.append(flavor_creator)
 425             flavor_id = flavor.id
 426
 427         if use_custom_flavors:
 428             logger.info("Creating 2nd flavor for Tempest suite")
 429             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 430             flavor_metadata_alt = None
 431             if 'ovs' in scenario or 'fdio' in scenario:
 432                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 433             flavor_creator_alt = OpenStackFlavor(
 434                 self.os_creds, FlavorSettings(
 435                     name=CONST.__getattribute__('openstack_flavor_name_alt'),
 436                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 437                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 438                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 439                     metadata=flavor_metadata_alt))
 440             flavor_alt = flavor_creator_alt.create()
 441             if flavor_alt is None:
 442                 raise Exception('Failed to create flavor')
 443             self.creators.append(flavor_creator_alt)
 444             flavor_id_alt = flavor_alt.id
 445
 446         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 447               "flavor_id: %s, flavor_id_alt: %s" % (
 448                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 449
 450         result = {
 451             'image_id': image_id,
 452             'image_id_alt': image_id_alt,
 453             'flavor_id': flavor_id,
 454             'flavor_id_alt': flavor_id_alt
 455         }
 456
 457         if create_project:
 458             result['project_id'] = project_id
 459             result['tenant_id'] = project_id  # for compatibility
 460             result['user_id'] = user_id
 461
 462         return result
 463
 464     def cleanup(self):
 465         """
 466         Cleanup all OpenStack objects. Should be called on completion.
 467         """
 468         for creator in reversed(self.creators):
 469             try:
 470                 creator.clean()
 471             except Exception as e:
 472                 logger.error('Unexpected error cleaning - %s', e)