functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.tempest import conf_utils
  24 from functest.utils.constants import CONST
  25 import functest.utils.functest_utils as ft_utils
  26 import functest.utils.openstack_utils as os_utils
  27
  28 from snaps.openstack import create_flavor
  29 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  30 from snaps.openstack.create_project import ProjectSettings
  31 from snaps.openstack.create_network import NetworkSettings, SubnetSettings
  32 from snaps.openstack.create_user import UserSettings
  33 from snaps.openstack.tests import openstack_tests
  34 from snaps.openstack.utils import deploy_utils
  35
  36
  37 """ logging configuration """
  38 logger = logging.getLogger(__name__)
  39
  40
  41 class TempestCommon(testcase.TestCase):
  42
  43     def __init__(self, **kwargs):
  44         super(TempestCommon, self).__init__(**kwargs)
  45         self.resources = TempestResourcesManager(**kwargs)
  46         self.MODE = ""
  47         self.OPTION = ""
  48         self.VERIFIER_ID = conf_utils.get_verifier_id()
  49         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  50             self.VERIFIER_ID)
  51         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  52         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  53             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  54         self.VERIFICATION_ID = None
  55
  56     @staticmethod
  57     def read_file(filename):
  58         with open(filename) as src:
  59             return [line.strip() for line in src.readlines()]
  60
  61     def generate_test_list(self, verifier_repo_dir):
  62         logger.debug("Generating test case list...")
  63         if self.MODE == 'defcore':
  64             shutil.copyfile(
  65                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  66         elif self.MODE == 'custom':
  67             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  68                 shutil.copyfile(
  69                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  70             else:
  71                 raise Exception("Tempest test list file %s NOT found."
  72                                 % conf_utils.TEMPEST_CUSTOM)
  73         else:
  74             if self.MODE == 'smoke':
  75                 testr_mode = "smoke"
  76             elif self.MODE == 'full':
  77                 testr_mode = ""
  78             else:
  79                 testr_mode = 'tempest.api.' + self.MODE
  80             cmd = ("cd {0};"
  81                    "testr list-tests {1} > {2};"
  82                    "cd -;".format(verifier_repo_dir,
  83                                   testr_mode,
  84                                   conf_utils.TEMPEST_RAW_LIST))
  85             ft_utils.execute_command(cmd)
  86
  87     def apply_tempest_blacklist(self):
  88         logger.debug("Applying tempest blacklist...")
  89         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  90         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  91         black_tests = []
  92         try:
  93             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  94             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  95             if (bool(installer_type) * bool(deploy_scenario)):
  96                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  97                 # file
  98                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
  99                 black_list_yaml = yaml.safe_load(black_list_file)
 100                 black_list_file.close()
 101                 for item in black_list_yaml:
 102                     scenarios = item['scenarios']
 103                     installers = item['installers']
 104                     if (deploy_scenario in scenarios and
 105                             installer_type in installers):
 106                         tests = item['tests']
 107                         for test in tests:
 108                             black_tests.append(test)
 109                         break
 110         except Exception:
 111             black_tests = []
 112             logger.debug("Tempest blacklist file does not exist.")
 113
 114         for cases_line in cases_file:
 115             for black_tests_line in black_tests:
 116                 if black_tests_line in cases_line:
 117                     break
 118             else:
 119                 result_file.write(str(cases_line) + '\n')
 120         result_file.close()
 121
 122     def run_verifier_tests(self):
 123         self.OPTION += (" --load-list {} --detailed"
 124                         .format(conf_utils.TEMPEST_LIST))
 125
 126         cmd_line = "rally verify start " + self.OPTION
 127         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 128
 129         header = ("Tempest environment:\n"
 130                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 131                   (CONST.__getattribute__('INSTALLER_TYPE'),
 132                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 133                    CONST.__getattribute__('NODE_NAME'),
 134                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 135
 136         f_stdout = open(
 137             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 138         f_stderr = open(
 139             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 140                          "tempest-error.log"), 'w+')
 141         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 142                                   "environment.log"), 'w+')
 143         f_env.write(header)
 144
 145         p = subprocess.Popen(
 146             cmd_line, shell=True,
 147             stdout=subprocess.PIPE,
 148             stderr=f_stderr,
 149             bufsize=1)
 150
 151         with p.stdout:
 152             for line in iter(p.stdout.readline, b''):
 153                 if re.search("\} tempest\.", line):
 154                     logger.info(line.replace('\n', ''))
 155                 elif re.search('Starting verification', line):
 156                     logger.info(line.replace('\n', ''))
 157                     first_pos = line.index("UUID=") + len("UUID=")
 158                     last_pos = line.index(") for deployment")
 159                     self.VERIFICATION_ID = line[first_pos:last_pos]
 160                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 161                 f_stdout.write(line)
 162         p.wait()
 163
 164         f_stdout.close()
 165         f_stderr.close()
 166         f_env.close()
 167
 168     def parse_verifier_result(self):
 169         if self.VERIFICATION_ID is None:
 170             raise Exception('Verification UUID not found')
 171
 172         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 173         logger.info("Showing result for a verification: '%s'." % cmd_line)
 174         p = subprocess.Popen(cmd_line,
 175                              shell=True,
 176                              stdout=subprocess.PIPE,
 177                              stderr=subprocess.STDOUT)
 178         for line in p.stdout:
 179             new_line = line.replace(' ', '').split('|')
 180             if 'Tests' in new_line:
 181                 break
 182
 183             logger.info(line)
 184             if 'Testscount' in new_line:
 185                 num_tests = new_line[2]
 186             elif 'Success' in new_line:
 187                 num_success = new_line[2]
 188             elif 'Skipped' in new_line:
 189                 num_skipped = new_line[2]
 190             elif 'Failures' in new_line:
 191                 num_failures = new_line[2]
 192
 193         try:
 194             num_executed = int(num_tests) - int(num_skipped)
 195             try:
 196                 self.result = 100 * int(num_success) / int(num_executed)
 197             except ZeroDivisionError:
 198                 self.result = 0
 199                 if int(num_tests) > 0:
 200                     logger.info("All tests have been skipped")
 201                 else:
 202                     logger.error("No test has been executed")
 203                     return
 204
 205             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 206                                    "tempest.log"), 'r') as logfile:
 207                 output = logfile.read()
 208
 209             success_testcases = []
 210             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 211                 success_testcases.append(match)
 212             failed_testcases = []
 213             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 214                 failed_testcases.append(match)
 215             skipped_testcases = []
 216             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 217                 skipped_testcases.append(match)
 218
 219             self.details = {"tests": int(num_tests),
 220                             "failures": int(num_failures),
 221                             "success": success_testcases,
 222                             "errors": failed_testcases,
 223                             "skipped": skipped_testcases}
 224         except Exception:
 225             self.result = 0
 226
 227         logger.info("Tempest %s success_rate is %s%%"
 228                     % (self.case_name, self.result))
 229
 230     def run(self):
 231
 232         self.start_time = time.time()
 233         try:
 234             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 235                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 236             resources = self.resources.create()
 237             conf_utils.configure_tempest(
 238                 self.DEPLOYMENT_DIR,
 239                 image_id=resources.get("image_id"),
 240                 flavor_id=resources.get("flavor_id"),
 241                 mode=self.MODE)
 242             self.generate_test_list(self.VERIFIER_REPO_DIR)
 243             self.apply_tempest_blacklist()
 244             self.run_verifier_tests()
 245             self.parse_verifier_result()
 246             res = testcase.TestCase.EX_OK
 247         except Exception as e:
 248             logger.error('Error with run: %s' % e)
 249             res = testcase.TestCase.EX_RUN_ERROR
 250         finally:
 251             self.resources.cleanup()
 252
 253         self.stop_time = time.time()
 254         return res
 255
 256     def create_snapshot(self):
 257         """
 258         Run the Tempest cleanup utility to initialize OS state.
 259
 260         :return: TestCase.EX_OK
 261         """
 262         logger.info("Initializing the saved state of the OpenStack deployment")
 263
 264         if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 265             os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 266
 267         # Make sure that the verifier is configured
 268         conf_utils.configure_verifier(self.DEPLOYMENT_DIR)
 269
 270         try:
 271             os_utils.init_tempest_cleanup(
 272                 self.DEPLOYMENT_DIR, 'tempest.conf',
 273                 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 274                              "tempest-cleanup-init.log"))
 275         except Exception as err:
 276             logger.error(str(err))
 277             return testcase.TestCase.EX_RUN_ERROR
 278
 279         return super(TempestCommon, self).create_snapshot()
 280
 281     def clean(self):
 282         """
 283         Run the Tempest cleanup utility to delete and destroy OS resources
 284         created by Tempest.
 285         """
 286         logger.info("Destroying the resources created for refstack")
 287
 288         os_utils.perform_tempest_cleanup(
 289             self.DEPLOYMENT_DIR, 'tempest.conf',
 290             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 291                          "tempest-cleanup.log")
 292         )
 293
 294         return super(TempestCommon, self).clean()
 295
 296
 297 class TempestSmokeSerial(TempestCommon):
 298
 299     def __init__(self, **kwargs):
 300         if "case_name" not in kwargs:
 301             kwargs["case_name"] = 'tempest_smoke_serial'
 302         TempestCommon.__init__(self, **kwargs)
 303         self.MODE = "smoke"
 304         self.OPTION = "--concurrency 1"
 305
 306
 307 class TempestSmokeParallel(TempestCommon):
 308
 309     def __init__(self, **kwargs):
 310         if "case_name" not in kwargs:
 311             kwargs["case_name"] = 'tempest_smoke_parallel'
 312         TempestCommon.__init__(self, **kwargs)
 313         self.MODE = "smoke"
 314         self.OPTION = ""
 315
 316
 317 class TempestFullParallel(TempestCommon):
 318
 319     def __init__(self, **kwargs):
 320         if "case_name" not in kwargs:
 321             kwargs["case_name"] = 'tempest_full_parallel'
 322         TempestCommon.__init__(self, **kwargs)
 323         self.MODE = "full"
 324
 325
 326 class TempestCustom(TempestCommon):
 327
 328     def __init__(self, **kwargs):
 329         if "case_name" not in kwargs:
 330             kwargs["case_name"] = 'tempest_custom'
 331         TempestCommon.__init__(self, **kwargs)
 332         self.MODE = "custom"
 333         self.OPTION = "--concurrency 1"
 334
 335
 336 class TempestDefcore(TempestCommon):
 337
 338     def __init__(self, **kwargs):
 339         if "case_name" not in kwargs:
 340             kwargs["case_name"] = 'tempest_defcore'
 341         TempestCommon.__init__(self, **kwargs)
 342         self.MODE = "defcore"
 343         self.OPTION = "--concurrency 1"
 344
 345
 346 class TempestResourcesManager(object):
 347
 348     def __init__(self, **kwargs):
 349         self.os_creds = None
 350         if 'os_creds' in kwargs:
 351             self.os_creds = kwargs['os_creds']
 352         else:
 353             self.os_creds = openstack_tests.get_credentials(
 354                 os_env_file=CONST.__getattribute__('openstack_creds'))
 355
 356         self.creators = list()
 357
 358         if hasattr(CONST, 'snaps_images_cirros'):
 359             self.cirros_image_config = CONST.__getattribute__(
 360                 'snaps_images_cirros')
 361         else:
 362             self.cirros_image_config = None
 363
 364     def create(self, use_custom_images=False, use_custom_flavors=False,
 365                create_project=False):
 366         if create_project:
 367             logger.debug("Creating project (tenant) for Tempest suite")
 368             project_name = CONST.__getattribute__(
 369                 'tempest_identity_tenant_name')
 370             project_creator = deploy_utils.create_project(
 371                 self.os_creds, ProjectSettings(
 372                     name=project_name,
 373                     description=CONST.__getattribute__(
 374                         'tempest_identity_tenant_description')))
 375             if (project_creator is None or
 376                     project_creator.get_project() is None):
 377                 raise Exception("Failed to create tenant")
 378             project_id = project_creator.get_project().id
 379             self.creators.append(project_creator)
 380
 381             logger.debug("Creating user for Tempest suite")
 382             user_creator = deploy_utils.create_user(
 383                 self.os_creds, UserSettings(
 384                     name=CONST.__getattribute__('tempest_identity_user_name'),
 385                     password=CONST.__getattribute__(
 386                         'tempest_identity_user_password'),
 387                     project_name=project_name))
 388             if user_creator is None or user_creator.get_user() is None:
 389                 raise Exception("Failed to create user")
 390             user_id = user_creator.get_user().id
 391             self.creators.append(user_creator)
 392         else:
 393             project_name = None
 394             project_id = None
 395             user_id = None
 396
 397         logger.debug("Creating private network for Tempest suite")
 398         network_creator = deploy_utils.create_network(
 399             self.os_creds, NetworkSettings(
 400                 name=CONST.__getattribute__('tempest_private_net_name'),
 401                 project_name=project_name,
 402                 subnet_settings=[SubnetSettings(
 403                     name=CONST.__getattribute__('tempest_private_subnet_name'),
 404                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 405                 ]))
 406         if network_creator is None or network_creator.get_network() is None:
 407             raise Exception("Failed to create private network")
 408         self.creators.append(network_creator)
 409
 410         image_id = None
 411         image_id_alt = None
 412         flavor_id = None
 413         flavor_id_alt = None
 414
 415         if (CONST.__getattribute__('tempest_use_custom_images') or
 416            use_custom_images):
 417             logger.debug("Creating image for Tempest suite")
 418             image_base_name = CONST.__getattribute__('openstack_image_name')
 419             os_image_settings = openstack_tests.cirros_image_settings(
 420                 image_base_name, public=True,
 421                 image_metadata=self.cirros_image_config)
 422             logger.debug("Creating image for Tempest suite")
 423             image_creator = deploy_utils.create_image(
 424                 self.os_creds, os_image_settings)
 425             if image_creator is None:
 426                 raise Exception('Failed to create image')
 427             self.creators.append(image_creator)
 428             image_id = image_creator.get_image().id
 429
 430         if use_custom_images:
 431             logger.debug("Creating 2nd image for Tempest suite")
 432             image_base_name_alt = CONST.__getattribute__(
 433                 'openstack_image_name_alt')
 434             os_image_settings_alt = openstack_tests.cirros_image_settings(
 435                 image_base_name_alt, public=True,
 436                 image_metadata=self.cirros_image_config)
 437             logger.debug("Creating 2nd image for Tempest suite")
 438             image_creator_alt = deploy_utils.create_image(
 439                 self.os_creds, os_image_settings_alt)
 440             if image_creator_alt is None:
 441                 raise Exception('Failed to create image')
 442             self.creators.append(image_creator_alt)
 443             image_id_alt = image_creator_alt.get_image().id
 444
 445         if (CONST.__getattribute__('tempest_use_custom_flavors') or
 446            use_custom_flavors):
 447             logger.info("Creating flavor for Tempest suite")
 448             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 449             flavor_metadata = None
 450             if 'ovs' in scenario or 'fdio' in scenario:
 451                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 452             flavor_creator = OpenStackFlavor(
 453                 self.os_creds, FlavorSettings(
 454                     name=CONST.__getattribute__('openstack_flavor_name'),
 455                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 456                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 457                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 458                     metadata=flavor_metadata))
 459             flavor = flavor_creator.create()
 460             if flavor is None:
 461                 raise Exception('Failed to create flavor')
 462             self.creators.append(flavor_creator)
 463             flavor_id = flavor.id
 464
 465         if use_custom_flavors:
 466             logger.info("Creating 2nd flavor for Tempest suite")
 467             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 468             flavor_metadata_alt = None
 469             if 'ovs' in scenario or 'fdio' in scenario:
 470                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 471             flavor_creator_alt = OpenStackFlavor(
 472                 self.os_creds, FlavorSettings(
 473                     name=CONST.__getattribute__('openstack_flavor_name_alt'),
 474                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 475                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 476                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 477                     metadata=flavor_metadata_alt))
 478             flavor_alt = flavor_creator_alt.create()
 479             if flavor_alt is None:
 480                 raise Exception('Failed to create flavor')
 481             self.creators.append(flavor_creator_alt)
 482             flavor_id_alt = flavor_alt.id
 483
 484         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 485               "flavor_id: %s, flavor_id_alt: %s" % (
 486                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 487
 488         result = {
 489             'image_id': image_id,
 490             'image_id_alt': image_id_alt,
 491             'flavor_id': flavor_id,
 492             'flavor_id_alt': flavor_id_alt
 493         }
 494
 495         if create_project:
 496             result['project_id'] = project_id
 497             result['tenant_id'] = project_id  # for compatibility
 498             result['user_id'] = user_id
 499
 500         return result
 501
 502     def cleanup(self):
 503         """
 504         Cleanup all OpenStack objects. Should be called on completion.
 505         """
 506         for creator in reversed(self.creators):
 507             try:
 508                 creator.clean()
 509             except Exception as e:
 510                 logger.error('Unexpected error cleaning - %s', e)