functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19
  20 import yaml
  21
  22 from functest.core import testcase
  23 from functest.opnfv_tests.openstack.snaps import snaps_utils
  24 from functest.opnfv_tests.openstack.tempest import conf_utils
  25 from functest.utils.constants import CONST
  26 import functest.utils.functest_utils as ft_utils
  27 import functest.utils.openstack_utils as os_utils
  28
  29 from snaps.openstack import create_flavor
  30 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  31 from snaps.openstack.create_project import ProjectSettings
  32 from snaps.openstack.create_network import NetworkSettings, SubnetSettings
  33 from snaps.openstack.create_user import UserSettings
  34 from snaps.openstack.tests import openstack_tests
  35 from snaps.openstack.utils import deploy_utils
  36
  37
  38 """ logging configuration """
  39 logger = logging.getLogger(__name__)
  40
  41
  42 class TempestCommon(testcase.TestCase):
  43
  44     def __init__(self, **kwargs):
  45         super(TempestCommon, self).__init__(**kwargs)
  46         self.resources = TempestResourcesManager(**kwargs)
  47         self.MODE = ""
  48         self.OPTION = ""
  49         self.VERIFIER_ID = conf_utils.get_verifier_id()
  50         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  51             self.VERIFIER_ID)
  52         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  53         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  54             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  55         self.VERIFICATION_ID = None
  56
  57     @staticmethod
  58     def read_file(filename):
  59         with open(filename) as src:
  60             return [line.strip() for line in src.readlines()]
  61
  62     def generate_test_list(self, verifier_repo_dir):
  63         logger.debug("Generating test case list...")
  64         if self.MODE == 'defcore':
  65             shutil.copyfile(
  66                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  67         elif self.MODE == 'custom':
  68             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  69                 shutil.copyfile(
  70                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  71             else:
  72                 raise Exception("Tempest test list file %s NOT found."
  73                                 % conf_utils.TEMPEST_CUSTOM)
  74         else:
  75             if self.MODE == 'smoke':
  76                 testr_mode = "smoke"
  77             elif self.MODE == 'full':
  78                 testr_mode = ""
  79             else:
  80                 testr_mode = 'tempest.api.' + self.MODE
  81             cmd = ("cd {0};"
  82                    "testr list-tests {1} > {2};"
  83                    "cd -;".format(verifier_repo_dir,
  84                                   testr_mode,
  85                                   conf_utils.TEMPEST_RAW_LIST))
  86             ft_utils.execute_command(cmd)
  87
  88     def apply_tempest_blacklist(self):
  89         logger.debug("Applying tempest blacklist...")
  90         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  91         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  92         black_tests = []
  93         try:
  94             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  95             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  96             if (bool(installer_type) * bool(deploy_scenario)):
  97                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  98                 # file
  99                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 100                 black_list_yaml = yaml.safe_load(black_list_file)
 101                 black_list_file.close()
 102                 for item in black_list_yaml:
 103                     scenarios = item['scenarios']
 104                     installers = item['installers']
 105                     if (deploy_scenario in scenarios and
 106                             installer_type in installers):
 107                         tests = item['tests']
 108                         for test in tests:
 109                             black_tests.append(test)
 110                         break
 111         except Exception:
 112             black_tests = []
 113             logger.debug("Tempest blacklist file does not exist.")
 114
 115         for cases_line in cases_file:
 116             for black_tests_line in black_tests:
 117                 if black_tests_line in cases_line:
 118                     break
 119             else:
 120                 result_file.write(str(cases_line) + '\n')
 121         result_file.close()
 122
 123     def run_verifier_tests(self):
 124         self.OPTION += (" --load-list {} --detailed"
 125                         .format(conf_utils.TEMPEST_LIST))
 126
 127         cmd_line = "rally verify start " + self.OPTION
 128         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 129
 130         header = ("Tempest environment:\n"
 131                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 132                   (CONST.__getattribute__('INSTALLER_TYPE'),
 133                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 134                    CONST.__getattribute__('NODE_NAME'),
 135                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 136
 137         f_stdout = open(
 138             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 139         f_stderr = open(
 140             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 141                          "tempest-error.log"), 'w+')
 142         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 143                                   "environment.log"), 'w+')
 144         f_env.write(header)
 145
 146         p = subprocess.Popen(
 147             cmd_line, shell=True,
 148             stdout=subprocess.PIPE,
 149             stderr=f_stderr,
 150             bufsize=1)
 151
 152         with p.stdout:
 153             for line in iter(p.stdout.readline, b''):
 154                 if re.search("\} tempest\.", line):
 155                     logger.info(line.replace('\n', ''))
 156                 elif re.search('Starting verification', line):
 157                     logger.info(line.replace('\n', ''))
 158                     first_pos = line.index("UUID=") + len("UUID=")
 159                     last_pos = line.index(") for deployment")
 160                     self.VERIFICATION_ID = line[first_pos:last_pos]
 161                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 162                 f_stdout.write(line)
 163         p.wait()
 164
 165         f_stdout.close()
 166         f_stderr.close()
 167         f_env.close()
 168
 169     def parse_verifier_result(self):
 170         if self.VERIFICATION_ID is None:
 171             raise Exception('Verification UUID not found')
 172
 173         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 174         logger.info("Showing result for a verification: '%s'." % cmd_line)
 175         p = subprocess.Popen(cmd_line,
 176                              shell=True,
 177                              stdout=subprocess.PIPE,
 178                              stderr=subprocess.STDOUT)
 179         for line in p.stdout:
 180             new_line = line.replace(' ', '').split('|')
 181             if 'Tests' in new_line:
 182                 break
 183
 184             logger.info(line)
 185             if 'Testscount' in new_line:
 186                 num_tests = new_line[2]
 187             elif 'Success' in new_line:
 188                 num_success = new_line[2]
 189             elif 'Skipped' in new_line:
 190                 num_skipped = new_line[2]
 191             elif 'Failures' in new_line:
 192                 num_failures = new_line[2]
 193
 194         try:
 195             num_executed = int(num_tests) - int(num_skipped)
 196             try:
 197                 self.result = 100 * int(num_success) / int(num_executed)
 198             except ZeroDivisionError:
 199                 self.result = 0
 200                 if int(num_tests) > 0:
 201                     logger.info("All tests have been skipped")
 202                 else:
 203                     logger.error("No test has been executed")
 204                     return
 205
 206             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 207                                    "tempest.log"), 'r') as logfile:
 208                 output = logfile.read()
 209
 210             success_testcases = []
 211             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 212                 success_testcases.append(match)
 213             failed_testcases = []
 214             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 215                 failed_testcases.append(match)
 216             skipped_testcases = []
 217             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 218                 skipped_testcases.append(match)
 219
 220             self.details = {"tests": int(num_tests),
 221                             "failures": int(num_failures),
 222                             "success": success_testcases,
 223                             "errors": failed_testcases,
 224                             "skipped": skipped_testcases}
 225         except Exception:
 226             self.result = 0
 227
 228         logger.info("Tempest %s success_rate is %s%%"
 229                     % (self.case_name, self.result))
 230
 231     def run(self):
 232
 233         self.start_time = time.time()
 234         try:
 235             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 236                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 237             resources = self.resources.create()
 238             compute_cnt = snaps_utils.get_active_compute_cnt(
 239                 self.resources.os_creds)
 240             conf_utils.configure_tempest(
 241                 self.DEPLOYMENT_DIR,
 242                 image_id=resources.get("image_id"),
 243                 flavor_id=resources.get("flavor_id"),
 244                 compute_cnt=compute_cnt)
 245             self.generate_test_list(self.VERIFIER_REPO_DIR)
 246             self.apply_tempest_blacklist()
 247             self.run_verifier_tests()
 248             self.parse_verifier_result()
 249             res = testcase.TestCase.EX_OK
 250         except Exception as e:
 251             logger.error('Error with run: %s' % e)
 252             res = testcase.TestCase.EX_RUN_ERROR
 253         finally:
 254             self.resources.cleanup()
 255
 256         self.stop_time = time.time()
 257         return res
 258
 259     def create_snapshot(self):
 260         """
 261         Run the Tempest cleanup utility to initialize OS state.
 262
 263         :return: TestCase.EX_OK
 264         """
 265         logger.info("Initializing the saved state of the OpenStack deployment")
 266
 267         if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 268             os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 269
 270         # Make sure that the verifier is configured
 271         conf_utils.configure_verifier(self.DEPLOYMENT_DIR)
 272
 273         try:
 274             os_utils.init_tempest_cleanup(
 275                 self.DEPLOYMENT_DIR, 'tempest.conf',
 276                 os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 277                              "tempest-cleanup-init.log"))
 278         except Exception as err:
 279             logger.error(str(err))
 280             return testcase.TestCase.EX_RUN_ERROR
 281
 282         return super(TempestCommon, self).create_snapshot()
 283
 284     def clean(self):
 285         """
 286         Run the Tempest cleanup utility to delete and destroy OS resources
 287         created by Tempest.
 288         """
 289         logger.info("Destroying the resources created for refstack")
 290
 291         os_utils.perform_tempest_cleanup(
 292             self.DEPLOYMENT_DIR, 'tempest.conf',
 293             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 294                          "tempest-cleanup.log")
 295         )
 296
 297         return super(TempestCommon, self).clean()
 298
 299
 300 class TempestSmokeSerial(TempestCommon):
 301
 302     def __init__(self, **kwargs):
 303         if "case_name" not in kwargs:
 304             kwargs["case_name"] = 'tempest_smoke_serial'
 305         TempestCommon.__init__(self, **kwargs)
 306         self.MODE = "smoke"
 307         self.OPTION = "--concurrency 1"
 308
 309
 310 class TempestSmokeParallel(TempestCommon):
 311
 312     def __init__(self, **kwargs):
 313         if "case_name" not in kwargs:
 314             kwargs["case_name"] = 'tempest_smoke_parallel'
 315         TempestCommon.__init__(self, **kwargs)
 316         self.MODE = "smoke"
 317         self.OPTION = ""
 318
 319
 320 class TempestFullParallel(TempestCommon):
 321
 322     def __init__(self, **kwargs):
 323         if "case_name" not in kwargs:
 324             kwargs["case_name"] = 'tempest_full_parallel'
 325         TempestCommon.__init__(self, **kwargs)
 326         self.MODE = "full"
 327
 328
 329 class TempestCustom(TempestCommon):
 330
 331     def __init__(self, **kwargs):
 332         if "case_name" not in kwargs:
 333             kwargs["case_name"] = 'tempest_custom'
 334         TempestCommon.__init__(self, **kwargs)
 335         self.MODE = "custom"
 336         self.OPTION = "--concurrency 1"
 337
 338
 339 class TempestDefcore(TempestCommon):
 340
 341     def __init__(self, **kwargs):
 342         if "case_name" not in kwargs:
 343             kwargs["case_name"] = 'tempest_defcore'
 344         TempestCommon.__init__(self, **kwargs)
 345         self.MODE = "defcore"
 346         self.OPTION = "--concurrency 1"
 347
 348
 349 class TempestResourcesManager(object):
 350
 351     def __init__(self, **kwargs):
 352         self.os_creds = None
 353         if 'os_creds' in kwargs:
 354             self.os_creds = kwargs['os_creds']
 355         else:
 356             self.os_creds = openstack_tests.get_credentials(
 357                 os_env_file=CONST.__getattribute__('openstack_creds'))
 358
 359         self.creators = list()
 360
 361         if hasattr(CONST, 'snaps_images_cirros'):
 362             self.cirros_image_config = CONST.__getattribute__(
 363                 'snaps_images_cirros')
 364         else:
 365             self.cirros_image_config = None
 366
 367     def create(self, use_custom_images=False, use_custom_flavors=False,
 368                create_project=False):
 369         if create_project:
 370             logger.debug("Creating project (tenant) for Tempest suite")
 371             project_name = CONST.__getattribute__(
 372                 'tempest_identity_tenant_name')
 373             project_creator = deploy_utils.create_project(
 374                 self.os_creds, ProjectSettings(
 375                     name=project_name,
 376                     description=CONST.__getattribute__(
 377                         'tempest_identity_tenant_description')))
 378             if (project_creator is None or
 379                     project_creator.get_project() is None):
 380                 raise Exception("Failed to create tenant")
 381             project_id = project_creator.get_project().id
 382             self.creators.append(project_creator)
 383
 384             logger.debug("Creating user for Tempest suite")
 385             user_creator = deploy_utils.create_user(
 386                 self.os_creds, UserSettings(
 387                     name=CONST.__getattribute__('tempest_identity_user_name'),
 388                     password=CONST.__getattribute__(
 389                         'tempest_identity_user_password'),
 390                     project_name=project_name))
 391             if user_creator is None or user_creator.get_user() is None:
 392                 raise Exception("Failed to create user")
 393             user_id = user_creator.get_user().id
 394             self.creators.append(user_creator)
 395         else:
 396             project_name = None
 397             project_id = None
 398             user_id = None
 399
 400         logger.debug("Creating private network for Tempest suite")
 401         network_creator = deploy_utils.create_network(
 402             self.os_creds, NetworkSettings(
 403                 name=CONST.__getattribute__('tempest_private_net_name'),
 404                 project_name=project_name,
 405                 subnet_settings=[SubnetSettings(
 406                     name=CONST.__getattribute__('tempest_private_subnet_name'),
 407                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 408                 ]))
 409         if network_creator is None or network_creator.get_network() is None:
 410             raise Exception("Failed to create private network")
 411         self.creators.append(network_creator)
 412
 413         image_id = None
 414         image_id_alt = None
 415         flavor_id = None
 416         flavor_id_alt = None
 417
 418         if (CONST.__getattribute__('tempest_use_custom_images') or
 419            use_custom_images):
 420             logger.debug("Creating image for Tempest suite")
 421             image_base_name = CONST.__getattribute__('openstack_image_name')
 422             os_image_settings = openstack_tests.cirros_image_settings(
 423                 image_base_name, public=True,
 424                 image_metadata=self.cirros_image_config)
 425             logger.debug("Creating image for Tempest suite")
 426             image_creator = deploy_utils.create_image(
 427                 self.os_creds, os_image_settings)
 428             if image_creator is None:
 429                 raise Exception('Failed to create image')
 430             self.creators.append(image_creator)
 431             image_id = image_creator.get_image().id
 432
 433         if use_custom_images:
 434             logger.debug("Creating 2nd image for Tempest suite")
 435             image_base_name_alt = CONST.__getattribute__(
 436                 'openstack_image_name_alt')
 437             os_image_settings_alt = openstack_tests.cirros_image_settings(
 438                 image_base_name_alt, public=True,
 439                 image_metadata=self.cirros_image_config)
 440             logger.debug("Creating 2nd image for Tempest suite")
 441             image_creator_alt = deploy_utils.create_image(
 442                 self.os_creds, os_image_settings_alt)
 443             if image_creator_alt is None:
 444                 raise Exception('Failed to create image')
 445             self.creators.append(image_creator_alt)
 446             image_id_alt = image_creator_alt.get_image().id
 447
 448         if (CONST.__getattribute__('tempest_use_custom_flavors') or
 449            use_custom_flavors):
 450             logger.info("Creating flavor for Tempest suite")
 451             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 452             flavor_metadata = None
 453             if 'ovs' in scenario or 'fdio' in scenario:
 454                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 455             flavor_creator = OpenStackFlavor(
 456                 self.os_creds, FlavorSettings(
 457                     name=CONST.__getattribute__('openstack_flavor_name'),
 458                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 459                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 460                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 461                     metadata=flavor_metadata))
 462             flavor = flavor_creator.create()
 463             if flavor is None:
 464                 raise Exception('Failed to create flavor')
 465             self.creators.append(flavor_creator)
 466             flavor_id = flavor.id
 467
 468         if use_custom_flavors:
 469             logger.info("Creating 2nd flavor for Tempest suite")
 470             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 471             flavor_metadata_alt = None
 472             if 'ovs' in scenario or 'fdio' in scenario:
 473                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 474             flavor_creator_alt = OpenStackFlavor(
 475                 self.os_creds, FlavorSettings(
 476                     name=CONST.__getattribute__('openstack_flavor_name_alt'),
 477                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 478                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 479                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 480                     metadata=flavor_metadata_alt))
 481             flavor_alt = flavor_creator_alt.create()
 482             if flavor_alt is None:
 483                 raise Exception('Failed to create flavor')
 484             self.creators.append(flavor_creator_alt)
 485             flavor_id_alt = flavor_alt.id
 486
 487         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 488               "flavor_id: %s, flavor_id_alt: %s" % (
 489                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 490
 491         result = {
 492             'image_id': image_id,
 493             'image_id_alt': image_id_alt,
 494             'flavor_id': flavor_id,
 495             'flavor_id_alt': flavor_id_alt
 496         }
 497
 498         if create_project:
 499             result['project_id'] = project_id
 500             result['tenant_id'] = project_id  # for compatibility
 501             result['user_id'] = user_id
 502
 503         return result
 504
 505     def cleanup(self):
 506         """
 507         Cleanup all OpenStack objects. Should be called on completion.
 508         """
 509         for creator in reversed(self.creators):
 510             try:
 511                 creator.clean()
 512             except Exception as e:
 513                 logger.error('Unexpected error cleaning - %s', e)