functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19 import uuid
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.opnfv_tests.openstack.tempest import conf_utils
  26 from functest.utils.constants import CONST
  27 import functest.utils.functest_utils as ft_utils
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.network import NetworkConfig, SubnetConfig
  31 from snaps.config.project import ProjectConfig
  32 from snaps.config.user import UserConfig
  33
  34 from snaps.openstack import create_flavor
  35 from snaps.openstack.create_flavor import OpenStackFlavor
  36 from snaps.openstack.tests import openstack_tests
  37 from snaps.openstack.utils import deploy_utils
  38
  39
  40 """ logging configuration """
  41 logger = logging.getLogger(__name__)
  42
  43
  44 class TempestCommon(testcase.TestCase):
  45
  46     def __init__(self, **kwargs):
  47         super(TempestCommon, self).__init__(**kwargs)
  48         self.resources = TempestResourcesManager(**kwargs)
  49         self.MODE = ""
  50         self.OPTION = ""
  51         self.VERIFIER_ID = conf_utils.get_verifier_id()
  52         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  53             self.VERIFIER_ID)
  54         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  55         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  56             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  57         self.VERIFICATION_ID = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     def generate_test_list(self, verifier_repo_dir):
  65         logger.debug("Generating test case list...")
  66         if self.MODE == 'defcore':
  67             shutil.copyfile(
  68                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  69         elif self.MODE == 'custom':
  70             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  71                 shutil.copyfile(
  72                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  73             else:
  74                 raise Exception("Tempest test list file %s NOT found."
  75                                 % conf_utils.TEMPEST_CUSTOM)
  76         else:
  77             if self.MODE == 'smoke':
  78                 testr_mode = "smoke"
  79             elif self.MODE == 'full':
  80                 testr_mode = ""
  81             else:
  82                 testr_mode = 'tempest.api.' + self.MODE
  83             cmd = ("cd {0};"
  84                    "testr list-tests {1} > {2};"
  85                    "cd -;".format(verifier_repo_dir,
  86                                   testr_mode,
  87                                   conf_utils.TEMPEST_RAW_LIST))
  88             ft_utils.execute_command(cmd)
  89
  90     def apply_tempest_blacklist(self):
  91         logger.debug("Applying tempest blacklist...")
  92         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  93         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  94         black_tests = []
  95         try:
  96             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  97             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  98             if (bool(installer_type) * bool(deploy_scenario)):
  99                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 100                 # file
 101                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 102                 black_list_yaml = yaml.safe_load(black_list_file)
 103                 black_list_file.close()
 104                 for item in black_list_yaml:
 105                     scenarios = item['scenarios']
 106                     installers = item['installers']
 107                     if (deploy_scenario in scenarios and
 108                             installer_type in installers):
 109                         tests = item['tests']
 110                         for test in tests:
 111                             black_tests.append(test)
 112                         break
 113         except Exception:
 114             black_tests = []
 115             logger.debug("Tempest blacklist file does not exist.")
 116
 117         for cases_line in cases_file:
 118             for black_tests_line in black_tests:
 119                 if black_tests_line in cases_line:
 120                     break
 121             else:
 122                 result_file.write(str(cases_line) + '\n')
 123         result_file.close()
 124
 125     def run_verifier_tests(self):
 126         self.OPTION += (" --load-list {} --detailed"
 127                         .format(conf_utils.TEMPEST_LIST))
 128
 129         cmd_line = "rally verify start " + self.OPTION
 130         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 131
 132         header = ("Tempest environment:\n"
 133                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 134                   (CONST.__getattribute__('INSTALLER_TYPE'),
 135                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 136                    CONST.__getattribute__('NODE_NAME'),
 137                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 138
 139         f_stdout = open(
 140             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 141         f_stderr = open(
 142             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 143                          "tempest-error.log"), 'w+')
 144         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 145                                   "environment.log"), 'w+')
 146         f_env.write(header)
 147
 148         p = subprocess.Popen(
 149             cmd_line, shell=True,
 150             stdout=subprocess.PIPE,
 151             stderr=f_stderr,
 152             bufsize=1)
 153
 154         with p.stdout:
 155             for line in iter(p.stdout.readline, b''):
 156                 if re.search("\} tempest\.", line):
 157                     logger.info(line.replace('\n', ''))
 158                 elif re.search('Starting verification', line):
 159                     logger.info(line.replace('\n', ''))
 160                     first_pos = line.index("UUID=") + len("UUID=")
 161                     last_pos = line.index(") for deployment")
 162                     self.VERIFICATION_ID = line[first_pos:last_pos]
 163                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 164                 f_stdout.write(line)
 165         p.wait()
 166
 167         f_stdout.close()
 168         f_stderr.close()
 169         f_env.close()
 170
 171     def parse_verifier_result(self):
 172         if self.VERIFICATION_ID is None:
 173             raise Exception('Verification UUID not found')
 174
 175         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 176         logger.info("Showing result for a verification: '%s'." % cmd_line)
 177         p = subprocess.Popen(cmd_line,
 178                              shell=True,
 179                              stdout=subprocess.PIPE,
 180                              stderr=subprocess.STDOUT)
 181         for line in p.stdout:
 182             new_line = line.replace(' ', '').split('|')
 183             if 'Tests' in new_line:
 184                 break
 185
 186             logger.info(line)
 187             if 'Testscount' in new_line:
 188                 num_tests = new_line[2]
 189             elif 'Success' in new_line:
 190                 num_success = new_line[2]
 191             elif 'Skipped' in new_line:
 192                 num_skipped = new_line[2]
 193             elif 'Failures' in new_line:
 194                 num_failures = new_line[2]
 195
 196         try:
 197             num_executed = int(num_tests) - int(num_skipped)
 198             try:
 199                 self.result = 100 * int(num_success) / int(num_executed)
 200             except ZeroDivisionError:
 201                 self.result = 0
 202                 if int(num_tests) > 0:
 203                     logger.info("All tests have been skipped")
 204                 else:
 205                     logger.error("No test has been executed")
 206                     return
 207
 208             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 209                                    "tempest.log"), 'r') as logfile:
 210                 output = logfile.read()
 211
 212             success_testcases = []
 213             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 214                 success_testcases.append(match)
 215             failed_testcases = []
 216             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 217                 failed_testcases.append(match)
 218             skipped_testcases = []
 219             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 220                 skipped_testcases.append(match)
 221
 222             self.details = {"tests": int(num_tests),
 223                             "failures": int(num_failures),
 224                             "success": success_testcases,
 225                             "errors": failed_testcases,
 226                             "skipped": skipped_testcases}
 227         except Exception:
 228             self.result = 0
 229
 230         logger.info("Tempest %s success_rate is %s%%"
 231                     % (self.case_name, self.result))
 232
 233     def run(self):
 234
 235         self.start_time = time.time()
 236         try:
 237             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 238                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 239             resources = self.resources.create()
 240             compute_cnt = snaps_utils.get_active_compute_cnt(
 241                 self.resources.os_creds)
 242             conf_utils.configure_tempest(
 243                 self.DEPLOYMENT_DIR,
 244                 image_id=resources.get("image_id"),
 245                 flavor_id=resources.get("flavor_id"),
 246                 compute_cnt=compute_cnt)
 247             self.generate_test_list(self.VERIFIER_REPO_DIR)
 248             self.apply_tempest_blacklist()
 249             self.run_verifier_tests()
 250             self.parse_verifier_result()
 251             res = testcase.TestCase.EX_OK
 252         except Exception as e:
 253             logger.error('Error with run: %s' % e)
 254             res = testcase.TestCase.EX_RUN_ERROR
 255         finally:
 256             self.resources.cleanup()
 257
 258         self.stop_time = time.time()
 259         return res
 260
 261
 262 class TempestSmokeSerial(TempestCommon):
 263
 264     def __init__(self, **kwargs):
 265         if "case_name" not in kwargs:
 266             kwargs["case_name"] = 'tempest_smoke_serial'
 267         TempestCommon.__init__(self, **kwargs)
 268         self.MODE = "smoke"
 269         self.OPTION = "--concurrency 1"
 270
 271
 272 class TempestSmokeParallel(TempestCommon):
 273
 274     def __init__(self, **kwargs):
 275         if "case_name" not in kwargs:
 276             kwargs["case_name"] = 'tempest_smoke_parallel'
 277         TempestCommon.__init__(self, **kwargs)
 278         self.MODE = "smoke"
 279         self.OPTION = ""
 280
 281
 282 class TempestFullParallel(TempestCommon):
 283
 284     def __init__(self, **kwargs):
 285         if "case_name" not in kwargs:
 286             kwargs["case_name"] = 'tempest_full_parallel'
 287         TempestCommon.__init__(self, **kwargs)
 288         self.MODE = "full"
 289
 290
 291 class TempestCustom(TempestCommon):
 292
 293     def __init__(self, **kwargs):
 294         if "case_name" not in kwargs:
 295             kwargs["case_name"] = 'tempest_custom'
 296         TempestCommon.__init__(self, **kwargs)
 297         self.MODE = "custom"
 298         self.OPTION = "--concurrency 1"
 299
 300
 301 class TempestDefcore(TempestCommon):
 302
 303     def __init__(self, **kwargs):
 304         if "case_name" not in kwargs:
 305             kwargs["case_name"] = 'tempest_defcore'
 306         TempestCommon.__init__(self, **kwargs)
 307         self.MODE = "defcore"
 308         self.OPTION = "--concurrency 1"
 309
 310
 311 class TempestResourcesManager(object):
 312
 313     def __init__(self, **kwargs):
 314         self.os_creds = None
 315         if 'os_creds' in kwargs:
 316             self.os_creds = kwargs['os_creds']
 317         else:
 318             self.os_creds = openstack_tests.get_credentials(
 319                 os_env_file=CONST.__getattribute__('openstack_creds'))
 320
 321         self.guid = '-' + str(uuid.uuid4())
 322
 323         self.creators = list()
 324
 325         if hasattr(CONST, 'snaps_images_cirros'):
 326             self.cirros_image_config = CONST.__getattribute__(
 327                 'snaps_images_cirros')
 328         else:
 329             self.cirros_image_config = None
 330
 331     def create(self, use_custom_images=False, use_custom_flavors=False,
 332                create_project=False):
 333         if create_project:
 334             logger.debug("Creating project (tenant) for Tempest suite")
 335             project_name = CONST.__getattribute__(
 336                 'tempest_identity_tenant_name') + self.guid
 337             project_creator = deploy_utils.create_project(
 338                 self.os_creds, ProjectConfig(
 339                     name=project_name,
 340                     description=CONST.__getattribute__(
 341                         'tempest_identity_tenant_description')))
 342             if (project_creator is None or
 343                     project_creator.get_project() is None):
 344                 raise Exception("Failed to create tenant")
 345             project_id = project_creator.get_project().id
 346             self.creators.append(project_creator)
 347
 348             logger.debug("Creating user for Tempest suite")
 349             user_creator = deploy_utils.create_user(
 350                 self.os_creds, UserConfig(
 351                     name=CONST.__getattribute__(
 352                         'tempest_identity_user_name') + self.guid,
 353                     password=CONST.__getattribute__(
 354                         'tempest_identity_user_password'),
 355                     project_name=project_name))
 356             if user_creator is None or user_creator.get_user() is None:
 357                 raise Exception("Failed to create user")
 358             user_id = user_creator.get_user().id
 359             self.creators.append(user_creator)
 360         else:
 361             project_name = None
 362             project_id = None
 363             user_id = None
 364
 365         logger.debug("Creating private network for Tempest suite")
 366
 367         tempest_network_type = None
 368         tempest_physical_network = None
 369         tempest_segmentation_id = None
 370
 371         if hasattr(CONST, 'tempest_network_type'):
 372             tempest_network_type = CONST.__getattribute__(
 373                 'tempest_network_type')
 374         if hasattr(CONST, 'tempest_physical_network'):
 375             tempest_physical_network = CONST.__getattribute__(
 376                 'tempest_physical_network')
 377         if hasattr(CONST, 'tempest_segmentation_id'):
 378             tempest_segmentation_id = CONST.__getattribute__(
 379                 'tempest_segmentation_id')
 380
 381         network_creator = deploy_utils.create_network(
 382             self.os_creds, NetworkConfig(
 383                 name=CONST.__getattribute__(
 384                     'tempest_private_net_name') + self.guid,
 385                 project_name=project_name,
 386                 network_type=tempest_network_type,
 387                 physical_network=tempest_physical_network,
 388                 segmentation_id=tempest_segmentation_id,
 389                 subnet_settings=[SubnetConfig(
 390                     name=CONST.__getattribute__(
 391                         'tempest_private_subnet_name') + self.guid,
 392                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 393                 ]))
 394         if network_creator is None or network_creator.get_network() is None:
 395             raise Exception("Failed to create private network")
 396         self.creators.append(network_creator)
 397
 398         image_id = None
 399         image_id_alt = None
 400         flavor_id = None
 401         flavor_id_alt = None
 402
 403         logger.debug("Creating image for Tempest suite")
 404         image_base_name = CONST.__getattribute__(
 405             'openstack_image_name') + self.guid
 406         os_image_settings = openstack_tests.cirros_image_settings(
 407             image_base_name, public=True,
 408             image_metadata=self.cirros_image_config)
 409         logger.debug("Creating image for Tempest suite")
 410         image_creator = deploy_utils.create_image(
 411             self.os_creds, os_image_settings)
 412         if image_creator is None:
 413             raise Exception('Failed to create image')
 414         self.creators.append(image_creator)
 415         image_id = image_creator.get_image().id
 416
 417         if use_custom_images:
 418             logger.debug("Creating 2nd image for Tempest suite")
 419             image_base_name_alt = CONST.__getattribute__(
 420                 'openstack_image_name_alt') + self.guid
 421             os_image_settings_alt = openstack_tests.cirros_image_settings(
 422                 image_base_name_alt, public=True,
 423                 image_metadata=self.cirros_image_config)
 424             logger.debug("Creating 2nd image for Tempest suite")
 425             image_creator_alt = deploy_utils.create_image(
 426                 self.os_creds, os_image_settings_alt)
 427             if image_creator_alt is None:
 428                 raise Exception('Failed to create image')
 429             self.creators.append(image_creator_alt)
 430             image_id_alt = image_creator_alt.get_image().id
 431
 432         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 433            use_custom_flavors):
 434             logger.info("Creating flavor for Tempest suite")
 435             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 436             flavor_metadata = None
 437             if 'ovs' in scenario or 'fdio' in scenario:
 438                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 439             flavor_creator = OpenStackFlavor(
 440                 self.os_creds, FlavorConfig(
 441                     name=CONST.__getattribute__(
 442                         'openstack_flavor_name') + self.guid,
 443                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 444                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 445                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 446                     metadata=flavor_metadata))
 447             flavor = flavor_creator.create()
 448             if flavor is None:
 449                 raise Exception('Failed to create flavor')
 450             self.creators.append(flavor_creator)
 451             flavor_id = flavor.id
 452
 453         if use_custom_flavors:
 454             logger.info("Creating 2nd flavor for Tempest suite")
 455             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 456             flavor_metadata_alt = None
 457             if 'ovs' in scenario or 'fdio' in scenario:
 458                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 459                 CONST.__setattr__('openstack_flavor_ram', 1024)
 460             flavor_creator_alt = OpenStackFlavor(
 461                 self.os_creds, FlavorConfig(
 462                     name=CONST.__getattribute__(
 463                         'openstack_flavor_name_alt') + self.guid,
 464                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 465                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 466                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 467                     metadata=flavor_metadata_alt))
 468             flavor_alt = flavor_creator_alt.create()
 469             if flavor_alt is None:
 470                 raise Exception('Failed to create flavor')
 471             self.creators.append(flavor_creator_alt)
 472             flavor_id_alt = flavor_alt.id
 473
 474         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 475               "flavor_id: %s, flavor_id_alt: %s" % (
 476                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 477
 478         result = {
 479             'image_id': image_id,
 480             'image_id_alt': image_id_alt,
 481             'flavor_id': flavor_id,
 482             'flavor_id_alt': flavor_id_alt
 483         }
 484
 485         if create_project:
 486             result['project_id'] = project_id
 487             result['tenant_id'] = project_id  # for compatibility
 488             result['user_id'] = user_id
 489
 490         return result
 491
 492     def cleanup(self):
 493         """
 494         Cleanup all OpenStack objects. Should be called on completion.
 495         """
 496         for creator in reversed(self.creators):
 497             try:
 498                 creator.clean()
 499             except Exception as e:
 500                 logger.error('Unexpected error cleaning - %s', e)