functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19 import uuid
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.opnfv_tests.openstack.tempest import conf_utils
  26 from functest.utils.constants import CONST
  27 import functest.utils.functest_utils as ft_utils
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.network import NetworkConfig, SubnetConfig
  31 from snaps.config.project import ProjectConfig
  32 from snaps.config.user import UserConfig
  33
  34 from snaps.openstack import create_flavor
  35 from snaps.openstack.create_flavor import OpenStackFlavor
  36 from snaps.openstack.tests import openstack_tests
  37 from snaps.openstack.utils import deploy_utils
  38
  39
  40 """ logging configuration """
  41 logger = logging.getLogger(__name__)
  42
  43
  44 class TempestCommon(testcase.TestCase):
  45
  46     def __init__(self, **kwargs):
  47         super(TempestCommon, self).__init__(**kwargs)
  48         self.resources = TempestResourcesManager(**kwargs)
  49         self.MODE = ""
  50         self.OPTION = ""
  51         self.VERIFIER_ID = conf_utils.get_verifier_id()
  52         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  53             self.VERIFIER_ID)
  54         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  55         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  56             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  57         self.VERIFICATION_ID = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     def generate_test_list(self, verifier_repo_dir):
  65         logger.debug("Generating test case list...")
  66         if self.MODE == 'defcore':
  67             shutil.copyfile(
  68                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  69         elif self.MODE == 'custom':
  70             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  71                 shutil.copyfile(
  72                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  73             else:
  74                 raise Exception("Tempest test list file %s NOT found."
  75                                 % conf_utils.TEMPEST_CUSTOM)
  76         else:
  77             if self.MODE == 'smoke':
  78                 testr_mode = "smoke"
  79             elif self.MODE == 'full':
  80                 testr_mode = ""
  81             else:
  82                 testr_mode = 'tempest.api.' + self.MODE
  83             cmd = ("cd {0};"
  84                    "testr list-tests {1} > {2};"
  85                    "cd -;".format(verifier_repo_dir,
  86                                   testr_mode,
  87                                   conf_utils.TEMPEST_RAW_LIST))
  88             ft_utils.execute_command(cmd)
  89
  90     def apply_tempest_blacklist(self):
  91         logger.debug("Applying tempest blacklist...")
  92         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  93         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  94         black_tests = []
  95         try:
  96             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  97             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  98             if (bool(installer_type) * bool(deploy_scenario)):
  99                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 100                 # file
 101                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 102                 black_list_yaml = yaml.safe_load(black_list_file)
 103                 black_list_file.close()
 104                 for item in black_list_yaml:
 105                     scenarios = item['scenarios']
 106                     installers = item['installers']
 107                     if (deploy_scenario in scenarios and
 108                             installer_type in installers):
 109                         tests = item['tests']
 110                         for test in tests:
 111                             black_tests.append(test)
 112                         break
 113         except Exception:
 114             black_tests = []
 115             logger.debug("Tempest blacklist file does not exist.")
 116
 117         for cases_line in cases_file:
 118             for black_tests_line in black_tests:
 119                 if black_tests_line in cases_line:
 120                     break
 121             else:
 122                 result_file.write(str(cases_line) + '\n')
 123         result_file.close()
 124
 125     def run_verifier_tests(self):
 126         self.OPTION += (" --load-list {} --detailed"
 127                         .format(conf_utils.TEMPEST_LIST))
 128
 129         cmd_line = "rally verify start " + self.OPTION
 130         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 131
 132         header = ("Tempest environment:\n"
 133                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 134                   (CONST.__getattribute__('INSTALLER_TYPE'),
 135                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 136                    CONST.__getattribute__('NODE_NAME'),
 137                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 138
 139         f_stdout = open(
 140             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 141         f_stderr = open(
 142             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 143                          "tempest-error.log"), 'w+')
 144         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 145                                   "environment.log"), 'w+')
 146         f_env.write(header)
 147
 148         p = subprocess.Popen(
 149             cmd_line, shell=True,
 150             stdout=subprocess.PIPE,
 151             stderr=f_stderr,
 152             bufsize=1)
 153
 154         with p.stdout:
 155             for line in iter(p.stdout.readline, b''):
 156                 if re.search("\} tempest\.", line):
 157                     logger.info(line.replace('\n', ''))
 158                 elif re.search('Starting verification', line):
 159                     logger.info(line.replace('\n', ''))
 160                     first_pos = line.index("UUID=") + len("UUID=")
 161                     last_pos = line.index(") for deployment")
 162                     self.VERIFICATION_ID = line[first_pos:last_pos]
 163                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 164                 f_stdout.write(line)
 165         p.wait()
 166
 167         f_stdout.close()
 168         f_stderr.close()
 169         f_env.close()
 170
 171     def parse_verifier_result(self):
 172         if self.VERIFICATION_ID is None:
 173             raise Exception('Verification UUID not found')
 174
 175         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 176         logger.info("Showing result for a verification: '%s'." % cmd_line)
 177         p = subprocess.Popen(cmd_line,
 178                              shell=True,
 179                              stdout=subprocess.PIPE,
 180                              stderr=subprocess.STDOUT)
 181         for line in p.stdout:
 182             new_line = line.replace(' ', '').split('|')
 183             if 'Tests' in new_line:
 184                 break
 185
 186             logger.info(line)
 187             if 'Testscount' in new_line:
 188                 num_tests = new_line[2]
 189             elif 'Success' in new_line:
 190                 num_success = new_line[2]
 191             elif 'Skipped' in new_line:
 192                 num_skipped = new_line[2]
 193             elif 'Failures' in new_line:
 194                 num_failures = new_line[2]
 195
 196         try:
 197             num_executed = int(num_tests) - int(num_skipped)
 198             try:
 199                 self.result = 100 * int(num_success) / int(num_executed)
 200             except ZeroDivisionError:
 201                 self.result = 0
 202                 if int(num_tests) > 0:
 203                     logger.info("All tests have been skipped")
 204                 else:
 205                     logger.error("No test has been executed")
 206                     return
 207
 208             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 209                                    "tempest.log"), 'r') as logfile:
 210                 output = logfile.read()
 211
 212             success_testcases = []
 213             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 214                 success_testcases.append(match)
 215             failed_testcases = []
 216             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 217                 failed_testcases.append(match)
 218             skipped_testcases = []
 219             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 220                 skipped_testcases.append(match)
 221
 222             self.details = {"tests": int(num_tests),
 223                             "failures": int(num_failures),
 224                             "success": success_testcases,
 225                             "errors": failed_testcases,
 226                             "skipped": skipped_testcases}
 227         except Exception:
 228             self.result = 0
 229
 230         logger.info("Tempest %s success_rate is %s%%"
 231                     % (self.case_name, self.result))
 232
 233     def run(self):
 234
 235         self.start_time = time.time()
 236         try:
 237             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 238                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 239             resources = self.resources.create()
 240             compute_cnt = snaps_utils.get_active_compute_cnt(
 241                 self.resources.os_creds)
 242             conf_utils.configure_tempest(
 243                 self.DEPLOYMENT_DIR,
 244                 image_id=resources.get("image_id"),
 245                 flavor_id=resources.get("flavor_id"),
 246                 compute_cnt=compute_cnt)
 247             self.generate_test_list(self.VERIFIER_REPO_DIR)
 248             self.apply_tempest_blacklist()
 249             self.run_verifier_tests()
 250             self.parse_verifier_result()
 251             res = testcase.TestCase.EX_OK
 252         except Exception as e:
 253             logger.error('Error with run: %s' % e)
 254             res = testcase.TestCase.EX_RUN_ERROR
 255         finally:
 256             self.resources.cleanup()
 257
 258         self.stop_time = time.time()
 259         return res
 260
 261
 262 class TempestSmokeSerial(TempestCommon):
 263
 264     def __init__(self, **kwargs):
 265         if "case_name" not in kwargs:
 266             kwargs["case_name"] = 'tempest_smoke_serial'
 267         TempestCommon.__init__(self, **kwargs)
 268         self.MODE = "smoke"
 269         self.OPTION = "--concurrency 1"
 270
 271
 272 class TempestSmokeParallel(TempestCommon):
 273
 274     def __init__(self, **kwargs):
 275         if "case_name" not in kwargs:
 276             kwargs["case_name"] = 'tempest_smoke_parallel'
 277         TempestCommon.__init__(self, **kwargs)
 278         self.MODE = "smoke"
 279         self.OPTION = ""
 280
 281
 282 class TempestFullParallel(TempestCommon):
 283
 284     def __init__(self, **kwargs):
 285         if "case_name" not in kwargs:
 286             kwargs["case_name"] = 'tempest_full_parallel'
 287         TempestCommon.__init__(self, **kwargs)
 288         self.MODE = "full"
 289
 290
 291 class TempestCustom(TempestCommon):
 292
 293     def __init__(self, **kwargs):
 294         if "case_name" not in kwargs:
 295             kwargs["case_name"] = 'tempest_custom'
 296         TempestCommon.__init__(self, **kwargs)
 297         self.MODE = "custom"
 298         self.OPTION = "--concurrency 1"
 299
 300
 301 class TempestDefcore(TempestCommon):
 302
 303     def __init__(self, **kwargs):
 304         if "case_name" not in kwargs:
 305             kwargs["case_name"] = 'tempest_defcore'
 306         TempestCommon.__init__(self, **kwargs)
 307         self.MODE = "defcore"
 308         self.OPTION = "--concurrency 1"
 309
 310
 311 class TempestResourcesManager(object):
 312
 313     def __init__(self, **kwargs):
 314         self.os_creds = None
 315         if 'os_creds' in kwargs:
 316             self.os_creds = kwargs['os_creds']
 317         else:
 318             self.os_creds = openstack_tests.get_credentials(
 319                 os_env_file=CONST.__getattribute__('openstack_creds'))
 320
 321         self.guid = ''
 322         if CONST.__getattribute__('tempest_unique_names'):
 323             self.guid = '-' + str(uuid.uuid4())
 324
 325         self.creators = list()
 326
 327         if hasattr(CONST, 'snaps_images_cirros'):
 328             self.cirros_image_config = CONST.__getattribute__(
 329                 'snaps_images_cirros')
 330         else:
 331             self.cirros_image_config = None
 332
 333     def create(self, use_custom_images=False, use_custom_flavors=False,
 334                create_project=False):
 335         if create_project:
 336             logger.debug("Creating project (tenant) for Tempest suite")
 337             project_name = CONST.__getattribute__(
 338                 'tempest_identity_tenant_name') + self.guid
 339             project_creator = deploy_utils.create_project(
 340                 self.os_creds, ProjectConfig(
 341                     name=project_name,
 342                     description=CONST.__getattribute__(
 343                         'tempest_identity_tenant_description')))
 344             if (project_creator is None or
 345                     project_creator.get_project() is None):
 346                 raise Exception("Failed to create tenant")
 347             project_id = project_creator.get_project().id
 348             self.creators.append(project_creator)
 349
 350             logger.debug("Creating user for Tempest suite")
 351             user_creator = deploy_utils.create_user(
 352                 self.os_creds, UserConfig(
 353                     name=CONST.__getattribute__(
 354                         'tempest_identity_user_name') + self.guid,
 355                     password=CONST.__getattribute__(
 356                         'tempest_identity_user_password'),
 357                     project_name=project_name))
 358             if user_creator is None or user_creator.get_user() is None:
 359                 raise Exception("Failed to create user")
 360             user_id = user_creator.get_user().id
 361             self.creators.append(user_creator)
 362         else:
 363             project_name = None
 364             project_id = None
 365             user_id = None
 366
 367         logger.debug("Creating private network for Tempest suite")
 368         network_creator = deploy_utils.create_network(
 369             self.os_creds, NetworkConfig(
 370                 name=CONST.__getattribute__(
 371                     'tempest_private_net_name') + self.guid,
 372                 project_name=project_name,
 373                 subnet_settings=[SubnetConfig(
 374                     name=CONST.__getattribute__(
 375                         'tempest_private_subnet_name') + self.guid,
 376                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 377                 ]))
 378         if network_creator is None or network_creator.get_network() is None:
 379             raise Exception("Failed to create private network")
 380         self.creators.append(network_creator)
 381
 382         image_id = None
 383         image_id_alt = None
 384         flavor_id = None
 385         flavor_id_alt = None
 386
 387         if (CONST.__getattribute__('tempest_use_custom_images') or
 388            use_custom_images):
 389             logger.debug("Creating image for Tempest suite")
 390             image_base_name = CONST.__getattribute__(
 391                 'openstack_image_name') + self.guid
 392             os_image_settings = openstack_tests.cirros_image_settings(
 393                 image_base_name, public=True,
 394                 image_metadata=self.cirros_image_config)
 395             logger.debug("Creating image for Tempest suite")
 396             image_creator = deploy_utils.create_image(
 397                 self.os_creds, os_image_settings)
 398             if image_creator is None:
 399                 raise Exception('Failed to create image')
 400             self.creators.append(image_creator)
 401             image_id = image_creator.get_image().id
 402
 403         if use_custom_images:
 404             logger.debug("Creating 2nd image for Tempest suite")
 405             image_base_name_alt = CONST.__getattribute__(
 406                 'openstack_image_name_alt') + self.guid
 407             os_image_settings_alt = openstack_tests.cirros_image_settings(
 408                 image_base_name_alt, public=True,
 409                 image_metadata=self.cirros_image_config)
 410             logger.debug("Creating 2nd image for Tempest suite")
 411             image_creator_alt = deploy_utils.create_image(
 412                 self.os_creds, os_image_settings_alt)
 413             if image_creator_alt is None:
 414                 raise Exception('Failed to create image')
 415             self.creators.append(image_creator_alt)
 416             image_id_alt = image_creator_alt.get_image().id
 417
 418         if (CONST.__getattribute__('tempest_use_custom_flavors') or
 419            use_custom_flavors):
 420             logger.info("Creating flavor for Tempest suite")
 421             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 422             flavor_metadata = None
 423             if 'ovs' in scenario or 'fdio' in scenario:
 424                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 425             flavor_creator = OpenStackFlavor(
 426                 self.os_creds, FlavorConfig(
 427                     name=CONST.__getattribute__(
 428                         'openstack_flavor_name') + self.guid,
 429                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 430                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 431                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 432                     metadata=flavor_metadata))
 433             flavor = flavor_creator.create()
 434             if flavor is None:
 435                 raise Exception('Failed to create flavor')
 436             self.creators.append(flavor_creator)
 437             flavor_id = flavor.id
 438
 439         if use_custom_flavors:
 440             logger.info("Creating 2nd flavor for Tempest suite")
 441             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 442             flavor_metadata_alt = None
 443             if 'ovs' in scenario or 'fdio' in scenario:
 444                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 445             flavor_creator_alt = OpenStackFlavor(
 446                 self.os_creds, FlavorConfig(
 447                     name=CONST.__getattribute__(
 448                         'openstack_flavor_name_alt') + self.guid,
 449                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 450                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 451                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 452                     metadata=flavor_metadata_alt))
 453             flavor_alt = flavor_creator_alt.create()
 454             if flavor_alt is None:
 455                 raise Exception('Failed to create flavor')
 456             self.creators.append(flavor_creator_alt)
 457             flavor_id_alt = flavor_alt.id
 458
 459         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 460               "flavor_id: %s, flavor_id_alt: %s" % (
 461                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 462
 463         result = {
 464             'image_id': image_id,
 465             'image_id_alt': image_id_alt,
 466             'flavor_id': flavor_id,
 467             'flavor_id_alt': flavor_id_alt
 468         }
 469
 470         if create_project:
 471             result['project_id'] = project_id
 472             result['tenant_id'] = project_id  # for compatibility
 473             result['user_id'] = user_id
 474
 475         return result
 476
 477     def cleanup(self):
 478         """
 479         Cleanup all OpenStack objects. Should be called on completion.
 480         """
 481         for creator in reversed(self.creators):
 482             try:
 483                 creator.clean()
 484             except Exception as e:
 485                 logger.error('Unexpected error cleaning - %s', e)