functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19 import uuid
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.opnfv_tests.openstack.tempest import conf_utils
  26 from functest.utils.constants import CONST
  27 import functest.utils.functest_utils as ft_utils
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.network import NetworkConfig, SubnetConfig
  31 from snaps.config.project import ProjectConfig
  32 from snaps.config.user import UserConfig
  33
  34 from snaps.openstack import create_flavor
  35 from snaps.openstack.create_flavor import OpenStackFlavor
  36 from snaps.openstack.tests import openstack_tests
  37 from snaps.openstack.utils import deploy_utils
  38
  39
  40 """ logging configuration """
  41 logger = logging.getLogger(__name__)
  42
  43
  44 class TempestCommon(testcase.TestCase):
  45
  46     def __init__(self, **kwargs):
  47         super(TempestCommon, self).__init__(**kwargs)
  48         self.resources = TempestResourcesManager(**kwargs)
  49         self.MODE = ""
  50         self.OPTION = []
  51         self.VERIFIER_ID = conf_utils.get_verifier_id()
  52         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  53             self.VERIFIER_ID)
  54         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  55         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  56             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  57         self.VERIFICATION_ID = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         with open(filename) as src:
  62             return [line.strip() for line in src.readlines()]
  63
  64     def generate_test_list(self, verifier_repo_dir):
  65         logger.debug("Generating test case list...")
  66         if self.MODE == 'defcore':
  67             shutil.copyfile(
  68                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  69         elif self.MODE == 'custom':
  70             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  71                 shutil.copyfile(
  72                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  73             else:
  74                 raise Exception("Tempest test list file %s NOT found."
  75                                 % conf_utils.TEMPEST_CUSTOM)
  76         else:
  77             if self.MODE == 'smoke':
  78                 testr_mode = "smoke"
  79             elif self.MODE == 'full':
  80                 testr_mode = ""
  81             else:
  82                 testr_mode = 'tempest.api.' + self.MODE
  83             cmd = ("cd {0};"
  84                    "testr list-tests {1} > {2};"
  85                    "cd -;".format(verifier_repo_dir,
  86                                   testr_mode,
  87                                   conf_utils.TEMPEST_RAW_LIST))
  88             ft_utils.execute_command(cmd)
  89
  90     def apply_tempest_blacklist(self):
  91         logger.debug("Applying tempest blacklist...")
  92         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  93         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  94         black_tests = []
  95         try:
  96             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  97             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  98             if (bool(installer_type) * bool(deploy_scenario)):
  99                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 100                 # file
 101                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 102                 black_list_yaml = yaml.safe_load(black_list_file)
 103                 black_list_file.close()
 104                 for item in black_list_yaml:
 105                     scenarios = item['scenarios']
 106                     installers = item['installers']
 107                     if (deploy_scenario in scenarios and
 108                             installer_type in installers):
 109                         tests = item['tests']
 110                         for test in tests:
 111                             black_tests.append(test)
 112                         break
 113         except Exception:
 114             black_tests = []
 115             logger.debug("Tempest blacklist file does not exist.")
 116
 117         for cases_line in cases_file:
 118             for black_tests_line in black_tests:
 119                 if black_tests_line in cases_line:
 120                     break
 121             else:
 122                 result_file.write(str(cases_line) + '\n')
 123         result_file.close()
 124
 125     def run_verifier_tests(self):
 126         cmd = ["rally", "verify", "start", "--load-list",
 127                conf_utils.TEMPEST_LIST]
 128         cmd.extend(self.OPTION)
 129         logger.info("Starting Tempest test suite: '%s'." % cmd)
 130
 131         header = ("Tempest environment:\n"
 132                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 133                   (CONST.__getattribute__('INSTALLER_TYPE'),
 134                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 135                    CONST.__getattribute__('NODE_NAME'),
 136                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 137
 138         f_stdout = open(
 139             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 140         f_stderr = open(
 141             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 142                          "tempest-error.log"), 'w+')
 143         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 144                                   "environment.log"), 'w+')
 145         f_env.write(header)
 146
 147         p = subprocess.Popen(
 148             cmd,
 149             stdout=subprocess.PIPE,
 150             stderr=f_stderr,
 151             bufsize=1)
 152
 153         with p.stdout:
 154             for line in iter(p.stdout.readline, b''):
 155                 if re.search("\} tempest\.", line):
 156                     logger.info(line.replace('\n', ''))
 157                 elif re.search('Starting verification', line):
 158                     logger.info(line.replace('\n', ''))
 159                     first_pos = line.index("UUID=") + len("UUID=")
 160                     last_pos = line.index(") for deployment")
 161                     self.VERIFICATION_ID = line[first_pos:last_pos]
 162                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 163                 f_stdout.write(line)
 164         p.wait()
 165
 166         f_stdout.close()
 167         f_stderr.close()
 168         f_env.close()
 169
 170     def parse_verifier_result(self):
 171         if self.VERIFICATION_ID is None:
 172             raise Exception('Verification UUID not found')
 173
 174         cmd = ["rally", "verify", "show", "--uuid", self.VERIFICATION_ID]
 175         logger.info("Showing result for a verification: '%s'." % cmd)
 176         p = subprocess.Popen(cmd,
 177                              stdout=subprocess.PIPE,
 178                              stderr=subprocess.STDOUT)
 179         for line in p.stdout:
 180             new_line = line.replace(' ', '').split('|')
 181             if 'Tests' in new_line:
 182                 break
 183
 184             logger.info(line)
 185             if 'Testscount' in new_line:
 186                 num_tests = new_line[2]
 187             elif 'Success' in new_line:
 188                 num_success = new_line[2]
 189             elif 'Skipped' in new_line:
 190                 num_skipped = new_line[2]
 191             elif 'Failures' in new_line:
 192                 num_failures = new_line[2]
 193
 194         try:
 195             num_executed = int(num_tests) - int(num_skipped)
 196             try:
 197                 self.result = 100 * int(num_success) / int(num_executed)
 198             except ZeroDivisionError:
 199                 self.result = 0
 200                 if int(num_tests) > 0:
 201                     logger.info("All tests have been skipped")
 202                 else:
 203                     logger.error("No test has been executed")
 204                     return
 205
 206             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 207                                    "tempest.log"), 'r') as logfile:
 208                 output = logfile.read()
 209
 210             success_testcases = []
 211             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 212                 success_testcases.append(match)
 213             failed_testcases = []
 214             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 215                 failed_testcases.append(match)
 216             skipped_testcases = []
 217             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 218                 skipped_testcases.append(match)
 219
 220             self.details = {"tests": int(num_tests),
 221                             "failures": int(num_failures),
 222                             "success": success_testcases,
 223                             "errors": failed_testcases,
 224                             "skipped": skipped_testcases}
 225         except Exception:
 226             self.result = 0
 227
 228         logger.info("Tempest %s success_rate is %s%%"
 229                     % (self.case_name, self.result))
 230
 231     def run(self):
 232
 233         self.start_time = time.time()
 234         try:
 235             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 236                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 237             resources = self.resources.create()
 238             compute_cnt = snaps_utils.get_active_compute_cnt(
 239                 self.resources.os_creds)
 240             conf_utils.configure_tempest(
 241                 self.DEPLOYMENT_DIR,
 242                 image_id=resources.get("image_id"),
 243                 flavor_id=resources.get("flavor_id"),
 244                 compute_cnt=compute_cnt)
 245             self.generate_test_list(self.VERIFIER_REPO_DIR)
 246             self.apply_tempest_blacklist()
 247             self.run_verifier_tests()
 248             self.parse_verifier_result()
 249             res = testcase.TestCase.EX_OK
 250         except Exception as e:
 251             logger.error('Error with run: %s' % e)
 252             res = testcase.TestCase.EX_RUN_ERROR
 253         finally:
 254             self.resources.cleanup()
 255
 256         self.stop_time = time.time()
 257         return res
 258
 259
 260 class TempestSmokeSerial(TempestCommon):
 261
 262     def __init__(self, **kwargs):
 263         if "case_name" not in kwargs:
 264             kwargs["case_name"] = 'tempest_smoke_serial'
 265         TempestCommon.__init__(self, **kwargs)
 266         self.MODE = "smoke"
 267         self.OPTION = ["--concurrency", "1"]
 268
 269
 270 class TempestSmokeParallel(TempestCommon):
 271
 272     def __init__(self, **kwargs):
 273         if "case_name" not in kwargs:
 274             kwargs["case_name"] = 'tempest_smoke_parallel'
 275         TempestCommon.__init__(self, **kwargs)
 276         self.MODE = "smoke"
 277
 278
 279 class TempestFullParallel(TempestCommon):
 280
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_full_parallel'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.MODE = "full"
 286
 287
 288 class TempestCustom(TempestCommon):
 289
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_custom'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.MODE = "custom"
 295         self.OPTION = ["--concurrency", "1"]
 296
 297
 298 class TempestDefcore(TempestCommon):
 299
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'tempest_defcore'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.MODE = "defcore"
 305         self.OPTION = ["--concurrency", "1"]
 306
 307
 308 class TempestResourcesManager(object):
 309
 310     def __init__(self, **kwargs):
 311         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 312         self.guid = '-' + str(uuid.uuid4())
 313         self.creators = list()
 314
 315         if hasattr(CONST, 'snaps_images_cirros'):
 316             self.cirros_image_config = CONST.__getattribute__(
 317                 'snaps_images_cirros')
 318         else:
 319             self.cirros_image_config = None
 320
 321     def create(self, use_custom_images=False, use_custom_flavors=False,
 322                create_project=False):
 323         if create_project:
 324             logger.debug("Creating project (tenant) for Tempest suite")
 325             project_name = CONST.__getattribute__(
 326                 'tempest_identity_tenant_name') + self.guid
 327             project_creator = deploy_utils.create_project(
 328                 self.os_creds, ProjectConfig(
 329                     name=project_name,
 330                     description=CONST.__getattribute__(
 331                         'tempest_identity_tenant_description')))
 332             if (project_creator is None or
 333                     project_creator.get_project() is None):
 334                 raise Exception("Failed to create tenant")
 335             project_id = project_creator.get_project().id
 336             self.creators.append(project_creator)
 337
 338             logger.debug("Creating user for Tempest suite")
 339             user_creator = deploy_utils.create_user(
 340                 self.os_creds, UserConfig(
 341                     name=CONST.__getattribute__(
 342                         'tempest_identity_user_name') + self.guid,
 343                     password=CONST.__getattribute__(
 344                         'tempest_identity_user_password'),
 345                     project_name=project_name))
 346             if user_creator is None or user_creator.get_user() is None:
 347                 raise Exception("Failed to create user")
 348             user_id = user_creator.get_user().id
 349             self.creators.append(user_creator)
 350         else:
 351             project_name = None
 352             project_id = None
 353             user_id = None
 354
 355         logger.debug("Creating private network for Tempest suite")
 356
 357         tempest_network_type = None
 358         tempest_physical_network = None
 359         tempest_segmentation_id = None
 360
 361         if hasattr(CONST, 'tempest_network_type'):
 362             tempest_network_type = CONST.__getattribute__(
 363                 'tempest_network_type')
 364         if hasattr(CONST, 'tempest_physical_network'):
 365             tempest_physical_network = CONST.__getattribute__(
 366                 'tempest_physical_network')
 367         if hasattr(CONST, 'tempest_segmentation_id'):
 368             tempest_segmentation_id = CONST.__getattribute__(
 369                 'tempest_segmentation_id')
 370
 371         network_creator = deploy_utils.create_network(
 372             self.os_creds, NetworkConfig(
 373                 name=CONST.__getattribute__(
 374                     'tempest_private_net_name') + self.guid,
 375                 project_name=project_name,
 376                 network_type=tempest_network_type,
 377                 physical_network=tempest_physical_network,
 378                 segmentation_id=tempest_segmentation_id,
 379                 subnet_settings=[SubnetConfig(
 380                     name=CONST.__getattribute__(
 381                         'tempest_private_subnet_name') + self.guid,
 382                     project_name=project_name,
 383                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 384                 ]))
 385         if network_creator is None or network_creator.get_network() is None:
 386             raise Exception("Failed to create private network")
 387         self.creators.append(network_creator)
 388
 389         image_id = None
 390         image_id_alt = None
 391         flavor_id = None
 392         flavor_id_alt = None
 393
 394         logger.debug("Creating image for Tempest suite")
 395         image_base_name = CONST.__getattribute__(
 396             'openstack_image_name') + self.guid
 397         os_image_settings = openstack_tests.cirros_image_settings(
 398             image_base_name, public=True,
 399             image_metadata=self.cirros_image_config)
 400         logger.debug("Creating image for Tempest suite")
 401         image_creator = deploy_utils.create_image(
 402             self.os_creds, os_image_settings)
 403         if image_creator is None:
 404             raise Exception('Failed to create image')
 405         self.creators.append(image_creator)
 406         image_id = image_creator.get_image().id
 407
 408         if use_custom_images:
 409             logger.debug("Creating 2nd image for Tempest suite")
 410             image_base_name_alt = CONST.__getattribute__(
 411                 'openstack_image_name_alt') + self.guid
 412             os_image_settings_alt = openstack_tests.cirros_image_settings(
 413                 image_base_name_alt, public=True,
 414                 image_metadata=self.cirros_image_config)
 415             logger.debug("Creating 2nd image for Tempest suite")
 416             image_creator_alt = deploy_utils.create_image(
 417                 self.os_creds, os_image_settings_alt)
 418             if image_creator_alt is None:
 419                 raise Exception('Failed to create image')
 420             self.creators.append(image_creator_alt)
 421             image_id_alt = image_creator_alt.get_image().id
 422
 423         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 424            use_custom_flavors):
 425             logger.info("Creating flavor for Tempest suite")
 426             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 427             flavor_metadata = None
 428             if 'ovs' in scenario or 'fdio' in scenario:
 429                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 430             flavor_creator = OpenStackFlavor(
 431                 self.os_creds, FlavorConfig(
 432                     name=CONST.__getattribute__(
 433                         'openstack_flavor_name') + self.guid,
 434                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 435                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 436                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 437                     metadata=flavor_metadata))
 438             flavor = flavor_creator.create()
 439             if flavor is None:
 440                 raise Exception('Failed to create flavor')
 441             self.creators.append(flavor_creator)
 442             flavor_id = flavor.id
 443
 444         if use_custom_flavors:
 445             logger.info("Creating 2nd flavor for Tempest suite")
 446             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 447             flavor_metadata_alt = None
 448             if 'ovs' in scenario or 'fdio' in scenario:
 449                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 450                 CONST.__setattr__('openstack_flavor_ram', 1024)
 451             flavor_creator_alt = OpenStackFlavor(
 452                 self.os_creds, FlavorConfig(
 453                     name=CONST.__getattribute__(
 454                         'openstack_flavor_name_alt') + self.guid,
 455                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 456                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 457                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 458                     metadata=flavor_metadata_alt))
 459             flavor_alt = flavor_creator_alt.create()
 460             if flavor_alt is None:
 461                 raise Exception('Failed to create flavor')
 462             self.creators.append(flavor_creator_alt)
 463             flavor_id_alt = flavor_alt.id
 464
 465         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 466               "flavor_id: %s, flavor_id_alt: %s" % (
 467                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 468
 469         result = {
 470             'image_id': image_id,
 471             'image_id_alt': image_id_alt,
 472             'flavor_id': flavor_id,
 473             'flavor_id_alt': flavor_id_alt
 474         }
 475
 476         if create_project:
 477             result['project_id'] = project_id
 478             result['tenant_id'] = project_id  # for compatibility
 479             result['user_id'] = user_id
 480
 481         return result
 482
 483     def cleanup(self):
 484         """
 485         Cleanup all OpenStack objects. Should be called on completion.
 486         """
 487         for creator in reversed(self.creators):
 488             try:
 489                 creator.clean()
 490             except Exception as e:
 491                 logger.error('Unexpected error cleaning - %s', e)