functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import logging
  14 import os
  15 import re
  16 import shutil
  17 import subprocess
  18 import time
  19 import uuid
  20
  21 import yaml
  22
  23 from functest.core import testcase
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.opnfv_tests.openstack.tempest import conf_utils
  26 from functest.utils.constants import CONST
  27 import functest.utils.functest_utils as ft_utils
  28
  29 from snaps.openstack import create_flavor
  30 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  31 from snaps.openstack.create_project import ProjectSettings
  32 from snaps.openstack.create_network import NetworkSettings, SubnetSettings
  33 from snaps.openstack.create_user import UserSettings
  34 from snaps.openstack.tests import openstack_tests
  35 from snaps.openstack.utils import deploy_utils
  36
  37
  38 """ logging configuration """
  39 logger = logging.getLogger(__name__)
  40
  41
  42 class TempestCommon(testcase.TestCase):
  43
  44     def __init__(self, **kwargs):
  45         super(TempestCommon, self).__init__(**kwargs)
  46         self.resources = TempestResourcesManager(**kwargs)
  47         self.MODE = ""
  48         self.OPTION = ""
  49         self.VERIFIER_ID = conf_utils.get_verifier_id()
  50         self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir(
  51             self.VERIFIER_ID)
  52         self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id()
  53         self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir(
  54             self.VERIFIER_ID, self.DEPLOYMENT_ID)
  55         self.VERIFICATION_ID = None
  56
  57     @staticmethod
  58     def read_file(filename):
  59         with open(filename) as src:
  60             return [line.strip() for line in src.readlines()]
  61
  62     def generate_test_list(self, verifier_repo_dir):
  63         logger.debug("Generating test case list...")
  64         if self.MODE == 'defcore':
  65             shutil.copyfile(
  66                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
  67         elif self.MODE == 'custom':
  68             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
  69                 shutil.copyfile(
  70                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
  71             else:
  72                 raise Exception("Tempest test list file %s NOT found."
  73                                 % conf_utils.TEMPEST_CUSTOM)
  74         else:
  75             if self.MODE == 'smoke':
  76                 testr_mode = "smoke"
  77             elif self.MODE == 'full':
  78                 testr_mode = ""
  79             else:
  80                 testr_mode = 'tempest.api.' + self.MODE
  81             cmd = ("cd {0};"
  82                    "testr list-tests {1} > {2};"
  83                    "cd -;".format(verifier_repo_dir,
  84                                   testr_mode,
  85                                   conf_utils.TEMPEST_RAW_LIST))
  86             ft_utils.execute_command(cmd)
  87
  88     def apply_tempest_blacklist(self):
  89         logger.debug("Applying tempest blacklist...")
  90         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
  91         result_file = open(conf_utils.TEMPEST_LIST, 'w')
  92         black_tests = []
  93         try:
  94             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
  95             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
  96             if (bool(installer_type) * bool(deploy_scenario)):
  97                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
  98                 # file
  99                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 100                 black_list_yaml = yaml.safe_load(black_list_file)
 101                 black_list_file.close()
 102                 for item in black_list_yaml:
 103                     scenarios = item['scenarios']
 104                     installers = item['installers']
 105                     if (deploy_scenario in scenarios and
 106                             installer_type in installers):
 107                         tests = item['tests']
 108                         for test in tests:
 109                             black_tests.append(test)
 110                         break
 111         except Exception:
 112             black_tests = []
 113             logger.debug("Tempest blacklist file does not exist.")
 114
 115         for cases_line in cases_file:
 116             for black_tests_line in black_tests:
 117                 if black_tests_line in cases_line:
 118                     break
 119             else:
 120                 result_file.write(str(cases_line) + '\n')
 121         result_file.close()
 122
 123     def run_verifier_tests(self):
 124         self.OPTION += (" --load-list {} --detailed"
 125                         .format(conf_utils.TEMPEST_LIST))
 126
 127         cmd_line = "rally verify start " + self.OPTION
 128         logger.info("Starting Tempest test suite: '%s'." % cmd_line)
 129
 130         header = ("Tempest environment:\n"
 131                   "  SUT: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 132                   (CONST.__getattribute__('INSTALLER_TYPE'),
 133                    CONST.__getattribute__('DEPLOY_SCENARIO'),
 134                    CONST.__getattribute__('NODE_NAME'),
 135                    time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 136
 137         f_stdout = open(
 138             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 139         f_stderr = open(
 140             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 141                          "tempest-error.log"), 'w+')
 142         f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 143                                   "environment.log"), 'w+')
 144         f_env.write(header)
 145
 146         p = subprocess.Popen(
 147             cmd_line, shell=True,
 148             stdout=subprocess.PIPE,
 149             stderr=f_stderr,
 150             bufsize=1)
 151
 152         with p.stdout:
 153             for line in iter(p.stdout.readline, b''):
 154                 if re.search("\} tempest\.", line):
 155                     logger.info(line.replace('\n', ''))
 156                 elif re.search('Starting verification', line):
 157                     logger.info(line.replace('\n', ''))
 158                     first_pos = line.index("UUID=") + len("UUID=")
 159                     last_pos = line.index(") for deployment")
 160                     self.VERIFICATION_ID = line[first_pos:last_pos]
 161                     logger.debug('Verification UUID: %s', self.VERIFICATION_ID)
 162                 f_stdout.write(line)
 163         p.wait()
 164
 165         f_stdout.close()
 166         f_stderr.close()
 167         f_env.close()
 168
 169     def parse_verifier_result(self):
 170         if self.VERIFICATION_ID is None:
 171             raise Exception('Verification UUID not found')
 172
 173         cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID)
 174         logger.info("Showing result for a verification: '%s'." % cmd_line)
 175         p = subprocess.Popen(cmd_line,
 176                              shell=True,
 177                              stdout=subprocess.PIPE,
 178                              stderr=subprocess.STDOUT)
 179         for line in p.stdout:
 180             new_line = line.replace(' ', '').split('|')
 181             if 'Tests' in new_line:
 182                 break
 183
 184             logger.info(line)
 185             if 'Testscount' in new_line:
 186                 num_tests = new_line[2]
 187             elif 'Success' in new_line:
 188                 num_success = new_line[2]
 189             elif 'Skipped' in new_line:
 190                 num_skipped = new_line[2]
 191             elif 'Failures' in new_line:
 192                 num_failures = new_line[2]
 193
 194         try:
 195             num_executed = int(num_tests) - int(num_skipped)
 196             try:
 197                 self.result = 100 * int(num_success) / int(num_executed)
 198             except ZeroDivisionError:
 199                 self.result = 0
 200                 if int(num_tests) > 0:
 201                     logger.info("All tests have been skipped")
 202                 else:
 203                     logger.error("No test has been executed")
 204                     return
 205
 206             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 207                                    "tempest.log"), 'r') as logfile:
 208                 output = logfile.read()
 209
 210             success_testcases = []
 211             for match in re.findall('.*\{0\} (.*?)[. ]*success ', output):
 212                 success_testcases.append(match)
 213             failed_testcases = []
 214             for match in re.findall('.*\{0\} (.*?)[. ]*fail ', output):
 215                 failed_testcases.append(match)
 216             skipped_testcases = []
 217             for match in re.findall('.*\{0\} (.*?)[. ]*skip:', output):
 218                 skipped_testcases.append(match)
 219
 220             self.details = {"tests": int(num_tests),
 221                             "failures": int(num_failures),
 222                             "success": success_testcases,
 223                             "errors": failed_testcases,
 224                             "skipped": skipped_testcases}
 225         except Exception:
 226             self.result = 0
 227
 228         logger.info("Tempest %s success_rate is %s%%"
 229                     % (self.case_name, self.result))
 230
 231     def run(self):
 232
 233         self.start_time = time.time()
 234         try:
 235             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 236                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 237             resources = self.resources.create()
 238             compute_cnt = snaps_utils.get_active_compute_cnt(
 239                 self.resources.os_creds)
 240             conf_utils.configure_tempest(
 241                 self.DEPLOYMENT_DIR,
 242                 image_id=resources.get("image_id"),
 243                 flavor_id=resources.get("flavor_id"),
 244                 compute_cnt=compute_cnt)
 245             self.generate_test_list(self.VERIFIER_REPO_DIR)
 246             self.apply_tempest_blacklist()
 247             self.run_verifier_tests()
 248             self.parse_verifier_result()
 249             res = testcase.TestCase.EX_OK
 250         except Exception as e:
 251             logger.error('Error with run: %s' % e)
 252             res = testcase.TestCase.EX_RUN_ERROR
 253         finally:
 254             self.resources.cleanup()
 255
 256         self.stop_time = time.time()
 257         return res
 258
 259
 260 class TempestSmokeSerial(TempestCommon):
 261
 262     def __init__(self, **kwargs):
 263         if "case_name" not in kwargs:
 264             kwargs["case_name"] = 'tempest_smoke_serial'
 265         TempestCommon.__init__(self, **kwargs)
 266         self.MODE = "smoke"
 267         self.OPTION = "--concurrency 1"
 268
 269
 270 class TempestSmokeParallel(TempestCommon):
 271
 272     def __init__(self, **kwargs):
 273         if "case_name" not in kwargs:
 274             kwargs["case_name"] = 'tempest_smoke_parallel'
 275         TempestCommon.__init__(self, **kwargs)
 276         self.MODE = "smoke"
 277         self.OPTION = ""
 278
 279
 280 class TempestFullParallel(TempestCommon):
 281
 282     def __init__(self, **kwargs):
 283         if "case_name" not in kwargs:
 284             kwargs["case_name"] = 'tempest_full_parallel'
 285         TempestCommon.__init__(self, **kwargs)
 286         self.MODE = "full"
 287
 288
 289 class TempestCustom(TempestCommon):
 290
 291     def __init__(self, **kwargs):
 292         if "case_name" not in kwargs:
 293             kwargs["case_name"] = 'tempest_custom'
 294         TempestCommon.__init__(self, **kwargs)
 295         self.MODE = "custom"
 296         self.OPTION = "--concurrency 1"
 297
 298
 299 class TempestDefcore(TempestCommon):
 300
 301     def __init__(self, **kwargs):
 302         if "case_name" not in kwargs:
 303             kwargs["case_name"] = 'tempest_defcore'
 304         TempestCommon.__init__(self, **kwargs)
 305         self.MODE = "defcore"
 306         self.OPTION = "--concurrency 1"
 307
 308
 309 class TempestResourcesManager(object):
 310
 311     def __init__(self, **kwargs):
 312         self.os_creds = None
 313         if 'os_creds' in kwargs:
 314             self.os_creds = kwargs['os_creds']
 315         else:
 316             self.os_creds = openstack_tests.get_credentials(
 317                 os_env_file=CONST.__getattribute__('openstack_creds'))
 318
 319         self.guid = ''
 320         if CONST.__getattribute__('tempest_unique_names'):
 321             self.guid = '-' + str(uuid.uuid4())
 322
 323         self.creators = list()
 324
 325         if hasattr(CONST, 'snaps_images_cirros'):
 326             self.cirros_image_config = CONST.__getattribute__(
 327                 'snaps_images_cirros')
 328         else:
 329             self.cirros_image_config = None
 330
 331     def create(self, use_custom_images=False, use_custom_flavors=False,
 332                create_project=False):
 333         if create_project:
 334             logger.debug("Creating project (tenant) for Tempest suite")
 335             project_name = CONST.__getattribute__(
 336                 'tempest_identity_tenant_name') + self.guid
 337             project_creator = deploy_utils.create_project(
 338                 self.os_creds, ProjectSettings(
 339                     name=project_name,
 340                     description=CONST.__getattribute__(
 341                         'tempest_identity_tenant_description')))
 342             if (project_creator is None or
 343                     project_creator.get_project() is None):
 344                 raise Exception("Failed to create tenant")
 345             project_id = project_creator.get_project().id
 346             self.creators.append(project_creator)
 347
 348             logger.debug("Creating user for Tempest suite")
 349             user_creator = deploy_utils.create_user(
 350                 self.os_creds, UserSettings(
 351                     name=CONST.__getattribute__(
 352                         'tempest_identity_user_name') + self.guid,
 353                     password=CONST.__getattribute__(
 354                         'tempest_identity_user_password'),
 355                     project_name=project_name))
 356             if user_creator is None or user_creator.get_user() is None:
 357                 raise Exception("Failed to create user")
 358             user_id = user_creator.get_user().id
 359             self.creators.append(user_creator)
 360         else:
 361             project_name = None
 362             project_id = None
 363             user_id = None
 364
 365         logger.debug("Creating private network for Tempest suite")
 366         network_creator = deploy_utils.create_network(
 367             self.os_creds, NetworkSettings(
 368                 name=CONST.__getattribute__(
 369                     'tempest_private_net_name') + self.guid,
 370                 project_name=project_name,
 371                 subnet_settings=[SubnetSettings(
 372                     name=CONST.__getattribute__(
 373                         'tempest_private_subnet_name') + self.guid,
 374                     cidr=CONST.__getattribute__('tempest_private_subnet_cidr'))
 375                 ]))
 376         if network_creator is None or network_creator.get_network() is None:
 377             raise Exception("Failed to create private network")
 378         self.creators.append(network_creator)
 379
 380         image_id = None
 381         image_id_alt = None
 382         flavor_id = None
 383         flavor_id_alt = None
 384
 385         if (CONST.__getattribute__('tempest_use_custom_images') or
 386            use_custom_images):
 387             logger.debug("Creating image for Tempest suite")
 388             image_base_name = CONST.__getattribute__(
 389                 'openstack_image_name') + self.guid
 390             os_image_settings = openstack_tests.cirros_image_settings(
 391                 image_base_name, public=True,
 392                 image_metadata=self.cirros_image_config)
 393             logger.debug("Creating image for Tempest suite")
 394             image_creator = deploy_utils.create_image(
 395                 self.os_creds, os_image_settings)
 396             if image_creator is None:
 397                 raise Exception('Failed to create image')
 398             self.creators.append(image_creator)
 399             image_id = image_creator.get_image().id
 400
 401         if use_custom_images:
 402             logger.debug("Creating 2nd image for Tempest suite")
 403             image_base_name_alt = CONST.__getattribute__(
 404                 'openstack_image_name_alt') + self.guid
 405             os_image_settings_alt = openstack_tests.cirros_image_settings(
 406                 image_base_name_alt, public=True,
 407                 image_metadata=self.cirros_image_config)
 408             logger.debug("Creating 2nd image for Tempest suite")
 409             image_creator_alt = deploy_utils.create_image(
 410                 self.os_creds, os_image_settings_alt)
 411             if image_creator_alt is None:
 412                 raise Exception('Failed to create image')
 413             self.creators.append(image_creator_alt)
 414             image_id_alt = image_creator_alt.get_image().id
 415
 416         if (CONST.__getattribute__('tempest_use_custom_flavors') or
 417            use_custom_flavors):
 418             logger.info("Creating flavor for Tempest suite")
 419             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 420             flavor_metadata = None
 421             if 'ovs' in scenario or 'fdio' in scenario:
 422                 flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 423             flavor_creator = OpenStackFlavor(
 424                 self.os_creds, FlavorSettings(
 425                     name=CONST.__getattribute__(
 426                         'openstack_flavor_name') + self.guid,
 427                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 428                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 429                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 430                     metadata=flavor_metadata))
 431             flavor = flavor_creator.create()
 432             if flavor is None:
 433                 raise Exception('Failed to create flavor')
 434             self.creators.append(flavor_creator)
 435             flavor_id = flavor.id
 436
 437         if use_custom_flavors:
 438             logger.info("Creating 2nd flavor for Tempest suite")
 439             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 440             flavor_metadata_alt = None
 441             if 'ovs' in scenario or 'fdio' in scenario:
 442                 flavor_metadata_alt = create_flavor.MEM_PAGE_SIZE_LARGE
 443             flavor_creator_alt = OpenStackFlavor(
 444                 self.os_creds, FlavorSettings(
 445                     name=CONST.__getattribute__(
 446                         'openstack_flavor_name_alt') + self.guid,
 447                     ram=CONST.__getattribute__('openstack_flavor_ram'),
 448                     disk=CONST.__getattribute__('openstack_flavor_disk'),
 449                     vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 450                     metadata=flavor_metadata_alt))
 451             flavor_alt = flavor_creator_alt.create()
 452             if flavor_alt is None:
 453                 raise Exception('Failed to create flavor')
 454             self.creators.append(flavor_creator_alt)
 455             flavor_id_alt = flavor_alt.id
 456
 457         print("RESOURCES CREATE: image_id: %s, image_id_alt: %s, "
 458               "flavor_id: %s, flavor_id_alt: %s" % (
 459                   image_id, image_id_alt, flavor_id, flavor_id_alt,))
 460
 461         result = {
 462             'image_id': image_id,
 463             'image_id_alt': image_id_alt,
 464             'flavor_id': flavor_id,
 465             'flavor_id_alt': flavor_id_alt
 466         }
 467
 468         if create_project:
 469             result['project_id'] = project_id
 470             result['tenant_id'] = project_id  # for compatibility
 471             result['user_id'] = user_id
 472
 473         return result
 474
 475     def cleanup(self):
 476         """
 477         Cleanup all OpenStack objects. Should be called on completion.
 478         """
 479         for creator in reversed(self.creators):
 480             try:
 481                 creator.clean()
 482             except Exception as e:
 483                 logger.error('Unexpected error cleaning - %s', e)