functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import yaml
  24
  25 from functest.core import testcase
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils.constants import CONST
  29 import functest.utils.functest_utils as ft_utils
  30
  31 from snaps.config.flavor import FlavorConfig
  32 from snaps.config.network import NetworkConfig, SubnetConfig
  33 from snaps.config.project import ProjectConfig
  34 from snaps.config.user import UserConfig
  35
  36 from snaps.openstack import create_flavor
  37 from snaps.openstack.create_flavor import OpenStackFlavor
  38 from snaps.openstack.tests import openstack_tests
  39 from snaps.openstack.utils import deploy_utils
  40
  41
  42 LOGGER = logging.getLogger(__name__)
  43
  44
  45 class TempestCommon(testcase.TestCase):
  46     # pylint: disable=too-many-instance-attributes
  47     """TempestCommon testcases implementation class."""
  48
  49     def __init__(self, **kwargs):
  50         super(TempestCommon, self).__init__(**kwargs)
  51         self.resources = TempestResourcesManager(**kwargs)
  52         self.mode = ""
  53         self.option = []
  54         self.verifier_id = conf_utils.get_verifier_id()
  55         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  56             self.verifier_id)
  57         self.deployment_id = conf_utils.get_verifier_deployment_id()
  58         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  59             self.verifier_id, self.deployment_id)
  60         self.verification_id = None
  61
  62     @staticmethod
  63     def read_file(filename):
  64         """Read file and return content as a stripped list."""
  65         with open(filename) as src:
  66             return [line.strip() for line in src.readlines()]
  67
  68     @staticmethod
  69     def get_verifier_result(verif_id):
  70         """Retrieve verification results."""
  71         result = {
  72             'num_tests': 0,
  73             'num_success': 0,
  74             'num_failures': 0,
  75             'num_skipped': 0
  76         }
  77         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  78         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  79         proc = subprocess.Popen(cmd,
  80                                 stdout=subprocess.PIPE,
  81                                 stderr=subprocess.STDOUT)
  82         for line in proc.stdout:
  83             new_line = line.replace(' ', '').split('|')
  84             if 'Tests' in new_line:
  85                 break
  86             LOGGER.info(line)
  87             if 'Testscount' in new_line:
  88                 result['num_tests'] = int(new_line[2])
  89             elif 'Success' in new_line:
  90                 result['num_success'] = int(new_line[2])
  91             elif 'Skipped' in new_line:
  92                 result['num_skipped'] = int(new_line[2])
  93             elif 'Failures' in new_line:
  94                 result['num_failures'] = int(new_line[2])
  95         return result
  96
  97     def generate_test_list(self, verifier_repo_dir):
  98         """Generate test list based on the test mode."""
  99         LOGGER.debug("Generating test case list...")
 100         if self.mode == 'defcore':
 101             shutil.copyfile(
 102                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
 103         elif self.mode == 'custom':
 104             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 105                 shutil.copyfile(
 106                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 107             else:
 108                 raise Exception("Tempest test list file %s NOT found."
 109                                 % conf_utils.TEMPEST_CUSTOM)
 110         else:
 111             if self.mode == 'smoke':
 112                 testr_mode = "smoke"
 113             elif self.mode == 'full':
 114                 testr_mode = ""
 115             else:
 116                 testr_mode = 'tempest.api.' + self.mode
 117             cmd = ("cd {0};"
 118                    "testr list-tests {1} > {2};"
 119                    "cd -;".format(verifier_repo_dir,
 120                                   testr_mode,
 121                                   conf_utils.TEMPEST_RAW_LIST))
 122             ft_utils.execute_command(cmd)
 123
 124     def apply_tempest_blacklist(self):
 125         """Exclude blacklisted test cases."""
 126         LOGGER.debug("Applying tempest blacklist...")
 127         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 128         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 129         black_tests = []
 130         try:
 131             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
 132             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 133             if bool(installer_type) * bool(deploy_scenario):
 134                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 135                 # file
 136                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 137                 black_list_yaml = yaml.safe_load(black_list_file)
 138                 black_list_file.close()
 139                 for item in black_list_yaml:
 140                     scenarios = item['scenarios']
 141                     installers = item['installers']
 142                     if (deploy_scenario in scenarios and
 143                             installer_type in installers):
 144                         tests = item['tests']
 145                         for test in tests:
 146                             black_tests.append(test)
 147                         break
 148         except Exception:  # pylint: disable=broad-except
 149             black_tests = []
 150             LOGGER.debug("Tempest blacklist file does not exist.")
 151
 152         for cases_line in cases_file:
 153             for black_tests_line in black_tests:
 154                 if black_tests_line in cases_line:
 155                     break
 156             else:
 157                 result_file.write(str(cases_line) + '\n')
 158         result_file.close()
 159
 160     def run_verifier_tests(self):
 161         """Execute tempest test cases."""
 162         cmd = ["rally", "verify", "start", "--load-list",
 163                conf_utils.TEMPEST_LIST]
 164         cmd.extend(self.option)
 165         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 166
 167         f_stdout = open(
 168             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 169         f_stderr = open(
 170             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 171                          "tempest-error.log"), 'w+')
 172
 173         proc = subprocess.Popen(
 174             cmd,
 175             stdout=subprocess.PIPE,
 176             stderr=f_stderr,
 177             bufsize=1)
 178
 179         with proc.stdout:
 180             for line in iter(proc.stdout.readline, b''):
 181                 if re.search(r"\} tempest\.", line):
 182                     LOGGER.info(line.replace('\n', ''))
 183                 elif re.search('Starting verification', line):
 184                     LOGGER.info(line.replace('\n', ''))
 185                     first_pos = line.index("UUID=") + len("UUID=")
 186                     last_pos = line.index(") for deployment")
 187                     self.verification_id = line[first_pos:last_pos]
 188                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 189                 f_stdout.write(line)
 190         proc.wait()
 191
 192         f_stdout.close()
 193         f_stderr.close()
 194
 195     def parse_verifier_result(self):
 196         """Parse and save test results."""
 197         if self.verification_id is None:
 198             raise Exception('Verification UUID not found')
 199
 200         stat = self.get_verifier_result(self.verification_id)
 201         try:
 202             num_executed = stat['num_tests'] - stat['num_skipped']
 203             try:
 204                 self.result = 100 * stat['num_success'] / num_executed
 205             except ZeroDivisionError:
 206                 self.result = 0
 207                 if stat['num_tests'] > 0:
 208                     LOGGER.info("All tests have been skipped")
 209                 else:
 210                     LOGGER.error("No test has been executed")
 211                     return
 212
 213             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 214                                    "tempest.log"), 'r') as logfile:
 215                 output = logfile.read()
 216
 217             success_testcases = []
 218             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 219                 success_testcases.append(match)
 220             failed_testcases = []
 221             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 222                 failed_testcases.append(match)
 223             skipped_testcases = []
 224             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 225                 skipped_testcases.append(match)
 226
 227             self.details = {"tests": stat['num_tests'],
 228                             "failures": stat['num_failures'],
 229                             "success": success_testcases,
 230                             "skipped": skipped_testcases,
 231                             "errors": failed_testcases}
 232         except Exception:  # pylint: disable=broad-except
 233             self.result = 0
 234
 235         LOGGER.info("Tempest %s success_rate is %s%%",
 236                     self.case_name, self.result)
 237
 238     def run(self, **kwargs):
 239
 240         self.start_time = time.time()
 241         try:
 242             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 243                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 244             resources = self.resources.create()
 245             compute_cnt = snaps_utils.get_active_compute_cnt(
 246                 self.resources.os_creds)
 247             conf_utils.configure_tempest(
 248                 self.deployment_dir,
 249                 network_name=resources.get("network_name"),
 250                 image_id=resources.get("image_id"),
 251                 flavor_id=resources.get("flavor_id"),
 252                 compute_cnt=compute_cnt)
 253             self.generate_test_list(self.verifier_repo_dir)
 254             self.apply_tempest_blacklist()
 255             self.run_verifier_tests()
 256             self.parse_verifier_result()
 257             res = testcase.TestCase.EX_OK
 258         except Exception as err:  # pylint: disable=broad-except
 259             LOGGER.error('Error with run: %s', err)
 260             res = testcase.TestCase.EX_RUN_ERROR
 261         finally:
 262             self.resources.cleanup()
 263
 264         self.stop_time = time.time()
 265         return res
 266
 267
 268 class TempestSmokeSerial(TempestCommon):
 269     """Tempest smoke serial testcase implementation."""
 270     def __init__(self, **kwargs):
 271         if "case_name" not in kwargs:
 272             kwargs["case_name"] = 'tempest_smoke_serial'
 273         TempestCommon.__init__(self, **kwargs)
 274         self.mode = "smoke"
 275         self.option = ["--concurrency", "1"]
 276
 277
 278 class TempestSmokeParallel(TempestCommon):
 279     """Tempest smoke parallel testcase implementation."""
 280     def __init__(self, **kwargs):
 281         if "case_name" not in kwargs:
 282             kwargs["case_name"] = 'tempest_smoke_parallel'
 283         TempestCommon.__init__(self, **kwargs)
 284         self.mode = "smoke"
 285
 286
 287 class TempestFullParallel(TempestCommon):
 288     """Tempest full parallel testcase implementation."""
 289     def __init__(self, **kwargs):
 290         if "case_name" not in kwargs:
 291             kwargs["case_name"] = 'tempest_full_parallel'
 292         TempestCommon.__init__(self, **kwargs)
 293         self.mode = "full"
 294
 295
 296 class TempestCustom(TempestCommon):
 297     """Tempest custom testcase implementation."""
 298     def __init__(self, **kwargs):
 299         if "case_name" not in kwargs:
 300             kwargs["case_name"] = 'tempest_custom'
 301         TempestCommon.__init__(self, **kwargs)
 302         self.mode = "custom"
 303         self.option = ["--concurrency", "1"]
 304
 305
 306 class TempestDefcore(TempestCommon):
 307     """Tempest Defcore testcase implementation."""
 308     def __init__(self, **kwargs):
 309         if "case_name" not in kwargs:
 310             kwargs["case_name"] = 'tempest_defcore'
 311         TempestCommon.__init__(self, **kwargs)
 312         self.mode = "defcore"
 313         self.option = ["--concurrency", "1"]
 314
 315
 316 class TempestResourcesManager(object):
 317     """Tempest resource manager."""
 318     def __init__(self, **kwargs):
 319         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 320         self.guid = '-' + str(uuid.uuid4())
 321         self.creators = list()
 322
 323         if hasattr(CONST, 'snaps_images_cirros'):
 324             self.cirros_image_config = CONST.__getattribute__(
 325                 'snaps_images_cirros')
 326         else:
 327             self.cirros_image_config = None
 328
 329     def _create_project(self):
 330         """Create project for tests."""
 331         project_creator = deploy_utils.create_project(
 332             self.os_creds, ProjectConfig(
 333                 name=CONST.__getattribute__(
 334                     'tempest_identity_tenant_name') + self.guid,
 335                 description=CONST.__getattribute__(
 336                     'tempest_identity_tenant_description')))
 337         if project_creator is None or project_creator.get_project() is None:
 338             raise Exception("Failed to create tenant")
 339         self.creators.append(project_creator)
 340         return project_creator.get_project().id
 341
 342     def _create_user(self):
 343         """Create user for tests."""
 344         user_creator = deploy_utils.create_user(
 345             self.os_creds, UserConfig(
 346                 name=CONST.__getattribute__(
 347                     'tempest_identity_user_name') + self.guid,
 348                 password=CONST.__getattribute__(
 349                     'tempest_identity_user_password'),
 350                 project_name=CONST.__getattribute__(
 351                     'tempest_identity_tenant_name') + self.guid))
 352         if user_creator is None or user_creator.get_user() is None:
 353             raise Exception("Failed to create user")
 354         self.creators.append(user_creator)
 355         return user_creator.get_user().id
 356
 357     def _create_network(self, project_name):
 358         """Create network for tests."""
 359         tempest_network_type = None
 360         tempest_physical_network = None
 361         tempest_segmentation_id = None
 362
 363         if hasattr(CONST, 'tempest_network_type'):
 364             tempest_network_type = CONST.__getattribute__(
 365                 'tempest_network_type')
 366         if hasattr(CONST, 'tempest_physical_network'):
 367             tempest_physical_network = CONST.__getattribute__(
 368                 'tempest_physical_network')
 369         if hasattr(CONST, 'tempest_segmentation_id'):
 370             tempest_segmentation_id = CONST.__getattribute__(
 371                 'tempest_segmentation_id')
 372
 373         tempest_net_name = CONST.__getattribute__(
 374             'tempest_private_net_name') + self.guid
 375
 376         network_creator = deploy_utils.create_network(
 377             self.os_creds, NetworkConfig(
 378                 name=tempest_net_name,
 379                 project_name=project_name,
 380                 network_type=tempest_network_type,
 381                 physical_network=tempest_physical_network,
 382                 segmentation_id=tempest_segmentation_id,
 383                 subnet_settings=[SubnetConfig(
 384                     name=CONST.__getattribute__(
 385                         'tempest_private_subnet_name') + self.guid,
 386                     project_name=project_name,
 387                     cidr=CONST.__getattribute__(
 388                         'tempest_private_subnet_cidr'))]))
 389         if network_creator is None or network_creator.get_network() is None:
 390             raise Exception("Failed to create private network")
 391         self.creators.append(network_creator)
 392         return tempest_net_name
 393
 394     def _create_image(self, name):
 395         """Create image for tests"""
 396         os_image_settings = openstack_tests.cirros_image_settings(
 397             name, public=True,
 398             image_metadata=self.cirros_image_config)
 399         image_creator = deploy_utils.create_image(
 400             self.os_creds, os_image_settings)
 401         if image_creator is None:
 402             raise Exception('Failed to create image')
 403         self.creators.append(image_creator)
 404         return image_creator.get_image().id
 405
 406     def _create_flavor(self, name):
 407         """Create flavor for tests."""
 408         scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 409         flavor_metadata = None
 410         if 'ovs' in scenario or 'fdio' in scenario:
 411             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 412         flavor_creator = OpenStackFlavor(
 413             self.os_creds, FlavorConfig(
 414                 name=name,
 415                 ram=CONST.__getattribute__('openstack_flavor_ram'),
 416                 disk=CONST.__getattribute__('openstack_flavor_disk'),
 417                 vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 418                 metadata=flavor_metadata))
 419         flavor = flavor_creator.create()
 420         if flavor is None:
 421             raise Exception('Failed to create flavor')
 422         self.creators.append(flavor_creator)
 423         return flavor.id
 424
 425     def create(self, use_custom_images=False, use_custom_flavors=False,
 426                create_project=False):
 427         """Create resources for Tempest test suite."""
 428         result = {
 429             'tempest_net_name': None,
 430             'image_id': None,
 431             'image_id_alt': None,
 432             'flavor_id': None,
 433             'flavor_id_alt': None
 434         }
 435         project_name = None
 436
 437         if create_project:
 438             LOGGER.debug("Creating project and user for Tempest suite")
 439             project_name = CONST.__getattribute__(
 440                 'tempest_identity_tenant_name') + self.guid
 441             result['project_id'] = self._create_project()
 442             result['user_id'] = self._create_user()
 443             result['tenant_id'] = result['project_id']  # for compatibility
 444
 445         LOGGER.debug("Creating private network for Tempest suite")
 446         result['tempest_net_name'] = self._create_network(project_name)
 447
 448         LOGGER.debug("Creating image for Tempest suite")
 449         image_name = CONST.__getattribute__('openstack_image_name') + self.guid
 450         result['image_id'] = self._create_image(image_name)
 451
 452         if use_custom_images:
 453             LOGGER.debug("Creating 2nd image for Tempest suite")
 454             image_name = CONST.__getattribute__(
 455                 'openstack_image_name_alt') + self.guid
 456             result['image_id_alt'] = self._create_image(image_name)
 457
 458         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 459                 use_custom_flavors):
 460             LOGGER.info("Creating flavor for Tempest suite")
 461             name = CONST.__getattribute__('openstack_flavor_name') + self.guid
 462             result['flavor_id'] = self._create_flavor(name)
 463
 464         if use_custom_flavors:
 465             LOGGER.info("Creating 2nd flavor for Tempest suite")
 466             scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 467             if 'ovs' in scenario or 'fdio' in scenario:
 468                 CONST.__setattr__('openstack_flavor_ram', 1024)
 469             name = CONST.__getattribute__(
 470                 'openstack_flavor_name_alt') + self.guid
 471             result['flavor_id_alt'] = self._create_flavor(name)
 472
 473         return result
 474
 475     def cleanup(self):
 476         """
 477         Cleanup all OpenStack objects. Should be called on completion.
 478         """
 479         for creator in reversed(self.creators):
 480             try:
 481                 creator.clean()
 482             except Exception as err:  # pylint: disable=broad-except
 483                 LOGGER.error('Unexpected error cleaning - %s', err)