functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import yaml
  24
  25 from functest.core import testcase
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils.constants import CONST
  29 from functest.utils import env
  30 import functest.utils.functest_utils as ft_utils
  31
  32 from snaps.config.flavor import FlavorConfig
  33 from snaps.config.network import NetworkConfig, SubnetConfig
  34 from snaps.config.project import ProjectConfig
  35 from snaps.config.user import UserConfig
  36
  37 from snaps.openstack import create_flavor
  38 from snaps.openstack.create_flavor import OpenStackFlavor
  39 from snaps.openstack.tests import openstack_tests
  40 from snaps.openstack.utils import deploy_utils
  41
  42
  43 LOGGER = logging.getLogger(__name__)
  44
  45
  46 class TempestCommon(testcase.TestCase):
  47     # pylint: disable=too-many-instance-attributes
  48     """TempestCommon testcases implementation class."""
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62
  63     @staticmethod
  64     def read_file(filename):
  65         """Read file and return content as a stripped list."""
  66         with open(filename) as src:
  67             return [line.strip() for line in src.readlines()]
  68
  69     @staticmethod
  70     def get_verifier_result(verif_id):
  71         """Retrieve verification results."""
  72         result = {
  73             'num_tests': 0,
  74             'num_success': 0,
  75             'num_failures': 0,
  76             'num_skipped': 0
  77         }
  78         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  79         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  80         proc = subprocess.Popen(cmd,
  81                                 stdout=subprocess.PIPE,
  82                                 stderr=subprocess.STDOUT)
  83         for line in proc.stdout:
  84             new_line = line.replace(' ', '').split('|')
  85             if 'Tests' in new_line:
  86                 break
  87             LOGGER.info(line)
  88             if 'Testscount' in new_line:
  89                 result['num_tests'] = int(new_line[2])
  90             elif 'Success' in new_line:
  91                 result['num_success'] = int(new_line[2])
  92             elif 'Skipped' in new_line:
  93                 result['num_skipped'] = int(new_line[2])
  94             elif 'Failures' in new_line:
  95                 result['num_failures'] = int(new_line[2])
  96         return result
  97
  98     def generate_test_list(self, verifier_repo_dir):
  99         """Generate test list based on the test mode."""
 100         LOGGER.debug("Generating test case list...")
 101         if self.mode == 'defcore':
 102             shutil.copyfile(
 103                 conf_utils.TEMPEST_DEFCORE, conf_utils.TEMPEST_RAW_LIST)
 104         elif self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = "smoke"
 114             elif self.mode == 'full':
 115                 testr_mode = ""
 116             else:
 117                 testr_mode = 'tempest.api.' + self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(verifier_repo_dir,
 121                                   testr_mode,
 122                                   conf_utils.TEMPEST_RAW_LIST))
 123             ft_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 129         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 130         black_tests = []
 131         try:
 132             installer_type = env.get('INSTALLER_TYPE')
 133             deploy_scenario = env.get('DEPLOY_SCENARIO')
 134             if bool(installer_type) * bool(deploy_scenario):
 135                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 136                 # file
 137                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 138                 black_list_yaml = yaml.safe_load(black_list_file)
 139                 black_list_file.close()
 140                 for item in black_list_yaml:
 141                     scenarios = item['scenarios']
 142                     installers = item['installers']
 143                     if (deploy_scenario in scenarios and
 144                             installer_type in installers):
 145                         tests = item['tests']
 146                         for test in tests:
 147                             black_tests.append(test)
 148                         break
 149         except Exception:  # pylint: disable=broad-except
 150             black_tests = []
 151             LOGGER.debug("Tempest blacklist file does not exist.")
 152
 153         for cases_line in cases_file:
 154             for black_tests_line in black_tests:
 155                 if black_tests_line in cases_line:
 156                     break
 157             else:
 158                 result_file.write(str(cases_line) + '\n')
 159         result_file.close()
 160
 161     def run_verifier_tests(self):
 162         """Execute tempest test cases."""
 163         cmd = ["rally", "verify", "start", "--load-list",
 164                conf_utils.TEMPEST_LIST]
 165         cmd.extend(self.option)
 166         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 167
 168         f_stdout = open(
 169             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 170         f_stderr = open(
 171             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 172                          "tempest-error.log"), 'w+')
 173
 174         proc = subprocess.Popen(
 175             cmd,
 176             stdout=subprocess.PIPE,
 177             stderr=f_stderr,
 178             bufsize=1)
 179
 180         with proc.stdout:
 181             for line in iter(proc.stdout.readline, b''):
 182                 if re.search(r"\} tempest\.", line):
 183                     LOGGER.info(line.replace('\n', ''))
 184                 elif re.search('Starting verification', line):
 185                     LOGGER.info(line.replace('\n', ''))
 186                     first_pos = line.index("UUID=") + len("UUID=")
 187                     last_pos = line.index(") for deployment")
 188                     self.verification_id = line[first_pos:last_pos]
 189                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 190                 f_stdout.write(line)
 191         proc.wait()
 192
 193         f_stdout.close()
 194         f_stderr.close()
 195
 196     def parse_verifier_result(self):
 197         """Parse and save test results."""
 198         if self.verification_id is None:
 199             raise Exception('Verification UUID not found')
 200
 201         stat = self.get_verifier_result(self.verification_id)
 202         try:
 203             num_executed = stat['num_tests'] - stat['num_skipped']
 204             try:
 205                 self.result = 100 * stat['num_success'] / num_executed
 206             except ZeroDivisionError:
 207                 self.result = 0
 208                 if stat['num_tests'] > 0:
 209                     LOGGER.info("All tests have been skipped")
 210                 else:
 211                     LOGGER.error("No test has been executed")
 212                     return
 213
 214             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 215                                    "tempest.log"), 'r') as logfile:
 216                 output = logfile.read()
 217
 218             success_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 220                 success_testcases.append(match)
 221             failed_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 223                 failed_testcases.append(match)
 224             skipped_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 226                 skipped_testcases.append(match)
 227
 228             self.details = {"tests": stat['num_tests'],
 229                             "failures": stat['num_failures'],
 230                             "success": success_testcases,
 231                             "skipped": skipped_testcases,
 232                             "errors": failed_testcases}
 233         except Exception:  # pylint: disable=broad-except
 234             self.result = 0
 235
 236         LOGGER.info("Tempest %s success_rate is %s%%",
 237                     self.case_name, self.result)
 238
 239     def run(self, **kwargs):
 240
 241         self.start_time = time.time()
 242         try:
 243             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 244                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 245             resources = self.resources.create()
 246             compute_cnt = snaps_utils.get_active_compute_cnt(
 247                 self.resources.os_creds)
 248             conf_utils.configure_tempest(
 249                 self.deployment_dir,
 250                 network_name=resources.get("network_name"),
 251                 image_id=resources.get("image_id"),
 252                 flavor_id=resources.get("flavor_id"),
 253                 compute_cnt=compute_cnt)
 254             self.generate_test_list(self.verifier_repo_dir)
 255             self.apply_tempest_blacklist()
 256             self.run_verifier_tests()
 257             self.parse_verifier_result()
 258             res = testcase.TestCase.EX_OK
 259         except Exception as err:  # pylint: disable=broad-except
 260             LOGGER.error('Error with run: %s', err)
 261             res = testcase.TestCase.EX_RUN_ERROR
 262         finally:
 263             self.resources.cleanup()
 264
 265         self.stop_time = time.time()
 266         return res
 267
 268
 269 class TempestSmokeSerial(TempestCommon):
 270     """Tempest smoke serial testcase implementation."""
 271     def __init__(self, **kwargs):
 272         if "case_name" not in kwargs:
 273             kwargs["case_name"] = 'tempest_smoke_serial'
 274         TempestCommon.__init__(self, **kwargs)
 275         self.mode = "smoke"
 276         self.option = ["--concurrency", "1"]
 277
 278
 279 class TempestSmokeParallel(TempestCommon):
 280     """Tempest smoke parallel testcase implementation."""
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_smoke_parallel'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.mode = "smoke"
 286
 287
 288 class TempestFullParallel(TempestCommon):
 289     """Tempest full parallel testcase implementation."""
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_full_parallel'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.mode = "full"
 295
 296
 297 class TempestCustom(TempestCommon):
 298     """Tempest custom testcase implementation."""
 299     def __init__(self, **kwargs):
 300         if "case_name" not in kwargs:
 301             kwargs["case_name"] = 'tempest_custom'
 302         TempestCommon.__init__(self, **kwargs)
 303         self.mode = "custom"
 304         self.option = ["--concurrency", "1"]
 305
 306
 307 class TempestDefcore(TempestCommon):
 308     """Tempest Defcore testcase implementation."""
 309     def __init__(self, **kwargs):
 310         if "case_name" not in kwargs:
 311             kwargs["case_name"] = 'tempest_defcore'
 312         TempestCommon.__init__(self, **kwargs)
 313         self.mode = "defcore"
 314         self.option = ["--concurrency", "1"]
 315
 316
 317 class TempestResourcesManager(object):
 318     """Tempest resource manager."""
 319     def __init__(self, **kwargs):
 320         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 321         self.guid = '-' + str(uuid.uuid4())
 322         self.creators = list()
 323
 324         if hasattr(CONST, 'snaps_images_cirros'):
 325             self.cirros_image_config = CONST.__getattribute__(
 326                 'snaps_images_cirros')
 327         else:
 328             self.cirros_image_config = None
 329
 330     def _create_project(self):
 331         """Create project for tests."""
 332         project_creator = deploy_utils.create_project(
 333             self.os_creds, ProjectConfig(
 334                 name=CONST.__getattribute__(
 335                     'tempest_identity_tenant_name') + self.guid,
 336                 description=CONST.__getattribute__(
 337                     'tempest_identity_tenant_description')))
 338         if project_creator is None or project_creator.get_project() is None:
 339             raise Exception("Failed to create tenant")
 340         self.creators.append(project_creator)
 341         return project_creator.get_project().id
 342
 343     def _create_user(self):
 344         """Create user for tests."""
 345         user_creator = deploy_utils.create_user(
 346             self.os_creds, UserConfig(
 347                 name=CONST.__getattribute__(
 348                     'tempest_identity_user_name') + self.guid,
 349                 password=CONST.__getattribute__(
 350                     'tempest_identity_user_password'),
 351                 project_name=CONST.__getattribute__(
 352                     'tempest_identity_tenant_name') + self.guid))
 353         if user_creator is None or user_creator.get_user() is None:
 354             raise Exception("Failed to create user")
 355         self.creators.append(user_creator)
 356         return user_creator.get_user().id
 357
 358     def _create_network(self, project_name):
 359         """Create network for tests."""
 360         tempest_network_type = None
 361         tempest_physical_network = None
 362         tempest_segmentation_id = None
 363
 364         if hasattr(CONST, 'tempest_network_type'):
 365             tempest_network_type = CONST.__getattribute__(
 366                 'tempest_network_type')
 367         if hasattr(CONST, 'tempest_physical_network'):
 368             tempest_physical_network = CONST.__getattribute__(
 369                 'tempest_physical_network')
 370         if hasattr(CONST, 'tempest_segmentation_id'):
 371             tempest_segmentation_id = CONST.__getattribute__(
 372                 'tempest_segmentation_id')
 373
 374         tempest_net_name = CONST.__getattribute__(
 375             'tempest_private_net_name') + self.guid
 376
 377         network_creator = deploy_utils.create_network(
 378             self.os_creds, NetworkConfig(
 379                 name=tempest_net_name,
 380                 project_name=project_name,
 381                 network_type=tempest_network_type,
 382                 physical_network=tempest_physical_network,
 383                 segmentation_id=tempest_segmentation_id,
 384                 subnet_settings=[SubnetConfig(
 385                     name=CONST.__getattribute__(
 386                         'tempest_private_subnet_name') + self.guid,
 387                     project_name=project_name,
 388                     cidr=CONST.__getattribute__(
 389                         'tempest_private_subnet_cidr'))]))
 390         if network_creator is None or network_creator.get_network() is None:
 391             raise Exception("Failed to create private network")
 392         self.creators.append(network_creator)
 393         return tempest_net_name
 394
 395     def _create_image(self, name):
 396         """Create image for tests"""
 397         os_image_settings = openstack_tests.cirros_image_settings(
 398             name, public=True,
 399             image_metadata=self.cirros_image_config)
 400         image_creator = deploy_utils.create_image(
 401             self.os_creds, os_image_settings)
 402         if image_creator is None:
 403             raise Exception('Failed to create image')
 404         self.creators.append(image_creator)
 405         return image_creator.get_image().id
 406
 407     def _create_flavor(self, name):
 408         """Create flavor for tests."""
 409         scenario = env.get('DEPLOY_SCENARIO')
 410         flavor_metadata = None
 411         if 'ovs' in scenario or 'fdio' in scenario:
 412             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 413         flavor_creator = OpenStackFlavor(
 414             self.os_creds, FlavorConfig(
 415                 name=name,
 416                 ram=CONST.__getattribute__('openstack_flavor_ram'),
 417                 disk=CONST.__getattribute__('openstack_flavor_disk'),
 418                 vcpus=CONST.__getattribute__('openstack_flavor_vcpus'),
 419                 metadata=flavor_metadata))
 420         flavor = flavor_creator.create()
 421         if flavor is None:
 422             raise Exception('Failed to create flavor')
 423         self.creators.append(flavor_creator)
 424         return flavor.id
 425
 426     def create(self, use_custom_images=False, use_custom_flavors=False,
 427                create_project=False):
 428         """Create resources for Tempest test suite."""
 429         result = {
 430             'tempest_net_name': None,
 431             'image_id': None,
 432             'image_id_alt': None,
 433             'flavor_id': None,
 434             'flavor_id_alt': None
 435         }
 436         project_name = None
 437
 438         if create_project:
 439             LOGGER.debug("Creating project and user for Tempest suite")
 440             project_name = CONST.__getattribute__(
 441                 'tempest_identity_tenant_name') + self.guid
 442             result['project_id'] = self._create_project()
 443             result['user_id'] = self._create_user()
 444             result['tenant_id'] = result['project_id']  # for compatibility
 445
 446         LOGGER.debug("Creating private network for Tempest suite")
 447         result['tempest_net_name'] = self._create_network(project_name)
 448
 449         LOGGER.debug("Creating image for Tempest suite")
 450         image_name = CONST.__getattribute__('openstack_image_name') + self.guid
 451         result['image_id'] = self._create_image(image_name)
 452
 453         if use_custom_images:
 454             LOGGER.debug("Creating 2nd image for Tempest suite")
 455             image_name = CONST.__getattribute__(
 456                 'openstack_image_name_alt') + self.guid
 457             result['image_id_alt'] = self._create_image(image_name)
 458
 459         if (CONST.__getattribute__('tempest_use_custom_flavors') == 'True' or
 460                 use_custom_flavors):
 461             LOGGER.info("Creating flavor for Tempest suite")
 462             name = CONST.__getattribute__('openstack_flavor_name') + self.guid
 463             result['flavor_id'] = self._create_flavor(name)
 464
 465         if use_custom_flavors:
 466             LOGGER.info("Creating 2nd flavor for Tempest suite")
 467             scenario = env.get('DEPLOY_SCENARIO')
 468             if 'ovs' in scenario or 'fdio' in scenario:
 469                 CONST.__setattr__('openstack_flavor_ram', 1024)
 470             name = CONST.__getattribute__(
 471                 'openstack_flavor_name_alt') + self.guid
 472             result['flavor_id_alt'] = self._create_flavor(name)
 473
 474         return result
 475
 476     def cleanup(self):
 477         """
 478         Cleanup all OpenStack objects. Should be called on completion.
 479         """
 480         for creator in reversed(self.creators):
 481             try:
 482                 creator.clean()
 483             except Exception as err:  # pylint: disable=broad-except
 484                 LOGGER.error('Unexpected error cleaning - %s', err)