functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import yaml
  24
  25 from functest.core import testcase
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.opnfv_tests.openstack.tempest import conf_utils
  28 from functest.utils import config
  29 from functest.utils import env
  30 import functest.utils.functest_utils as ft_utils
  31
  32 from snaps.config.flavor import FlavorConfig
  33 from snaps.config.network import NetworkConfig, SubnetConfig
  34 from snaps.config.project import ProjectConfig
  35 from snaps.config.user import UserConfig
  36
  37 from snaps.openstack import create_flavor
  38 from snaps.openstack.create_flavor import OpenStackFlavor
  39 from snaps.openstack.tests import openstack_tests
  40 from snaps.openstack.utils import deploy_utils
  41
  42
  43 LOGGER = logging.getLogger(__name__)
  44
  45
  46 class TempestCommon(testcase.TestCase):
  47     # pylint: disable=too-many-instance-attributes
  48     """TempestCommon testcases implementation class."""
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62
  63     @staticmethod
  64     def read_file(filename):
  65         """Read file and return content as a stripped list."""
  66         with open(filename) as src:
  67             return [line.strip() for line in src.readlines()]
  68
  69     @staticmethod
  70     def get_verifier_result(verif_id):
  71         """Retrieve verification results."""
  72         result = {
  73             'num_tests': 0,
  74             'num_success': 0,
  75             'num_failures': 0,
  76             'num_skipped': 0
  77         }
  78         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  79         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  80         proc = subprocess.Popen(cmd,
  81                                 stdout=subprocess.PIPE,
  82                                 stderr=subprocess.STDOUT)
  83         for line in proc.stdout:
  84             new_line = line.replace(' ', '').split('|')
  85             if 'Tests' in new_line:
  86                 break
  87             LOGGER.info(line)
  88             if 'Testscount' in new_line:
  89                 result['num_tests'] = int(new_line[2])
  90             elif 'Success' in new_line:
  91                 result['num_success'] = int(new_line[2])
  92             elif 'Skipped' in new_line:
  93                 result['num_skipped'] = int(new_line[2])
  94             elif 'Failures' in new_line:
  95                 result['num_failures'] = int(new_line[2])
  96         return result
  97
  98     def generate_test_list(self, verifier_repo_dir):
  99         """Generate test list based on the test mode."""
 100         LOGGER.debug("Generating test case list...")
 101         if self.mode == 'custom':
 102             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 103                 shutil.copyfile(
 104                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 105             else:
 106                 raise Exception("Tempest test list file %s NOT found."
 107                                 % conf_utils.TEMPEST_CUSTOM)
 108         else:
 109             if self.mode == 'smoke':
 110                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 111             elif self.mode == 'full':
 112                 testr_mode = r"'^tempest\.'"
 113             else:
 114                 testr_mode = self.mode
 115             cmd = ("cd {0};"
 116                    "testr list-tests {1} > {2};"
 117                    "cd -;".format(verifier_repo_dir,
 118                                   testr_mode,
 119                                   conf_utils.TEMPEST_RAW_LIST))
 120             ft_utils.execute_command(cmd)
 121
 122     def apply_tempest_blacklist(self):
 123         """Exclude blacklisted test cases."""
 124         LOGGER.debug("Applying tempest blacklist...")
 125         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 126         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 127         black_tests = []
 128         try:
 129             installer_type = env.get('INSTALLER_TYPE')
 130             deploy_scenario = env.get('DEPLOY_SCENARIO')
 131             if bool(installer_type) * bool(deploy_scenario):
 132                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 133                 # file
 134                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 135                 black_list_yaml = yaml.safe_load(black_list_file)
 136                 black_list_file.close()
 137                 for item in black_list_yaml:
 138                     scenarios = item['scenarios']
 139                     installers = item['installers']
 140                     if (deploy_scenario in scenarios and
 141                             installer_type in installers):
 142                         tests = item['tests']
 143                         for test in tests:
 144                             black_tests.append(test)
 145                         break
 146         except Exception:  # pylint: disable=broad-except
 147             black_tests = []
 148             LOGGER.debug("Tempest blacklist file does not exist.")
 149
 150         for cases_line in cases_file:
 151             for black_tests_line in black_tests:
 152                 if black_tests_line in cases_line:
 153                     break
 154             else:
 155                 result_file.write(str(cases_line) + '\n')
 156         result_file.close()
 157
 158     def run_verifier_tests(self):
 159         """Execute tempest test cases."""
 160         cmd = ["rally", "verify", "start", "--load-list",
 161                conf_utils.TEMPEST_LIST]
 162         cmd.extend(self.option)
 163         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 164
 165         f_stdout = open(
 166             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 167         f_stderr = open(
 168             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 169                          "tempest-error.log"), 'w+')
 170
 171         proc = subprocess.Popen(
 172             cmd,
 173             stdout=subprocess.PIPE,
 174             stderr=f_stderr,
 175             bufsize=1)
 176
 177         with proc.stdout:
 178             for line in iter(proc.stdout.readline, b''):
 179                 if re.search(r"\} tempest\.", line):
 180                     LOGGER.info(line.replace('\n', ''))
 181                 elif re.search('Starting verification', line):
 182                     LOGGER.info(line.replace('\n', ''))
 183                     first_pos = line.index("UUID=") + len("UUID=")
 184                     last_pos = line.index(") for deployment")
 185                     self.verification_id = line[first_pos:last_pos]
 186                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 187                 f_stdout.write(line)
 188         proc.wait()
 189
 190         f_stdout.close()
 191         f_stderr.close()
 192
 193         if self.verification_id is None:
 194             raise Exception('Verification UUID not found')
 195
 196     def parse_verifier_result(self):
 197         """Parse and save test results."""
 198         stat = self.get_verifier_result(self.verification_id)
 199         try:
 200             num_executed = stat['num_tests'] - stat['num_skipped']
 201             try:
 202                 self.result = 100 * stat['num_success'] / num_executed
 203             except ZeroDivisionError:
 204                 self.result = 0
 205                 if stat['num_tests'] > 0:
 206                     LOGGER.info("All tests have been skipped")
 207                 else:
 208                     LOGGER.error("No test has been executed")
 209                     return
 210
 211             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 212                                    "tempest.log"), 'r') as logfile:
 213                 output = logfile.read()
 214
 215             success_testcases = []
 216             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 217                 success_testcases.append(match)
 218             failed_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 220                 failed_testcases.append(match)
 221             skipped_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 223                 skipped_testcases.append(match)
 224
 225             self.details = {"tests": stat['num_tests'],
 226                             "failures": stat['num_failures'],
 227                             "success": success_testcases,
 228                             "skipped": skipped_testcases,
 229                             "errors": failed_testcases}
 230         except Exception:  # pylint: disable=broad-except
 231             self.result = 0
 232
 233         LOGGER.info("Tempest %s success_rate is %s%%",
 234                     self.case_name, self.result)
 235
 236     def generate_report(self):
 237         """Generate verification report."""
 238         html_file = os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 239                                  "tempest-report.html")
 240         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 241                self.verification_id, "--to", html_file]
 242         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 243                          stderr=subprocess.STDOUT)
 244
 245     def run(self, **kwargs):
 246
 247         self.start_time = time.time()
 248         try:
 249             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 250                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 251             resources = self.resources.create()
 252             compute_cnt = snaps_utils.get_active_compute_cnt(
 253                 self.resources.os_creds)
 254             conf_utils.configure_tempest(
 255                 self.deployment_dir,
 256                 network_name=resources.get("network_name"),
 257                 image_id=resources.get("image_id"),
 258                 flavor_id=resources.get("flavor_id"),
 259                 compute_cnt=compute_cnt)
 260             self.generate_test_list(self.verifier_repo_dir)
 261             self.apply_tempest_blacklist()
 262             self.run_verifier_tests()
 263             self.parse_verifier_result()
 264             self.generate_report()
 265             res = testcase.TestCase.EX_OK
 266         except Exception as err:  # pylint: disable=broad-except
 267             LOGGER.error('Error with run: %s', err)
 268             res = testcase.TestCase.EX_RUN_ERROR
 269         finally:
 270             self.resources.cleanup()
 271
 272         self.stop_time = time.time()
 273         return res
 274
 275
 276 class TempestSmokeSerial(TempestCommon):
 277     """Tempest smoke serial testcase implementation."""
 278     def __init__(self, **kwargs):
 279         if "case_name" not in kwargs:
 280             kwargs["case_name"] = 'tempest_smoke_serial'
 281         TempestCommon.__init__(self, **kwargs)
 282         self.mode = "smoke"
 283         self.option = ["--concurrency", "1"]
 284
 285
 286 class TempestSmokeParallel(TempestCommon):
 287     """Tempest smoke parallel testcase implementation."""
 288     def __init__(self, **kwargs):
 289         if "case_name" not in kwargs:
 290             kwargs["case_name"] = 'tempest_smoke_parallel'
 291         TempestCommon.__init__(self, **kwargs)
 292         self.mode = "smoke"
 293
 294
 295 class TempestFullParallel(TempestCommon):
 296     """Tempest full parallel testcase implementation."""
 297     def __init__(self, **kwargs):
 298         if "case_name" not in kwargs:
 299             kwargs["case_name"] = 'tempest_full_parallel'
 300         TempestCommon.__init__(self, **kwargs)
 301         self.mode = "full"
 302
 303
 304 class TempestCustom(TempestCommon):
 305     """Tempest custom testcase implementation."""
 306     def __init__(self, **kwargs):
 307         if "case_name" not in kwargs:
 308             kwargs["case_name"] = 'tempest_custom'
 309         TempestCommon.__init__(self, **kwargs)
 310         self.mode = "custom"
 311         self.option = ["--concurrency", "1"]
 312
 313
 314 class TempestDefcore(TempestCommon):
 315     """Tempest Defcore testcase implementation."""
 316     def __init__(self, **kwargs):
 317         if "case_name" not in kwargs:
 318             kwargs["case_name"] = 'tempest_defcore'
 319         TempestCommon.__init__(self, **kwargs)
 320         self.mode = "defcore"
 321         self.option = ["--concurrency", "1"]
 322
 323
 324 class TempestResourcesManager(object):
 325     """Tempest resource manager."""
 326     def __init__(self, **kwargs):
 327         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 328         self.guid = '-' + str(uuid.uuid4())
 329         self.creators = list()
 330         self.cirros_image_config = getattr(
 331             config.CONF, 'snaps_images_cirros', None)
 332
 333     def _create_project(self):
 334         """Create project for tests."""
 335         project_creator = deploy_utils.create_project(
 336             self.os_creds, ProjectConfig(
 337                 name=getattr(
 338                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 339                 description=getattr(
 340                     config.CONF, 'tempest_identity_tenant_description')))
 341         if project_creator is None or project_creator.get_project() is None:
 342             raise Exception("Failed to create tenant")
 343         self.creators.append(project_creator)
 344         return project_creator.get_project().id
 345
 346     def _create_user(self):
 347         """Create user for tests."""
 348         user_creator = deploy_utils.create_user(
 349             self.os_creds, UserConfig(
 350                 name=getattr(
 351                     config.CONF, 'tempest_identity_user_name') + self.guid,
 352                 password=getattr(
 353                     config.CONF, 'tempest_identity_user_password'),
 354                 project_name=getattr(
 355                     config.CONF, 'tempest_identity_tenant_name') + self.guid))
 356         if user_creator is None or user_creator.get_user() is None:
 357             raise Exception("Failed to create user")
 358         self.creators.append(user_creator)
 359         return user_creator.get_user().id
 360
 361     def _create_network(self, project_name):
 362         """Create network for tests."""
 363         tempest_network_type = None
 364         tempest_physical_network = None
 365         tempest_segmentation_id = None
 366
 367         tempest_network_type = getattr(
 368             config.CONF, 'tempest_network_type', None)
 369         tempest_physical_network = getattr(
 370             config.CONF, 'tempest_physical_network', None)
 371         tempest_segmentation_id = getattr(
 372             config.CONF, 'tempest_segmentation_id', None)
 373         tempest_net_name = getattr(
 374             config.CONF, 'tempest_private_net_name') + self.guid
 375
 376         network_creator = deploy_utils.create_network(
 377             self.os_creds, NetworkConfig(
 378                 name=tempest_net_name,
 379                 project_name=project_name,
 380                 network_type=tempest_network_type,
 381                 physical_network=tempest_physical_network,
 382                 segmentation_id=tempest_segmentation_id,
 383                 subnet_settings=[SubnetConfig(
 384                     name=getattr(
 385                         config.CONF,
 386                         'tempest_private_subnet_name') + self.guid,
 387                     project_name=project_name,
 388                     cidr=getattr(
 389                         config.CONF, 'tempest_private_subnet_cidr'))]))
 390         if network_creator is None or network_creator.get_network() is None:
 391             raise Exception("Failed to create private network")
 392         self.creators.append(network_creator)
 393         return tempest_net_name
 394
 395     def _create_image(self, name):
 396         """Create image for tests"""
 397         os_image_settings = openstack_tests.cirros_image_settings(
 398             name, public=True,
 399             image_metadata=self.cirros_image_config)
 400         image_creator = deploy_utils.create_image(
 401             self.os_creds, os_image_settings)
 402         if image_creator is None:
 403             raise Exception('Failed to create image')
 404         self.creators.append(image_creator)
 405         return image_creator.get_image().id
 406
 407     def _create_flavor(self, name):
 408         """Create flavor for tests."""
 409         scenario = env.get('DEPLOY_SCENARIO')
 410         flavor_metadata = None
 411         if 'ovs' in scenario or 'fdio' in scenario:
 412             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 413         flavor_creator = OpenStackFlavor(
 414             self.os_creds, FlavorConfig(
 415                 name=name,
 416                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 417                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 418                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 419                 metadata=flavor_metadata))
 420         flavor = flavor_creator.create()
 421         if flavor is None:
 422             raise Exception('Failed to create flavor')
 423         self.creators.append(flavor_creator)
 424         return flavor.id
 425
 426     def create(self, use_custom_images=False, use_custom_flavors=False,
 427                create_project=False):
 428         """Create resources for Tempest test suite."""
 429         result = {
 430             'tempest_net_name': None,
 431             'image_id': None,
 432             'image_id_alt': None,
 433             'flavor_id': None,
 434             'flavor_id_alt': None
 435         }
 436         project_name = None
 437
 438         if create_project:
 439             LOGGER.debug("Creating project and user for Tempest suite")
 440             project_name = getattr(
 441                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 442             result['project_id'] = self._create_project()
 443             result['user_id'] = self._create_user()
 444             result['tenant_id'] = result['project_id']  # for compatibility
 445
 446         LOGGER.debug("Creating private network for Tempest suite")
 447         result['tempest_net_name'] = self._create_network(project_name)
 448
 449         LOGGER.debug("Creating image for Tempest suite")
 450         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 451         result['image_id'] = self._create_image(image_name)
 452
 453         if use_custom_images:
 454             LOGGER.debug("Creating 2nd image for Tempest suite")
 455             image_name = getattr(
 456                 config.CONF, 'openstack_image_name_alt') + self.guid
 457             result['image_id_alt'] = self._create_image(image_name)
 458
 459         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 460                 use_custom_flavors):
 461             LOGGER.info("Creating flavor for Tempest suite")
 462             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 463             result['flavor_id'] = self._create_flavor(name)
 464
 465         if use_custom_flavors:
 466             LOGGER.info("Creating 2nd flavor for Tempest suite")
 467             scenario = env.get('DEPLOY_SCENARIO')
 468             if 'ovs' in scenario or 'fdio' in scenario:
 469                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 470             name = getattr(
 471                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 472             result['flavor_id_alt'] = self._create_flavor(name)
 473
 474         return result
 475
 476     def cleanup(self):
 477         """
 478         Cleanup all OpenStack objects. Should be called on completion.
 479         """
 480         for creator in reversed(self.creators):
 481             try:
 482                 creator.clean()
 483             except Exception as err:  # pylint: disable=broad-except
 484                 LOGGER.error('Unexpected error cleaning - %s', err)