functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     TEMPEST_RESULTS_DIR = os.path.join(
  48         getattr(config.CONF, 'dir_results'), 'tempest')
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  63         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  64         self.list = os.path.join(self.res_dir, 'test_list.txt')
  65
  66     @staticmethod
  67     def read_file(filename):
  68         """Read file and return content as a stripped list."""
  69         with open(filename) as src:
  70             return [line.strip() for line in src.readlines()]
  71
  72     @staticmethod
  73     def get_verifier_result(verif_id):
  74         """Retrieve verification results."""
  75         result = {
  76             'num_tests': 0,
  77             'num_success': 0,
  78             'num_failures': 0,
  79             'num_skipped': 0
  80         }
  81         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  82         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  83         proc = subprocess.Popen(cmd,
  84                                 stdout=subprocess.PIPE,
  85                                 stderr=subprocess.STDOUT)
  86         for line in proc.stdout:
  87             new_line = line.replace(' ', '').split('|')
  88             if 'Tests' in new_line:
  89                 break
  90             LOGGER.info(line)
  91             if 'Testscount' in new_line:
  92                 result['num_tests'] = int(new_line[2])
  93             elif 'Success' in new_line:
  94                 result['num_success'] = int(new_line[2])
  95             elif 'Skipped' in new_line:
  96                 result['num_skipped'] = int(new_line[2])
  97             elif 'Failures' in new_line:
  98                 result['num_failures'] = int(new_line[2])
  99         return result
 100
 101     def generate_test_list(self, verifier_repo_dir):
 102         """Generate test list based on the test mode."""
 103         LOGGER.debug("Generating test case list...")
 104         if self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, self.raw_list)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 114             elif self.mode == 'full':
 115                 testr_mode = r"'^tempest\.'"
 116             else:
 117                 testr_mode = self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(verifier_repo_dir,
 121                                   testr_mode,
 122                                   self.raw_list))
 123             functest_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         cases_file = self.read_file(self.raw_list)
 129         result_file = open(self.list, 'w')
 130         black_tests = []
 131         try:
 132             installer_type = env.get('INSTALLER_TYPE')
 133             deploy_scenario = env.get('DEPLOY_SCENARIO')
 134             if bool(installer_type) * bool(deploy_scenario):
 135                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 136                 # file
 137                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 138                 black_list_yaml = yaml.safe_load(black_list_file)
 139                 black_list_file.close()
 140                 for item in black_list_yaml:
 141                     scenarios = item['scenarios']
 142                     installers = item['installers']
 143                     if (deploy_scenario in scenarios and
 144                             installer_type in installers):
 145                         tests = item['tests']
 146                         for test in tests:
 147                             black_tests.append(test)
 148                         break
 149         except Exception:  # pylint: disable=broad-except
 150             black_tests = []
 151             LOGGER.debug("Tempest blacklist file does not exist.")
 152
 153         for cases_line in cases_file:
 154             for black_tests_line in black_tests:
 155                 if black_tests_line in cases_line:
 156                     break
 157             else:
 158                 result_file.write(str(cases_line) + '\n')
 159         result_file.close()
 160
 161     def run_verifier_tests(self):
 162         """Execute tempest test cases."""
 163         cmd = ["rally", "verify", "start", "--load-list",
 164                self.list]
 165         cmd.extend(self.option)
 166         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 167
 168         f_stdout = open(
 169             os.path.join(self.res_dir, "tempest.log"), 'w+')
 170         f_stderr = open(
 171             os.path.join(self.res_dir,
 172                          "tempest-error.log"), 'w+')
 173
 174         proc = subprocess.Popen(
 175             cmd,
 176             stdout=subprocess.PIPE,
 177             stderr=f_stderr,
 178             bufsize=1)
 179
 180         with proc.stdout:
 181             for line in iter(proc.stdout.readline, b''):
 182                 if re.search(r"\} tempest\.", line):
 183                     LOGGER.info(line.replace('\n', ''))
 184                 elif re.search('Starting verification', line):
 185                     LOGGER.info(line.replace('\n', ''))
 186                     first_pos = line.index("UUID=") + len("UUID=")
 187                     last_pos = line.index(") for deployment")
 188                     self.verification_id = line[first_pos:last_pos]
 189                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 190                 f_stdout.write(line)
 191         proc.wait()
 192
 193         f_stdout.close()
 194         f_stderr.close()
 195
 196         if self.verification_id is None:
 197             raise Exception('Verification UUID not found')
 198
 199     def parse_verifier_result(self):
 200         """Parse and save test results."""
 201         stat = self.get_verifier_result(self.verification_id)
 202         try:
 203             num_executed = stat['num_tests'] - stat['num_skipped']
 204             try:
 205                 self.result = 100 * stat['num_success'] / num_executed
 206             except ZeroDivisionError:
 207                 self.result = 0
 208                 if stat['num_tests'] > 0:
 209                     LOGGER.info("All tests have been skipped")
 210                 else:
 211                     LOGGER.error("No test has been executed")
 212                     return
 213
 214             with open(os.path.join(self.res_dir,
 215                                    "tempest.log"), 'r') as logfile:
 216                 output = logfile.read()
 217
 218             success_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 220                 success_testcases.append(match)
 221             failed_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 223                 failed_testcases.append(match)
 224             skipped_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 226                 skipped_testcases.append(match)
 227
 228             self.details = {"tests": stat['num_tests'],
 229                             "failures": stat['num_failures'],
 230                             "success": success_testcases,
 231                             "skipped": skipped_testcases,
 232                             "errors": failed_testcases}
 233         except Exception:  # pylint: disable=broad-except
 234             self.result = 0
 235
 236         LOGGER.info("Tempest %s success_rate is %s%%",
 237                     self.case_name, self.result)
 238
 239     def generate_report(self):
 240         """Generate verification report."""
 241         html_file = os.path.join(self.res_dir,
 242                                  "tempest-report.html")
 243         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 244                self.verification_id, "--to", html_file]
 245         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 246                          stderr=subprocess.STDOUT)
 247
 248     def run(self, **kwargs):
 249
 250         self.start_time = time.time()
 251         try:
 252             if not os.path.exists(self.res_dir):
 253                 os.makedirs(self.res_dir)
 254             resources = self.resources.create()
 255             compute_cnt = snaps_utils.get_active_compute_cnt(
 256                 self.resources.os_creds)
 257             conf_utils.configure_tempest(
 258                 self.deployment_dir, self.res_dir,
 259                 network_name=resources.get("network_name"),
 260                 image_id=resources.get("image_id"),
 261                 flavor_id=resources.get("flavor_id"),
 262                 compute_cnt=compute_cnt)
 263             self.generate_test_list(self.verifier_repo_dir)
 264             self.apply_tempest_blacklist()
 265             self.run_verifier_tests()
 266             self.parse_verifier_result()
 267             self.generate_report()
 268             res = testcase.TestCase.EX_OK
 269         except Exception as err:  # pylint: disable=broad-except
 270             LOGGER.error('Error with run: %s', err)
 271             res = testcase.TestCase.EX_RUN_ERROR
 272         finally:
 273             self.resources.cleanup()
 274
 275         self.stop_time = time.time()
 276         return res
 277
 278
 279 class TempestSmokeSerial(TempestCommon):
 280     """Tempest smoke serial testcase implementation."""
 281     def __init__(self, **kwargs):
 282         if "case_name" not in kwargs:
 283             kwargs["case_name"] = 'tempest_smoke_serial'
 284         TempestCommon.__init__(self, **kwargs)
 285         self.mode = "smoke"
 286         self.option = ["--concurrency", "1"]
 287
 288
 289 class TempestSmokeParallel(TempestCommon):
 290     """Tempest smoke parallel testcase implementation."""
 291     def __init__(self, **kwargs):
 292         if "case_name" not in kwargs:
 293             kwargs["case_name"] = 'tempest_smoke_parallel'
 294         TempestCommon.__init__(self, **kwargs)
 295         self.mode = "smoke"
 296
 297
 298 class TempestFullParallel(TempestCommon):
 299     """Tempest full parallel testcase implementation."""
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'tempest_full_parallel'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.mode = "full"
 305
 306
 307 class TempestCustom(TempestCommon):
 308     """Tempest custom testcase implementation."""
 309     def __init__(self, **kwargs):
 310         if "case_name" not in kwargs:
 311             kwargs["case_name"] = 'tempest_custom'
 312         TempestCommon.__init__(self, **kwargs)
 313         self.mode = "custom"
 314         self.option = ["--concurrency", "1"]
 315
 316
 317 class TempestDefcore(TempestCommon):
 318     """Tempest Defcore testcase implementation."""
 319     def __init__(self, **kwargs):
 320         if "case_name" not in kwargs:
 321             kwargs["case_name"] = 'tempest_defcore'
 322         TempestCommon.__init__(self, **kwargs)
 323         self.mode = "defcore"
 324         self.option = ["--concurrency", "1"]
 325
 326
 327 class TempestResourcesManager(object):
 328     """Tempest resource manager."""
 329     def __init__(self, **kwargs):
 330         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 331         self.guid = '-' + str(uuid.uuid4())
 332         self.creators = list()
 333         self.cirros_image_config = getattr(
 334             config.CONF, 'snaps_images_cirros', None)
 335
 336     def _create_project(self):
 337         """Create project for tests."""
 338         project_creator = deploy_utils.create_project(
 339             self.os_creds, ProjectConfig(
 340                 name=getattr(
 341                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 342                 description=getattr(
 343                     config.CONF, 'tempest_identity_tenant_description'),
 344                 domain=self.os_creds.project_domain_name))
 345         if project_creator is None or project_creator.get_project() is None:
 346             raise Exception("Failed to create tenant")
 347         self.creators.append(project_creator)
 348         return project_creator.get_project().id
 349
 350     def _create_user(self):
 351         """Create user for tests."""
 352         user_creator = deploy_utils.create_user(
 353             self.os_creds, UserConfig(
 354                 name=getattr(
 355                     config.CONF, 'tempest_identity_user_name') + self.guid,
 356                 password=getattr(
 357                     config.CONF, 'tempest_identity_user_password'),
 358                 project_name=getattr(
 359                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 360                 domain_name=self.os_creds.user_domain_name))
 361         if user_creator is None or user_creator.get_user() is None:
 362             raise Exception("Failed to create user")
 363         self.creators.append(user_creator)
 364         return user_creator.get_user().id
 365
 366     def _create_network(self, project_name):
 367         """Create network for tests."""
 368         tempest_network_type = None
 369         tempest_physical_network = None
 370         tempest_segmentation_id = None
 371
 372         tempest_network_type = getattr(
 373             config.CONF, 'tempest_network_type', None)
 374         tempest_physical_network = getattr(
 375             config.CONF, 'tempest_physical_network', None)
 376         tempest_segmentation_id = getattr(
 377             config.CONF, 'tempest_segmentation_id', None)
 378         tempest_net_name = getattr(
 379             config.CONF, 'tempest_private_net_name') + self.guid
 380
 381         network_creator = deploy_utils.create_network(
 382             self.os_creds, NetworkConfig(
 383                 name=tempest_net_name,
 384                 project_name=project_name,
 385                 network_type=tempest_network_type,
 386                 physical_network=tempest_physical_network,
 387                 segmentation_id=tempest_segmentation_id,
 388                 subnet_settings=[SubnetConfig(
 389                     name=getattr(
 390                         config.CONF,
 391                         'tempest_private_subnet_name') + self.guid,
 392                     project_name=project_name,
 393                     cidr=getattr(
 394                         config.CONF, 'tempest_private_subnet_cidr'),
 395                     dns_nameservers=[env.get('NAMESERVER')])]))
 396         if network_creator is None or network_creator.get_network() is None:
 397             raise Exception("Failed to create private network")
 398         self.creators.append(network_creator)
 399         return tempest_net_name
 400
 401     def _create_image(self, name):
 402         """Create image for tests"""
 403         os_image_settings = openstack_tests.cirros_image_settings(
 404             name, public=True,
 405             image_metadata=self.cirros_image_config)
 406         image_creator = deploy_utils.create_image(
 407             self.os_creds, os_image_settings)
 408         if image_creator is None:
 409             raise Exception('Failed to create image')
 410         self.creators.append(image_creator)
 411         return image_creator.get_image().id
 412
 413     def _create_flavor(self, name):
 414         """Create flavor for tests."""
 415         scenario = env.get('DEPLOY_SCENARIO')
 416         flavor_metadata = None
 417         if 'ovs' in scenario or 'fdio' in scenario:
 418             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 419         flavor_creator = OpenStackFlavor(
 420             self.os_creds, FlavorConfig(
 421                 name=name,
 422                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 423                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 424                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 425                 metadata=flavor_metadata))
 426         flavor = flavor_creator.create()
 427         if flavor is None:
 428             raise Exception('Failed to create flavor')
 429         self.creators.append(flavor_creator)
 430         return flavor.id
 431
 432     def create(self, use_custom_images=False, use_custom_flavors=False,
 433                create_project=False):
 434         """Create resources for Tempest test suite."""
 435         result = {
 436             'tempest_net_name': None,
 437             'image_id': None,
 438             'image_id_alt': None,
 439             'flavor_id': None,
 440             'flavor_id_alt': None
 441         }
 442         project_name = None
 443
 444         if create_project:
 445             LOGGER.debug("Creating project and user for Tempest suite")
 446             project_name = getattr(
 447                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 448             result['project_id'] = self._create_project()
 449             result['user_id'] = self._create_user()
 450             result['tenant_id'] = result['project_id']  # for compatibility
 451
 452         LOGGER.debug("Creating private network for Tempest suite")
 453         result['tempest_net_name'] = self._create_network(project_name)
 454
 455         LOGGER.debug("Creating image for Tempest suite")
 456         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 457         result['image_id'] = self._create_image(image_name)
 458
 459         if use_custom_images:
 460             LOGGER.debug("Creating 2nd image for Tempest suite")
 461             image_name = getattr(
 462                 config.CONF, 'openstack_image_name_alt') + self.guid
 463             result['image_id_alt'] = self._create_image(image_name)
 464
 465         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 466                 use_custom_flavors):
 467             LOGGER.info("Creating flavor for Tempest suite")
 468             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 469             result['flavor_id'] = self._create_flavor(name)
 470
 471         if use_custom_flavors:
 472             LOGGER.info("Creating 2nd flavor for Tempest suite")
 473             scenario = env.get('DEPLOY_SCENARIO')
 474             if 'ovs' in scenario or 'fdio' in scenario:
 475                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 476             name = getattr(
 477                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 478             result['flavor_id_alt'] = self._create_flavor(name)
 479
 480         return result
 481
 482     def cleanup(self):
 483         """
 484         Cleanup all OpenStack objects. Should be called on completion.
 485         """
 486         for creator in reversed(self.creators):
 487             try:
 488                 creator.clean()
 489             except Exception as err:  # pylint: disable=broad-except
 490                 LOGGER.error('Unexpected error cleaning - %s', err)