functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     TEMPEST_RESULTS_DIR = os.path.join(
  48         getattr(config.CONF, 'dir_results'), 'tempest')
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  63         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  64         self.list = os.path.join(self.res_dir, 'test_list.txt')
  65
  66     @staticmethod
  67     def read_file(filename):
  68         """Read file and return content as a stripped list."""
  69         with open(filename) as src:
  70             return [line.strip() for line in src.readlines()]
  71
  72     @staticmethod
  73     def get_verifier_result(verif_id):
  74         """Retrieve verification results."""
  75         result = {
  76             'num_tests': 0,
  77             'num_success': 0,
  78             'num_failures': 0,
  79             'num_skipped': 0
  80         }
  81         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  82         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  83         proc = subprocess.Popen(cmd,
  84                                 stdout=subprocess.PIPE,
  85                                 stderr=subprocess.STDOUT)
  86         for line in proc.stdout:
  87             new_line = line.replace(' ', '').split('|')
  88             if 'Tests' in new_line:
  89                 break
  90             LOGGER.info(line)
  91             if 'Testscount' in new_line:
  92                 result['num_tests'] = int(new_line[2])
  93             elif 'Success' in new_line:
  94                 result['num_success'] = int(new_line[2])
  95             elif 'Skipped' in new_line:
  96                 result['num_skipped'] = int(new_line[2])
  97             elif 'Failures' in new_line:
  98                 result['num_failures'] = int(new_line[2])
  99         return result
 100
 101     def generate_test_list(self, verifier_repo_dir):
 102         """Generate test list based on the test mode."""
 103         LOGGER.debug("Generating test case list...")
 104         if self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, self.list)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 114             elif self.mode == 'full':
 115                 testr_mode = r"'^tempest\.'"
 116             else:
 117                 testr_mode = self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(verifier_repo_dir,
 121                                   testr_mode,
 122                                   self.list))
 123             functest_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search('Starting verification', line):
 188                     LOGGER.info(line.replace('\n', ''))
 189                     first_pos = line.index("UUID=") + len("UUID=")
 190                     last_pos = line.index(") for deployment")
 191                     self.verification_id = line[first_pos:last_pos]
 192                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 193                 f_stdout.write(line)
 194         proc.wait()
 195
 196         f_stdout.close()
 197         f_stderr.close()
 198
 199         if self.verification_id is None:
 200             raise Exception('Verification UUID not found')
 201
 202     def parse_verifier_result(self):
 203         """Parse and save test results."""
 204         stat = self.get_verifier_result(self.verification_id)
 205         try:
 206             num_executed = stat['num_tests'] - stat['num_skipped']
 207             try:
 208                 self.result = 100 * stat['num_success'] / num_executed
 209             except ZeroDivisionError:
 210                 self.result = 0
 211                 if stat['num_tests'] > 0:
 212                     LOGGER.info("All tests have been skipped")
 213                 else:
 214                     LOGGER.error("No test has been executed")
 215                     return
 216
 217             with open(os.path.join(self.res_dir,
 218                                    "tempest.log"), 'r') as logfile:
 219                 output = logfile.read()
 220
 221             success_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 223                 success_testcases.append(match)
 224             failed_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 226                 failed_testcases.append(match)
 227             skipped_testcases = []
 228             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests": stat['num_tests'],
 232                             "failures": stat['num_failures'],
 233                             "success": success_testcases,
 234                             "skipped": skipped_testcases,
 235                             "errors": failed_testcases}
 236         except Exception:  # pylint: disable=broad-except
 237             self.result = 0
 238
 239         LOGGER.info("Tempest %s success_rate is %s%%",
 240                     self.case_name, self.result)
 241
 242     def generate_report(self):
 243         """Generate verification report."""
 244         html_file = os.path.join(self.res_dir,
 245                                  "tempest-report.html")
 246         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 247                self.verification_id, "--to", html_file]
 248         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 249                          stderr=subprocess.STDOUT)
 250
 251     def run(self, **kwargs):
 252
 253         self.start_time = time.time()
 254         try:
 255             if not os.path.exists(self.res_dir):
 256                 os.makedirs(self.res_dir)
 257             resources = self.resources.create()
 258             compute_cnt = snaps_utils.get_active_compute_cnt(
 259                 self.resources.os_creds)
 260             conf_utils.configure_tempest(
 261                 self.deployment_dir, self.res_dir,
 262                 network_name=resources.get("network_name"),
 263                 image_id=resources.get("image_id"),
 264                 flavor_id=resources.get("flavor_id"),
 265                 compute_cnt=compute_cnt)
 266             self.generate_test_list(self.verifier_repo_dir)
 267             self.apply_tempest_blacklist()
 268             self.run_verifier_tests()
 269             self.parse_verifier_result()
 270             self.generate_report()
 271             res = testcase.TestCase.EX_OK
 272         except Exception as err:  # pylint: disable=broad-except
 273             LOGGER.error('Error with run: %s', err)
 274             res = testcase.TestCase.EX_RUN_ERROR
 275         finally:
 276             self.resources.cleanup()
 277
 278         self.stop_time = time.time()
 279         return res
 280
 281
 282 class TempestSmokeSerial(TempestCommon):
 283     """Tempest smoke serial testcase implementation."""
 284     def __init__(self, **kwargs):
 285         if "case_name" not in kwargs:
 286             kwargs["case_name"] = 'tempest_smoke_serial'
 287         TempestCommon.__init__(self, **kwargs)
 288         self.mode = "smoke"
 289         self.option = ["--concurrency", "1"]
 290
 291
 292 class TempestSmokeParallel(TempestCommon):
 293     """Tempest smoke parallel testcase implementation."""
 294     def __init__(self, **kwargs):
 295         if "case_name" not in kwargs:
 296             kwargs["case_name"] = 'tempest_smoke_parallel'
 297         TempestCommon.__init__(self, **kwargs)
 298         self.mode = "smoke"
 299
 300
 301 class TempestFullParallel(TempestCommon):
 302     """Tempest full parallel testcase implementation."""
 303     def __init__(self, **kwargs):
 304         if "case_name" not in kwargs:
 305             kwargs["case_name"] = 'tempest_full_parallel'
 306         TempestCommon.__init__(self, **kwargs)
 307         self.mode = "full"
 308
 309
 310 class TempestCustom(TempestCommon):
 311     """Tempest custom testcase implementation."""
 312     def __init__(self, **kwargs):
 313         if "case_name" not in kwargs:
 314             kwargs["case_name"] = 'tempest_custom'
 315         TempestCommon.__init__(self, **kwargs)
 316         self.mode = "custom"
 317         self.option = ["--concurrency", "1"]
 318
 319
 320 class TempestDefcore(TempestCommon):
 321     """Tempest Defcore testcase implementation."""
 322     def __init__(self, **kwargs):
 323         if "case_name" not in kwargs:
 324             kwargs["case_name"] = 'tempest_defcore'
 325         TempestCommon.__init__(self, **kwargs)
 326         self.mode = "defcore"
 327         self.option = ["--concurrency", "1"]
 328
 329
 330 class TempestResourcesManager(object):
 331     """Tempest resource manager."""
 332     def __init__(self, **kwargs):
 333         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 334         self.guid = '-' + str(uuid.uuid4())
 335         self.creators = list()
 336         self.cirros_image_config = getattr(
 337             config.CONF, 'snaps_images_cirros', None)
 338
 339     def _create_project(self):
 340         """Create project for tests."""
 341         project_creator = deploy_utils.create_project(
 342             self.os_creds, ProjectConfig(
 343                 name=getattr(
 344                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 345                 description=getattr(
 346                     config.CONF, 'tempest_identity_tenant_description'),
 347                 domain=self.os_creds.project_domain_name))
 348         if project_creator is None or project_creator.get_project() is None:
 349             raise Exception("Failed to create tenant")
 350         self.creators.append(project_creator)
 351         return project_creator.get_project().id
 352
 353     def _create_user(self):
 354         """Create user for tests."""
 355         user_creator = deploy_utils.create_user(
 356             self.os_creds, UserConfig(
 357                 name=getattr(
 358                     config.CONF, 'tempest_identity_user_name') + self.guid,
 359                 password=getattr(
 360                     config.CONF, 'tempest_identity_user_password'),
 361                 project_name=getattr(
 362                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 363                 domain_name=self.os_creds.user_domain_name))
 364         if user_creator is None or user_creator.get_user() is None:
 365             raise Exception("Failed to create user")
 366         self.creators.append(user_creator)
 367         return user_creator.get_user().id
 368
 369     def _create_network(self, project_name):
 370         """Create network for tests."""
 371         tempest_network_type = None
 372         tempest_physical_network = None
 373         tempest_segmentation_id = None
 374
 375         tempest_network_type = getattr(
 376             config.CONF, 'tempest_network_type', None)
 377         tempest_physical_network = getattr(
 378             config.CONF, 'tempest_physical_network', None)
 379         tempest_segmentation_id = getattr(
 380             config.CONF, 'tempest_segmentation_id', None)
 381         tempest_net_name = getattr(
 382             config.CONF, 'tempest_private_net_name') + self.guid
 383
 384         network_creator = deploy_utils.create_network(
 385             self.os_creds, NetworkConfig(
 386                 name=tempest_net_name,
 387                 project_name=project_name,
 388                 network_type=tempest_network_type,
 389                 physical_network=tempest_physical_network,
 390                 segmentation_id=tempest_segmentation_id,
 391                 subnet_settings=[SubnetConfig(
 392                     name=getattr(
 393                         config.CONF,
 394                         'tempest_private_subnet_name') + self.guid,
 395                     project_name=project_name,
 396                     cidr=getattr(
 397                         config.CONF, 'tempest_private_subnet_cidr'),
 398                     dns_nameservers=[env.get('NAMESERVER')])]))
 399         if network_creator is None or network_creator.get_network() is None:
 400             raise Exception("Failed to create private network")
 401         self.creators.append(network_creator)
 402         return tempest_net_name
 403
 404     def _create_image(self, name):
 405         """Create image for tests"""
 406         os_image_settings = openstack_tests.cirros_image_settings(
 407             name, public=True,
 408             image_metadata=self.cirros_image_config)
 409         image_creator = deploy_utils.create_image(
 410             self.os_creds, os_image_settings)
 411         if image_creator is None:
 412             raise Exception('Failed to create image')
 413         self.creators.append(image_creator)
 414         return image_creator.get_image().id
 415
 416     def _create_flavor(self, name):
 417         """Create flavor for tests."""
 418         scenario = env.get('DEPLOY_SCENARIO')
 419         flavor_metadata = None
 420         if 'ovs' in scenario or 'fdio' in scenario:
 421             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 422         flavor_creator = OpenStackFlavor(
 423             self.os_creds, FlavorConfig(
 424                 name=name,
 425                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 426                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 427                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 428                 metadata=flavor_metadata))
 429         flavor = flavor_creator.create()
 430         if flavor is None:
 431             raise Exception('Failed to create flavor')
 432         self.creators.append(flavor_creator)
 433         return flavor.id
 434
 435     def create(self, use_custom_images=False, use_custom_flavors=False,
 436                create_project=False):
 437         """Create resources for Tempest test suite."""
 438         result = {
 439             'tempest_net_name': None,
 440             'image_id': None,
 441             'image_id_alt': None,
 442             'flavor_id': None,
 443             'flavor_id_alt': None
 444         }
 445         project_name = None
 446
 447         if create_project:
 448             LOGGER.debug("Creating project and user for Tempest suite")
 449             project_name = getattr(
 450                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 451             result['project_id'] = self._create_project()
 452             result['user_id'] = self._create_user()
 453             result['tenant_id'] = result['project_id']  # for compatibility
 454
 455         LOGGER.debug("Creating private network for Tempest suite")
 456         result['tempest_net_name'] = self._create_network(project_name)
 457
 458         LOGGER.debug("Creating image for Tempest suite")
 459         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 460         result['image_id'] = self._create_image(image_name)
 461
 462         if use_custom_images:
 463             LOGGER.debug("Creating 2nd image for Tempest suite")
 464             image_name = getattr(
 465                 config.CONF, 'openstack_image_name_alt') + self.guid
 466             result['image_id_alt'] = self._create_image(image_name)
 467
 468         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 469                 use_custom_flavors):
 470             LOGGER.info("Creating flavor for Tempest suite")
 471             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 472             result['flavor_id'] = self._create_flavor(name)
 473
 474         if use_custom_flavors:
 475             LOGGER.info("Creating 2nd flavor for Tempest suite")
 476             scenario = env.get('DEPLOY_SCENARIO')
 477             if 'ovs' in scenario or 'fdio' in scenario:
 478                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 479             name = getattr(
 480                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 481             result['flavor_id_alt'] = self._create_flavor(name)
 482
 483         return result
 484
 485     def cleanup(self):
 486         """
 487         Cleanup all OpenStack objects. Should be called on completion.
 488         """
 489         for creator in reversed(self.creators):
 490             try:
 491                 creator.clean()
 492             except Exception as err:  # pylint: disable=broad-except
 493                 LOGGER.error('Unexpected error cleaning - %s', err)