functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack.create_flavor import OpenStackFlavor
  28 from snaps.openstack.tests import openstack_tests
  29 from snaps.openstack.utils import deploy_utils
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.opnfv_tests.openstack.snaps import snaps_utils
  34 from functest.opnfv_tests.openstack.tempest import conf_utils
  35 from functest.utils import config
  36 from functest.utils import env
  37
  38 LOGGER = logging.getLogger(__name__)
  39
  40
  41 class TempestCommon(testcase.TestCase):
  42     # pylint: disable=too-many-instance-attributes
  43     """TempestCommon testcases implementation class."""
  44
  45     TEMPEST_RESULTS_DIR = os.path.join(
  46         getattr(config.CONF, 'dir_results'), 'tempest')
  47
  48     def __init__(self, **kwargs):
  49         super(TempestCommon, self).__init__(**kwargs)
  50         self.resources = TempestResourcesManager(**kwargs)
  51         self.mode = ""
  52         self.option = []
  53         self.verifier_id = conf_utils.get_verifier_id()
  54         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  55             self.verifier_id)
  56         self.deployment_id = conf_utils.get_verifier_deployment_id()
  57         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  58             self.verifier_id, self.deployment_id)
  59         self.verification_id = None
  60         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  61         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  62         self.list = os.path.join(self.res_dir, 'test_list.txt')
  63         self.conf_file = None
  64
  65     @staticmethod
  66     def read_file(filename):
  67         """Read file and return content as a stripped list."""
  68         with open(filename) as src:
  69             return [line.strip() for line in src.readlines()]
  70
  71     @staticmethod
  72     def get_verifier_result(verif_id):
  73         """Retrieve verification results."""
  74         result = {
  75             'num_tests': 0,
  76             'num_success': 0,
  77             'num_failures': 0,
  78             'num_skipped': 0
  79         }
  80         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  81         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  82         proc = subprocess.Popen(cmd,
  83                                 stdout=subprocess.PIPE,
  84                                 stderr=subprocess.STDOUT)
  85         for line in proc.stdout:
  86             new_line = line.replace(' ', '').split('|')
  87             if 'Tests' in new_line:
  88                 break
  89             LOGGER.info(line)
  90             if 'Testscount' in new_line:
  91                 result['num_tests'] = int(new_line[2])
  92             elif 'Success' in new_line:
  93                 result['num_success'] = int(new_line[2])
  94             elif 'Skipped' in new_line:
  95                 result['num_skipped'] = int(new_line[2])
  96             elif 'Failures' in new_line:
  97                 result['num_failures'] = int(new_line[2])
  98         return result
  99
 100     def generate_test_list(self):
 101         """Generate test list based on the test mode."""
 102         LOGGER.debug("Generating test case list...")
 103         if self.mode == 'custom':
 104             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 105                 shutil.copyfile(
 106                     conf_utils.TEMPEST_CUSTOM, self.list)
 107             else:
 108                 raise Exception("Tempest test list file %s NOT found."
 109                                 % conf_utils.TEMPEST_CUSTOM)
 110         else:
 111             if self.mode == 'smoke':
 112                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 113             elif self.mode == 'full':
 114                 testr_mode = r"'^tempest\.'"
 115             else:
 116                 testr_mode = self.mode
 117             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 118                 self.verifier_repo_dir, testr_mode, self.list)
 119             output = subprocess.check_output(cmd, shell=True)
 120             LOGGER.info("%s\n%s", cmd, output)
 121
 122     def apply_tempest_blacklist(self):
 123         """Exclude blacklisted test cases."""
 124         LOGGER.debug("Applying tempest blacklist...")
 125         if os.path.exists(self.raw_list):
 126             os.remove(self.raw_list)
 127         os.rename(self.list, self.raw_list)
 128         cases_file = self.read_file(self.raw_list)
 129         result_file = open(self.list, 'w')
 130         black_tests = []
 131         try:
 132             installer_type = env.get('INSTALLER_TYPE')
 133             deploy_scenario = env.get('DEPLOY_SCENARIO')
 134             if bool(installer_type) * bool(deploy_scenario):
 135                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 136                 # file
 137                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 138                 black_list_yaml = yaml.safe_load(black_list_file)
 139                 black_list_file.close()
 140                 for item in black_list_yaml:
 141                     scenarios = item['scenarios']
 142                     installers = item['installers']
 143                     if (deploy_scenario in scenarios and
 144                             installer_type in installers):
 145                         tests = item['tests']
 146                         for test in tests:
 147                             black_tests.append(test)
 148                         break
 149         except Exception:  # pylint: disable=broad-except
 150             black_tests = []
 151             LOGGER.debug("Tempest blacklist file does not exist.")
 152
 153         for cases_line in cases_file:
 154             for black_tests_line in black_tests:
 155                 if black_tests_line in cases_line:
 156                     break
 157             else:
 158                 result_file.write(str(cases_line) + '\n')
 159         result_file.close()
 160
 161     def run_verifier_tests(self):
 162         """Execute tempest test cases."""
 163         cmd = ["rally", "verify", "start", "--load-list",
 164                self.list]
 165         cmd.extend(self.option)
 166         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 167
 168         f_stdout = open(
 169             os.path.join(self.res_dir, "tempest.log"), 'w+')
 170         f_stderr = open(
 171             os.path.join(self.res_dir,
 172                          "tempest-error.log"), 'w+')
 173
 174         proc = subprocess.Popen(
 175             cmd,
 176             stdout=subprocess.PIPE,
 177             stderr=f_stderr,
 178             bufsize=1)
 179
 180         with proc.stdout:
 181             for line in iter(proc.stdout.readline, b''):
 182                 if re.search(r"\} tempest\.", line):
 183                     LOGGER.info(line.replace('\n', ''))
 184                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 185                     self.verification_id = re.search(
 186                         r'(?=\(UUID=(.*)\))', line).group(1)
 187                     LOGGER.info('Verification UUID: %s', self.verification_id)
 188                 f_stdout.write(line)
 189         proc.wait()
 190
 191         f_stdout.close()
 192         f_stderr.close()
 193
 194         if self.verification_id is None:
 195             raise Exception('Verification UUID not found')
 196
 197     def parse_verifier_result(self):
 198         """Parse and save test results."""
 199         stat = self.get_verifier_result(self.verification_id)
 200         try:
 201             num_executed = stat['num_tests'] - stat['num_skipped']
 202             try:
 203                 self.result = 100 * stat['num_success'] / num_executed
 204             except ZeroDivisionError:
 205                 self.result = 0
 206                 if stat['num_tests'] > 0:
 207                     LOGGER.info("All tests have been skipped")
 208                 else:
 209                     LOGGER.error("No test has been executed")
 210                     return
 211
 212             with open(os.path.join(self.res_dir,
 213                                    "tempest.log"), 'r') as logfile:
 214                 output = logfile.read()
 215
 216             success_testcases = []
 217             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 218                 success_testcases.append(match)
 219             failed_testcases = []
 220             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail', output):
 221                 failed_testcases.append(match)
 222             skipped_testcases = []
 223             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 224                 skipped_testcases.append(match)
 225
 226             self.details = {"tests_number": stat['num_tests'],
 227                             "success_number": stat['num_success'],
 228                             "skipped_number": stat['num_skipped'],
 229                             "failures_number": stat['num_failures'],
 230                             "success": success_testcases,
 231                             "skipped": skipped_testcases,
 232                             "failures": failed_testcases}
 233         except Exception:  # pylint: disable=broad-except
 234             self.result = 0
 235
 236         LOGGER.info("Tempest %s success_rate is %s%%",
 237                     self.case_name, self.result)
 238
 239     def generate_report(self):
 240         """Generate verification report."""
 241         html_file = os.path.join(self.res_dir,
 242                                  "tempest-report.html")
 243         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 244                self.verification_id, "--to", html_file]
 245         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 246                          stderr=subprocess.STDOUT)
 247
 248     def configure(self, **kwargs):  # pylint: disable=unused-argument
 249         """
 250         Create all openstack resources for tempest-based testcases and write
 251         tempest.conf.
 252         """
 253         if not os.path.exists(self.res_dir):
 254             os.makedirs(self.res_dir)
 255         resources = self.resources.create()
 256         compute_cnt = snaps_utils.get_active_compute_cnt(
 257             self.resources.os_creds)
 258         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 259         conf_utils.configure_tempest_update_params(
 260             self.conf_file, self.res_dir,
 261             network_name=resources.get("network_name"),
 262             image_id=resources.get("image_id"),
 263             flavor_id=resources.get("flavor_id"),
 264             compute_cnt=compute_cnt)
 265
 266     def run(self, **kwargs):
 267         self.start_time = time.time()
 268         try:
 269             self.configure()
 270             self.generate_test_list()
 271             self.apply_tempest_blacklist()
 272             self.run_verifier_tests()
 273             self.parse_verifier_result()
 274             self.generate_report()
 275             res = testcase.TestCase.EX_OK
 276         except Exception:  # pylint: disable=broad-except
 277             LOGGER.exception('Error with run')
 278             res = testcase.TestCase.EX_RUN_ERROR
 279         finally:
 280             self.resources.cleanup()
 281         self.stop_time = time.time()
 282         return res
 283
 284
 285 class TempestSmokeSerial(TempestCommon):
 286     """Tempest smoke serial testcase implementation."""
 287     def __init__(self, **kwargs):
 288         if "case_name" not in kwargs:
 289             kwargs["case_name"] = 'tempest_smoke_serial'
 290         TempestCommon.__init__(self, **kwargs)
 291         self.mode = "smoke"
 292         self.option = ["--concurrency", "1"]
 293
 294
 295 class TempestNeutronTrunk(TempestCommon):
 296     """Tempest neutron trunk testcase implementation."""
 297     def __init__(self, **kwargs):
 298         if "case_name" not in kwargs:
 299             kwargs["case_name"] = 'neutron_trunk'
 300         TempestCommon.__init__(self, **kwargs)
 301         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 302         self.res_dir = os.path.join(
 303             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 304         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 305         self.list = os.path.join(self.res_dir, 'test_list.txt')
 306
 307     def configure(self, **kwargs):
 308         super(TempestNeutronTrunk, self).configure(**kwargs)
 309         rconfig = conf_utils.ConfigParser.RawConfigParser()
 310         rconfig.read(self.conf_file)
 311         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 312         with open(self.conf_file, 'wb') as config_file:
 313             rconfig.write(config_file)
 314
 315
 316 class TempestSmokeParallel(TempestCommon):
 317     """Tempest smoke parallel testcase implementation."""
 318     def __init__(self, **kwargs):
 319         if "case_name" not in kwargs:
 320             kwargs["case_name"] = 'tempest_smoke_parallel'
 321         TempestCommon.__init__(self, **kwargs)
 322         self.mode = "smoke"
 323
 324
 325 class TempestFullParallel(TempestCommon):
 326     """Tempest full parallel testcase implementation."""
 327     def __init__(self, **kwargs):
 328         if "case_name" not in kwargs:
 329             kwargs["case_name"] = 'tempest_full_parallel'
 330         TempestCommon.__init__(self, **kwargs)
 331         self.mode = "full"
 332
 333
 334 class TempestCustom(TempestCommon):
 335     """Tempest custom testcase implementation."""
 336     def __init__(self, **kwargs):
 337         if "case_name" not in kwargs:
 338             kwargs["case_name"] = 'tempest_custom'
 339         TempestCommon.__init__(self, **kwargs)
 340         self.mode = "custom"
 341         self.option = ["--concurrency", "1"]
 342
 343
 344 class TempestDefcore(TempestCommon):
 345     """Tempest Defcore testcase implementation."""
 346     def __init__(self, **kwargs):
 347         if "case_name" not in kwargs:
 348             kwargs["case_name"] = 'tempest_defcore'
 349         TempestCommon.__init__(self, **kwargs)
 350         self.mode = "defcore"
 351         self.option = ["--concurrency", "1"]
 352
 353
 354 class TempestResourcesManager(object):
 355     """Tempest resource manager."""
 356     def __init__(self, **kwargs):
 357         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 358         self.guid = '-' + str(uuid.uuid4())
 359         self.creators = list()
 360         self.cirros_image_config = getattr(
 361             config.CONF, 'snaps_images_cirros', None)
 362
 363     def _create_project(self):
 364         """Create project for tests."""
 365         project_creator = deploy_utils.create_project(
 366             self.os_creds, ProjectConfig(
 367                 name=getattr(
 368                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 369                 description=getattr(
 370                     config.CONF, 'tempest_identity_tenant_description'),
 371                 domain=self.os_creds.project_domain_name))
 372         if project_creator is None or project_creator.get_project() is None:
 373             raise Exception("Failed to create tenant")
 374         self.creators.append(project_creator)
 375         return project_creator.get_project().id
 376
 377     def _create_user(self):
 378         """Create user for tests."""
 379         user_creator = deploy_utils.create_user(
 380             self.os_creds, UserConfig(
 381                 name=getattr(
 382                     config.CONF, 'tempest_identity_user_name') + self.guid,
 383                 password=getattr(
 384                     config.CONF, 'tempest_identity_user_password'),
 385                 project_name=getattr(
 386                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 387                 domain_name=self.os_creds.user_domain_name))
 388         if user_creator is None or user_creator.get_user() is None:
 389             raise Exception("Failed to create user")
 390         self.creators.append(user_creator)
 391         return user_creator.get_user().id
 392
 393     def _create_network(self, project_name):
 394         """Create network for tests."""
 395         tempest_network_type = None
 396         tempest_physical_network = None
 397         tempest_segmentation_id = None
 398
 399         tempest_network_type = getattr(
 400             config.CONF, 'tempest_network_type', None)
 401         tempest_physical_network = getattr(
 402             config.CONF, 'tempest_physical_network', None)
 403         tempest_segmentation_id = getattr(
 404             config.CONF, 'tempest_segmentation_id', None)
 405         tempest_net_name = getattr(
 406             config.CONF, 'tempest_private_net_name') + self.guid
 407
 408         network_creator = deploy_utils.create_network(
 409             self.os_creds, NetworkConfig(
 410                 name=tempest_net_name,
 411                 project_name=project_name,
 412                 network_type=tempest_network_type,
 413                 physical_network=tempest_physical_network,
 414                 segmentation_id=tempest_segmentation_id,
 415                 subnet_settings=[SubnetConfig(
 416                     name=getattr(
 417                         config.CONF,
 418                         'tempest_private_subnet_name') + self.guid,
 419                     project_name=project_name,
 420                     cidr=getattr(
 421                         config.CONF, 'tempest_private_subnet_cidr'),
 422                     dns_nameservers=[env.get('NAMESERVER')])]))
 423         if network_creator is None or network_creator.get_network() is None:
 424             raise Exception("Failed to create private network")
 425         self.creators.append(network_creator)
 426         return tempest_net_name
 427
 428     def _create_image(self, name):
 429         """Create image for tests"""
 430         os_image_settings = openstack_tests.cirros_image_settings(
 431             name, public=True,
 432             image_metadata=self.cirros_image_config)
 433         image_creator = deploy_utils.create_image(
 434             self.os_creds, os_image_settings)
 435         if image_creator is None:
 436             raise Exception('Failed to create image')
 437         self.creators.append(image_creator)
 438         return image_creator.get_image().id
 439
 440     def _create_flavor(self, name):
 441         """Create flavor for tests."""
 442         flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
 443         flavor_creator = OpenStackFlavor(
 444             self.os_creds, FlavorConfig(
 445                 name=name,
 446                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 447                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 448                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 449                 metadata=flavor_metadata))
 450         flavor = flavor_creator.create()
 451         if flavor is None:
 452             raise Exception('Failed to create flavor')
 453         self.creators.append(flavor_creator)
 454         return flavor.id
 455
 456     def create(self, create_project=False):
 457         """Create resources for Tempest test suite."""
 458         result = {
 459             'tempest_net_name': None,
 460             'image_id': None,
 461             'image_id_alt': None,
 462             'flavor_id': None,
 463             'flavor_id_alt': None
 464         }
 465         project_name = None
 466
 467         if create_project:
 468             LOGGER.debug("Creating project and user for Tempest suite")
 469             project_name = getattr(
 470                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 471             result['project_id'] = self._create_project()
 472             result['user_id'] = self._create_user()
 473             result['tenant_id'] = result['project_id']  # for compatibility
 474
 475         LOGGER.debug("Creating private network for Tempest suite")
 476         result['tempest_net_name'] = self._create_network(project_name)
 477
 478         LOGGER.debug("Creating two images for Tempest suite")
 479         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 480         result['image_id'] = self._create_image(image_name)
 481         image_name = getattr(
 482             config.CONF, 'openstack_image_name_alt') + self.guid
 483         result['image_id_alt'] = self._create_image(image_name)
 484
 485         LOGGER.info("Creating two flavors for Tempest suite")
 486         name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 487         result['flavor_id'] = self._create_flavor(name)
 488
 489         name = getattr(
 490             config.CONF, 'openstack_flavor_name_alt') + self.guid
 491         result['flavor_id_alt'] = self._create_flavor(name)
 492
 493         return result
 494
 495     def cleanup(self):
 496         """
 497         Cleanup all OpenStack objects. Should be called on completion.
 498         """
 499         for creator in reversed(self.creators):
 500             try:
 501                 creator.clean()
 502             except Exception as err:  # pylint: disable=broad-except
 503                 LOGGER.error('Unexpected error cleaning - %s', err)