functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack.create_flavor import OpenStackFlavor
  28 from snaps.openstack.tests import openstack_tests
  29 from snaps.openstack.utils import deploy_utils
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.opnfv_tests.openstack.snaps import snaps_utils
  34 from functest.opnfv_tests.openstack.tempest import conf_utils
  35 from functest.utils import config
  36 from functest.utils import env
  37 from functest.utils import functest_utils
  38
  39 LOGGER = logging.getLogger(__name__)
  40
  41
  42 class TempestCommon(testcase.TestCase):
  43     # pylint: disable=too-many-instance-attributes
  44     """TempestCommon testcases implementation class."""
  45
  46     TEMPEST_RESULTS_DIR = os.path.join(
  47         getattr(config.CONF, 'dir_results'), 'tempest')
  48
  49     def __init__(self, **kwargs):
  50         super(TempestCommon, self).__init__(**kwargs)
  51         self.resources = TempestResourcesManager(**kwargs)
  52         self.mode = ""
  53         self.option = []
  54         self.verifier_id = conf_utils.get_verifier_id()
  55         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  56             self.verifier_id)
  57         self.deployment_id = conf_utils.get_verifier_deployment_id()
  58         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  59             self.verifier_id, self.deployment_id)
  60         self.verification_id = None
  61         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  62         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  63         self.list = os.path.join(self.res_dir, 'test_list.txt')
  64         self.conf_file = None
  65
  66     @staticmethod
  67     def read_file(filename):
  68         """Read file and return content as a stripped list."""
  69         with open(filename) as src:
  70             return [line.strip() for line in src.readlines()]
  71
  72     @staticmethod
  73     def get_verifier_result(verif_id):
  74         """Retrieve verification results."""
  75         result = {
  76             'num_tests': 0,
  77             'num_success': 0,
  78             'num_failures': 0,
  79             'num_skipped': 0
  80         }
  81         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  82         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  83         proc = subprocess.Popen(cmd,
  84                                 stdout=subprocess.PIPE,
  85                                 stderr=subprocess.STDOUT)
  86         for line in proc.stdout:
  87             new_line = line.replace(' ', '').split('|')
  88             if 'Tests' in new_line:
  89                 break
  90             LOGGER.info(line)
  91             if 'Testscount' in new_line:
  92                 result['num_tests'] = int(new_line[2])
  93             elif 'Success' in new_line:
  94                 result['num_success'] = int(new_line[2])
  95             elif 'Skipped' in new_line:
  96                 result['num_skipped'] = int(new_line[2])
  97             elif 'Failures' in new_line:
  98                 result['num_failures'] = int(new_line[2])
  99         return result
 100
 101     def generate_test_list(self):
 102         """Generate test list based on the test mode."""
 103         LOGGER.debug("Generating test case list...")
 104         if self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, self.list)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 114             elif self.mode == 'full':
 115                 testr_mode = r"'^tempest\.'"
 116             else:
 117                 testr_mode = self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(self.verifier_repo_dir,
 121                                   testr_mode,
 122                                   self.list))
 123             functest_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search('Starting verification', line):
 188                     LOGGER.info(line.replace('\n', ''))
 189                     first_pos = line.index("UUID=") + len("UUID=")
 190                     last_pos = line.index(") for deployment")
 191                     self.verification_id = line[first_pos:last_pos]
 192                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 193                 f_stdout.write(line)
 194         proc.wait()
 195
 196         f_stdout.close()
 197         f_stderr.close()
 198
 199         if self.verification_id is None:
 200             raise Exception('Verification UUID not found')
 201
 202     def parse_verifier_result(self):
 203         """Parse and save test results."""
 204         stat = self.get_verifier_result(self.verification_id)
 205         try:
 206             num_executed = stat['num_tests'] - stat['num_skipped']
 207             try:
 208                 self.result = 100 * stat['num_success'] / num_executed
 209             except ZeroDivisionError:
 210                 self.result = 0
 211                 if stat['num_tests'] > 0:
 212                     LOGGER.info("All tests have been skipped")
 213                 else:
 214                     LOGGER.error("No test has been executed")
 215                     return
 216
 217             with open(os.path.join(self.res_dir,
 218                                    "tempest.log"), 'r') as logfile:
 219                 output = logfile.read()
 220
 221             success_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 223                 success_testcases.append(match)
 224             failed_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 226                 failed_testcases.append(match)
 227             skipped_testcases = []
 228             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests": stat['num_tests'],
 232                             "failures": stat['num_failures'],
 233                             "success": success_testcases,
 234                             "skipped": skipped_testcases,
 235                             "errors": failed_testcases}
 236         except Exception:  # pylint: disable=broad-except
 237             self.result = 0
 238
 239         LOGGER.info("Tempest %s success_rate is %s%%",
 240                     self.case_name, self.result)
 241
 242     def generate_report(self):
 243         """Generate verification report."""
 244         html_file = os.path.join(self.res_dir,
 245                                  "tempest-report.html")
 246         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 247                self.verification_id, "--to", html_file]
 248         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 249                          stderr=subprocess.STDOUT)
 250
 251     def configure(self, **kwargs):  # pylint: disable=unused-argument
 252         """
 253         Create all openstack resources for tempest-based testcases and write
 254         tempest.conf.
 255         """
 256         if not os.path.exists(self.res_dir):
 257             os.makedirs(self.res_dir)
 258         resources = self.resources.create()
 259         compute_cnt = snaps_utils.get_active_compute_cnt(
 260             self.resources.os_creds)
 261         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 262         conf_utils.configure_tempest_update_params(
 263             self.conf_file, self.res_dir,
 264             network_name=resources.get("network_name"),
 265             image_id=resources.get("image_id"),
 266             flavor_id=resources.get("flavor_id"),
 267             compute_cnt=compute_cnt)
 268
 269     def run(self, **kwargs):
 270         self.start_time = time.time()
 271         try:
 272             self.configure()
 273             self.generate_test_list()
 274             self.apply_tempest_blacklist()
 275             self.run_verifier_tests()
 276             self.parse_verifier_result()
 277             self.generate_report()
 278             res = testcase.TestCase.EX_OK
 279         except Exception:  # pylint: disable=broad-except
 280             LOGGER.exception('Error with run')
 281             res = testcase.TestCase.EX_RUN_ERROR
 282         finally:
 283             self.resources.cleanup()
 284         self.stop_time = time.time()
 285         return res
 286
 287
 288 class TempestSmokeSerial(TempestCommon):
 289     """Tempest smoke serial testcase implementation."""
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_smoke_serial'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.mode = "smoke"
 295         self.option = ["--concurrency", "1"]
 296
 297
 298 class TempestNeutronTrunk(TempestCommon):
 299     """Tempest neutron trunk testcase implementation."""
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'neutron_trunk'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 305         self.res_dir = os.path.join(
 306             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 307         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 308         self.list = os.path.join(self.res_dir, 'test_list.txt')
 309
 310     def configure(self, **kwargs):
 311         super(TempestNeutronTrunk, self).configure(**kwargs)
 312         rconfig = conf_utils.ConfigParser.RawConfigParser()
 313         rconfig.read(self.conf_file)
 314         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 315         with open(self.conf_file, 'wb') as config_file:
 316             rconfig.write(config_file)
 317
 318
 319 class TempestSmokeParallel(TempestCommon):
 320     """Tempest smoke parallel testcase implementation."""
 321     def __init__(self, **kwargs):
 322         if "case_name" not in kwargs:
 323             kwargs["case_name"] = 'tempest_smoke_parallel'
 324         TempestCommon.__init__(self, **kwargs)
 325         self.mode = "smoke"
 326
 327
 328 class TempestFullParallel(TempestCommon):
 329     """Tempest full parallel testcase implementation."""
 330     def __init__(self, **kwargs):
 331         if "case_name" not in kwargs:
 332             kwargs["case_name"] = 'tempest_full_parallel'
 333         TempestCommon.__init__(self, **kwargs)
 334         self.mode = "full"
 335
 336
 337 class TempestCustom(TempestCommon):
 338     """Tempest custom testcase implementation."""
 339     def __init__(self, **kwargs):
 340         if "case_name" not in kwargs:
 341             kwargs["case_name"] = 'tempest_custom'
 342         TempestCommon.__init__(self, **kwargs)
 343         self.mode = "custom"
 344         self.option = ["--concurrency", "1"]
 345
 346
 347 class TempestDefcore(TempestCommon):
 348     """Tempest Defcore testcase implementation."""
 349     def __init__(self, **kwargs):
 350         if "case_name" not in kwargs:
 351             kwargs["case_name"] = 'tempest_defcore'
 352         TempestCommon.__init__(self, **kwargs)
 353         self.mode = "defcore"
 354         self.option = ["--concurrency", "1"]
 355
 356
 357 class TempestResourcesManager(object):
 358     """Tempest resource manager."""
 359     def __init__(self, **kwargs):
 360         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 361         self.guid = '-' + str(uuid.uuid4())
 362         self.creators = list()
 363         self.cirros_image_config = getattr(
 364             config.CONF, 'snaps_images_cirros', None)
 365
 366     def _create_project(self):
 367         """Create project for tests."""
 368         project_creator = deploy_utils.create_project(
 369             self.os_creds, ProjectConfig(
 370                 name=getattr(
 371                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 372                 description=getattr(
 373                     config.CONF, 'tempest_identity_tenant_description'),
 374                 domain=self.os_creds.project_domain_name))
 375         if project_creator is None or project_creator.get_project() is None:
 376             raise Exception("Failed to create tenant")
 377         self.creators.append(project_creator)
 378         return project_creator.get_project().id
 379
 380     def _create_user(self):
 381         """Create user for tests."""
 382         user_creator = deploy_utils.create_user(
 383             self.os_creds, UserConfig(
 384                 name=getattr(
 385                     config.CONF, 'tempest_identity_user_name') + self.guid,
 386                 password=getattr(
 387                     config.CONF, 'tempest_identity_user_password'),
 388                 project_name=getattr(
 389                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 390                 domain_name=self.os_creds.user_domain_name))
 391         if user_creator is None or user_creator.get_user() is None:
 392             raise Exception("Failed to create user")
 393         self.creators.append(user_creator)
 394         return user_creator.get_user().id
 395
 396     def _create_network(self, project_name):
 397         """Create network for tests."""
 398         tempest_network_type = None
 399         tempest_physical_network = None
 400         tempest_segmentation_id = None
 401
 402         tempest_network_type = getattr(
 403             config.CONF, 'tempest_network_type', None)
 404         tempest_physical_network = getattr(
 405             config.CONF, 'tempest_physical_network', None)
 406         tempest_segmentation_id = getattr(
 407             config.CONF, 'tempest_segmentation_id', None)
 408         tempest_net_name = getattr(
 409             config.CONF, 'tempest_private_net_name') + self.guid
 410
 411         network_creator = deploy_utils.create_network(
 412             self.os_creds, NetworkConfig(
 413                 name=tempest_net_name,
 414                 project_name=project_name,
 415                 network_type=tempest_network_type,
 416                 physical_network=tempest_physical_network,
 417                 segmentation_id=tempest_segmentation_id,
 418                 subnet_settings=[SubnetConfig(
 419                     name=getattr(
 420                         config.CONF,
 421                         'tempest_private_subnet_name') + self.guid,
 422                     project_name=project_name,
 423                     cidr=getattr(
 424                         config.CONF, 'tempest_private_subnet_cidr'),
 425                     dns_nameservers=[env.get('NAMESERVER')])]))
 426         if network_creator is None or network_creator.get_network() is None:
 427             raise Exception("Failed to create private network")
 428         self.creators.append(network_creator)
 429         return tempest_net_name
 430
 431     def _create_image(self, name):
 432         """Create image for tests"""
 433         os_image_settings = openstack_tests.cirros_image_settings(
 434             name, public=True,
 435             image_metadata=self.cirros_image_config)
 436         image_creator = deploy_utils.create_image(
 437             self.os_creds, os_image_settings)
 438         if image_creator is None:
 439             raise Exception('Failed to create image')
 440         self.creators.append(image_creator)
 441         return image_creator.get_image().id
 442
 443     def _create_flavor(self, name):
 444         """Create flavor for tests."""
 445         flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
 446         flavor_creator = OpenStackFlavor(
 447             self.os_creds, FlavorConfig(
 448                 name=name,
 449                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 450                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 451                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 452                 metadata=flavor_metadata))
 453         flavor = flavor_creator.create()
 454         if flavor is None:
 455             raise Exception('Failed to create flavor')
 456         self.creators.append(flavor_creator)
 457         return flavor.id
 458
 459     def create(self, create_project=False):
 460         """Create resources for Tempest test suite."""
 461         result = {
 462             'tempest_net_name': None,
 463             'image_id': None,
 464             'image_id_alt': None,
 465             'flavor_id': None,
 466             'flavor_id_alt': None
 467         }
 468         project_name = None
 469
 470         if create_project:
 471             LOGGER.debug("Creating project and user for Tempest suite")
 472             project_name = getattr(
 473                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 474             result['project_id'] = self._create_project()
 475             result['user_id'] = self._create_user()
 476             result['tenant_id'] = result['project_id']  # for compatibility
 477
 478         LOGGER.debug("Creating private network for Tempest suite")
 479         result['tempest_net_name'] = self._create_network(project_name)
 480
 481         LOGGER.debug("Creating two images for Tempest suite")
 482         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 483         result['image_id'] = self._create_image(image_name)
 484         image_name = getattr(
 485             config.CONF, 'openstack_image_name_alt') + self.guid
 486         result['image_id_alt'] = self._create_image(image_name)
 487
 488         LOGGER.info("Creating two flavors for Tempest suite")
 489         name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 490         result['flavor_id'] = self._create_flavor(name)
 491
 492         name = getattr(
 493             config.CONF, 'openstack_flavor_name_alt') + self.guid
 494         result['flavor_id_alt'] = self._create_flavor(name)
 495
 496         return result
 497
 498     def cleanup(self):
 499         """
 500         Cleanup all OpenStack objects. Should be called on completion.
 501         """
 502         for creator in reversed(self.creators):
 503             try:
 504                 creator.clean()
 505             except Exception as err:  # pylint: disable=broad-except
 506                 LOGGER.error('Unexpected error cleaning - %s', err)