functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack.create_flavor import OpenStackFlavor
  28 from snaps.openstack.tests import openstack_tests
  29 from snaps.openstack.utils import deploy_utils
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.opnfv_tests.openstack.snaps import snaps_utils
  34 from functest.opnfv_tests.openstack.tempest import conf_utils
  35 from functest.utils import config
  36 from functest.utils import env
  37
  38 LOGGER = logging.getLogger(__name__)
  39
  40
  41 class TempestCommon(testcase.TestCase):
  42     # pylint: disable=too-many-instance-attributes
  43     """TempestCommon testcases implementation class."""
  44
  45     TEMPEST_RESULTS_DIR = os.path.join(
  46         getattr(config.CONF, 'dir_results'), 'tempest')
  47
  48     def __init__(self, **kwargs):
  49         super(TempestCommon, self).__init__(**kwargs)
  50         self.resources = TempestResourcesManager(**kwargs)
  51         self.mode = ""
  52         self.option = []
  53         self.verifier_id = conf_utils.get_verifier_id()
  54         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  55             self.verifier_id)
  56         self.deployment_id = conf_utils.get_verifier_deployment_id()
  57         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  58             self.verifier_id, self.deployment_id)
  59         self.verification_id = None
  60         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  61         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  62         self.list = os.path.join(self.res_dir, 'test_list.txt')
  63         self.conf_file = None
  64
  65     @staticmethod
  66     def read_file(filename):
  67         """Read file and return content as a stripped list."""
  68         with open(filename) as src:
  69             return [line.strip() for line in src.readlines()]
  70
  71     @staticmethod
  72     def get_verifier_result(verif_id):
  73         """Retrieve verification results."""
  74         result = {
  75             'num_tests': 0,
  76             'num_success': 0,
  77             'num_failures': 0,
  78             'num_skipped': 0
  79         }
  80         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  81         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  82         proc = subprocess.Popen(cmd,
  83                                 stdout=subprocess.PIPE,
  84                                 stderr=subprocess.STDOUT)
  85         for line in proc.stdout:
  86             new_line = line.replace(' ', '').split('|')
  87             if 'Tests' in new_line:
  88                 break
  89             LOGGER.info(line)
  90             if 'Testscount' in new_line:
  91                 result['num_tests'] = int(new_line[2])
  92             elif 'Success' in new_line:
  93                 result['num_success'] = int(new_line[2])
  94             elif 'Skipped' in new_line:
  95                 result['num_skipped'] = int(new_line[2])
  96             elif 'Failures' in new_line:
  97                 result['num_failures'] = int(new_line[2])
  98         return result
  99
 100     @staticmethod
 101     def backup_tempest_config(conf_file, res_dir):
 102         """
 103         Copy config file to tempest results directory
 104         """
 105         if not os.path.exists(res_dir):
 106             os.makedirs(res_dir)
 107         shutil.copyfile(conf_file,
 108                         os.path.join(res_dir, 'tempest.conf'))
 109
 110     def generate_test_list(self):
 111         """Generate test list based on the test mode."""
 112         LOGGER.debug("Generating test case list...")
 113         if self.mode == 'custom':
 114             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 115                 shutil.copyfile(
 116                     conf_utils.TEMPEST_CUSTOM, self.list)
 117             else:
 118                 raise Exception("Tempest test list file %s NOT found."
 119                                 % conf_utils.TEMPEST_CUSTOM)
 120         else:
 121             if self.mode == 'smoke':
 122                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 123             elif self.mode == 'full':
 124                 testr_mode = r"'^tempest\.'"
 125             else:
 126                 testr_mode = self.mode
 127             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 128                 self.verifier_repo_dir, testr_mode, self.list)
 129             output = subprocess.check_output(cmd, shell=True)
 130             LOGGER.info("%s\n%s", cmd, output)
 131
 132     def apply_tempest_blacklist(self):
 133         """Exclude blacklisted test cases."""
 134         LOGGER.debug("Applying tempest blacklist...")
 135         if os.path.exists(self.raw_list):
 136             os.remove(self.raw_list)
 137         os.rename(self.list, self.raw_list)
 138         cases_file = self.read_file(self.raw_list)
 139         result_file = open(self.list, 'w')
 140         black_tests = []
 141         try:
 142             installer_type = env.get('INSTALLER_TYPE')
 143             deploy_scenario = env.get('DEPLOY_SCENARIO')
 144             if bool(installer_type) * bool(deploy_scenario):
 145                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 146                 # file
 147                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 148                 black_list_yaml = yaml.safe_load(black_list_file)
 149                 black_list_file.close()
 150                 for item in black_list_yaml:
 151                     scenarios = item['scenarios']
 152                     installers = item['installers']
 153                     if (deploy_scenario in scenarios and
 154                             installer_type in installers):
 155                         tests = item['tests']
 156                         for test in tests:
 157                             black_tests.append(test)
 158                         break
 159         except Exception:  # pylint: disable=broad-except
 160             black_tests = []
 161             LOGGER.debug("Tempest blacklist file does not exist.")
 162
 163         for cases_line in cases_file:
 164             for black_tests_line in black_tests:
 165                 if black_tests_line in cases_line:
 166                     break
 167             else:
 168                 result_file.write(str(cases_line) + '\n')
 169         result_file.close()
 170
 171     def run_verifier_tests(self):
 172         """Execute tempest test cases."""
 173         cmd = ["rally", "verify", "start", "--load-list",
 174                self.list]
 175         cmd.extend(self.option)
 176         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 177
 178         f_stdout = open(
 179             os.path.join(self.res_dir, "tempest.log"), 'w+')
 180         f_stderr = open(
 181             os.path.join(self.res_dir,
 182                          "tempest-error.log"), 'w+')
 183
 184         proc = subprocess.Popen(
 185             cmd,
 186             stdout=subprocess.PIPE,
 187             stderr=f_stderr,
 188             bufsize=1)
 189
 190         with proc.stdout:
 191             for line in iter(proc.stdout.readline, b''):
 192                 if re.search(r"\} tempest\.", line):
 193                     LOGGER.info(line.replace('\n', ''))
 194                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 195                     self.verification_id = re.search(
 196                         r'(?=\(UUID=(.*)\))', line).group(1)
 197                     LOGGER.info('Verification UUID: %s', self.verification_id)
 198                 f_stdout.write(line)
 199         proc.wait()
 200
 201         f_stdout.close()
 202         f_stderr.close()
 203
 204         if self.verification_id is None:
 205             raise Exception('Verification UUID not found')
 206
 207     def parse_verifier_result(self):
 208         """Parse and save test results."""
 209         stat = self.get_verifier_result(self.verification_id)
 210         try:
 211             num_executed = stat['num_tests'] - stat['num_skipped']
 212             try:
 213                 self.result = 100 * stat['num_success'] / num_executed
 214             except ZeroDivisionError:
 215                 self.result = 0
 216                 if stat['num_tests'] > 0:
 217                     LOGGER.info("All tests have been skipped")
 218                 else:
 219                     LOGGER.error("No test has been executed")
 220                     return
 221
 222             with open(os.path.join(self.res_dir,
 223                                    "tempest.log"), 'r') as logfile:
 224                 output = logfile.read()
 225
 226             success_testcases = []
 227             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 228                 success_testcases.append(match)
 229             failed_testcases = []
 230             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail', output):
 231                 failed_testcases.append(match)
 232             skipped_testcases = []
 233             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 234                 skipped_testcases.append(match)
 235
 236             self.details = {"tests_number": stat['num_tests'],
 237                             "success_number": stat['num_success'],
 238                             "skipped_number": stat['num_skipped'],
 239                             "failures_number": stat['num_failures'],
 240                             "success": success_testcases,
 241                             "skipped": skipped_testcases,
 242                             "failures": failed_testcases}
 243         except Exception:  # pylint: disable=broad-except
 244             self.result = 0
 245
 246         LOGGER.info("Tempest %s success_rate is %s%%",
 247                     self.case_name, self.result)
 248
 249     def generate_report(self):
 250         """Generate verification report."""
 251         html_file = os.path.join(self.res_dir,
 252                                  "tempest-report.html")
 253         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 254                self.verification_id, "--to", html_file]
 255         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 256                          stderr=subprocess.STDOUT)
 257
 258     def configure(self, **kwargs):  # pylint: disable=unused-argument
 259         """
 260         Create all openstack resources for tempest-based testcases and write
 261         tempest.conf.
 262         """
 263         if not os.path.exists(self.res_dir):
 264             os.makedirs(self.res_dir)
 265         resources = self.resources.create()
 266         compute_cnt = snaps_utils.get_active_compute_cnt(
 267             self.resources.os_creds)
 268         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 269         conf_utils.configure_tempest_update_params(
 270             self.conf_file, network_name=resources.get("network_name"),
 271             image_id=resources.get("image_id"),
 272             flavor_id=resources.get("flavor_id"),
 273             compute_cnt=compute_cnt)
 274         self.backup_tempest_config(self.conf_file, self.res_dir)
 275
 276     def run(self, **kwargs):
 277         self.start_time = time.time()
 278         try:
 279             self.configure(**kwargs)
 280             self.generate_test_list()
 281             self.apply_tempest_blacklist()
 282             self.run_verifier_tests()
 283             self.parse_verifier_result()
 284             self.generate_report()
 285             res = testcase.TestCase.EX_OK
 286         except Exception:  # pylint: disable=broad-except
 287             LOGGER.exception('Error with run')
 288             res = testcase.TestCase.EX_RUN_ERROR
 289         finally:
 290             self.resources.cleanup()
 291         self.stop_time = time.time()
 292         return res
 293
 294
 295 class TempestSmokeSerial(TempestCommon):
 296     """Tempest smoke serial testcase implementation."""
 297     def __init__(self, **kwargs):
 298         if "case_name" not in kwargs:
 299             kwargs["case_name"] = 'tempest_smoke_serial'
 300         TempestCommon.__init__(self, **kwargs)
 301         self.mode = "smoke"
 302         self.option = ["--concurrency", "1"]
 303
 304
 305 class TempestNeutronTrunk(TempestCommon):
 306     """Tempest neutron trunk testcase implementation."""
 307     def __init__(self, **kwargs):
 308         if "case_name" not in kwargs:
 309             kwargs["case_name"] = 'neutron_trunk'
 310         TempestCommon.__init__(self, **kwargs)
 311         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 312         self.res_dir = os.path.join(
 313             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 314         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 315         self.list = os.path.join(self.res_dir, 'test_list.txt')
 316
 317     def configure(self, **kwargs):
 318         super(TempestNeutronTrunk, self).configure(**kwargs)
 319         rconfig = conf_utils.ConfigParser.RawConfigParser()
 320         rconfig.read(self.conf_file)
 321         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 322         with open(self.conf_file, 'wb') as config_file:
 323             rconfig.write(config_file)
 324
 325
 326 class TempestSmokeParallel(TempestCommon):
 327     """Tempest smoke parallel testcase implementation."""
 328     def __init__(self, **kwargs):
 329         if "case_name" not in kwargs:
 330             kwargs["case_name"] = 'tempest_smoke_parallel'
 331         TempestCommon.__init__(self, **kwargs)
 332         self.mode = "smoke"
 333
 334
 335 class TempestFullParallel(TempestCommon):
 336     """Tempest full parallel testcase implementation."""
 337     def __init__(self, **kwargs):
 338         if "case_name" not in kwargs:
 339             kwargs["case_name"] = 'tempest_full_parallel'
 340         TempestCommon.__init__(self, **kwargs)
 341         self.mode = "full"
 342
 343
 344 class TempestCustom(TempestCommon):
 345     """Tempest custom testcase implementation."""
 346     def __init__(self, **kwargs):
 347         if "case_name" not in kwargs:
 348             kwargs["case_name"] = 'tempest_custom'
 349         TempestCommon.__init__(self, **kwargs)
 350         self.mode = "custom"
 351         self.option = ["--concurrency", "1"]
 352
 353
 354 class TempestDefcore(TempestCommon):
 355     """Tempest Defcore testcase implementation."""
 356     def __init__(self, **kwargs):
 357         if "case_name" not in kwargs:
 358             kwargs["case_name"] = 'tempest_defcore'
 359         TempestCommon.__init__(self, **kwargs)
 360         self.mode = "defcore"
 361         self.option = ["--concurrency", "1"]
 362
 363
 364 class TempestResourcesManager(object):
 365     """Tempest resource manager."""
 366     def __init__(self, **kwargs):
 367         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 368         self.guid = '-' + str(uuid.uuid4())
 369         self.creators = list()
 370         self.cirros_image_config = getattr(
 371             config.CONF, 'snaps_images_cirros', None)
 372
 373     def _create_project(self):
 374         """Create project for tests."""
 375         project_creator = deploy_utils.create_project(
 376             self.os_creds, ProjectConfig(
 377                 name=getattr(
 378                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 379                 description=getattr(
 380                     config.CONF, 'tempest_identity_tenant_description'),
 381                 domain=self.os_creds.project_domain_name))
 382         if project_creator is None or project_creator.get_project() is None:
 383             raise Exception("Failed to create tenant")
 384         self.creators.append(project_creator)
 385         return project_creator.get_project().id
 386
 387     def _create_user(self):
 388         """Create user for tests."""
 389         user_creator = deploy_utils.create_user(
 390             self.os_creds, UserConfig(
 391                 name=getattr(
 392                     config.CONF, 'tempest_identity_user_name') + self.guid,
 393                 password=getattr(
 394                     config.CONF, 'tempest_identity_user_password'),
 395                 project_name=getattr(
 396                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 397                 domain_name=self.os_creds.user_domain_name))
 398         if user_creator is None or user_creator.get_user() is None:
 399             raise Exception("Failed to create user")
 400         self.creators.append(user_creator)
 401         return user_creator.get_user().id
 402
 403     def _create_network(self, project_name):
 404         """Create network for tests."""
 405         tempest_network_type = None
 406         tempest_physical_network = None
 407         tempest_segmentation_id = None
 408
 409         tempest_network_type = getattr(
 410             config.CONF, 'tempest_network_type', None)
 411         tempest_physical_network = getattr(
 412             config.CONF, 'tempest_physical_network', None)
 413         tempest_segmentation_id = getattr(
 414             config.CONF, 'tempest_segmentation_id', None)
 415         tempest_net_name = getattr(
 416             config.CONF, 'tempest_private_net_name') + self.guid
 417
 418         network_creator = deploy_utils.create_network(
 419             self.os_creds, NetworkConfig(
 420                 name=tempest_net_name,
 421                 project_name=project_name,
 422                 network_type=tempest_network_type,
 423                 physical_network=tempest_physical_network,
 424                 segmentation_id=tempest_segmentation_id,
 425                 subnet_settings=[SubnetConfig(
 426                     name=getattr(
 427                         config.CONF,
 428                         'tempest_private_subnet_name') + self.guid,
 429                     project_name=project_name,
 430                     cidr=getattr(
 431                         config.CONF, 'tempest_private_subnet_cidr'),
 432                     dns_nameservers=[env.get('NAMESERVER')])]))
 433         if network_creator is None or network_creator.get_network() is None:
 434             raise Exception("Failed to create private network")
 435         self.creators.append(network_creator)
 436         return tempest_net_name
 437
 438     def _create_image(self, name):
 439         """Create image for tests"""
 440         os_image_settings = openstack_tests.cirros_image_settings(
 441             name, public=True,
 442             image_metadata=self.cirros_image_config)
 443         image_creator = deploy_utils.create_image(
 444             self.os_creds, os_image_settings)
 445         if image_creator is None:
 446             raise Exception('Failed to create image')
 447         self.creators.append(image_creator)
 448         return image_creator.get_image().id
 449
 450     def _create_flavor(self, name):
 451         """Create flavor for tests."""
 452         flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
 453         flavor_creator = OpenStackFlavor(
 454             self.os_creds, FlavorConfig(
 455                 name=name,
 456                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 457                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 458                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 459                 metadata=flavor_metadata))
 460         flavor = flavor_creator.create()
 461         if flavor is None:
 462             raise Exception('Failed to create flavor')
 463         self.creators.append(flavor_creator)
 464         return flavor.id
 465
 466     def create(self, create_project=False):
 467         """Create resources for Tempest test suite."""
 468         result = {
 469             'tempest_net_name': None,
 470             'image_id': None,
 471             'image_id_alt': None,
 472             'flavor_id': None,
 473             'flavor_id_alt': None
 474         }
 475         project_name = None
 476
 477         if create_project:
 478             LOGGER.debug("Creating project and user for Tempest suite")
 479             project_name = getattr(
 480                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 481             result['project_id'] = self._create_project()
 482             result['user_id'] = self._create_user()
 483             result['tenant_id'] = result['project_id']  # for compatibility
 484
 485         LOGGER.debug("Creating private network for Tempest suite")
 486         result['tempest_net_name'] = self._create_network(project_name)
 487
 488         LOGGER.debug("Creating two images for Tempest suite")
 489         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 490         result['image_id'] = self._create_image(image_name)
 491         image_name = getattr(
 492             config.CONF, 'openstack_image_name_alt') + self.guid
 493         result['image_id_alt'] = self._create_image(image_name)
 494
 495         LOGGER.info("Creating two flavors for Tempest suite")
 496         name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 497         result['flavor_id'] = self._create_flavor(name)
 498
 499         name = getattr(
 500             config.CONF, 'openstack_flavor_name_alt') + self.guid
 501         result['flavor_id_alt'] = self._create_flavor(name)
 502
 503         return result
 504
 505     def cleanup(self):
 506         """
 507         Cleanup all OpenStack objects. Should be called on completion.
 508         """
 509         for creator in reversed(self.creators):
 510             try:
 511                 creator.clean()
 512             except Exception as err:  # pylint: disable=broad-except
 513                 LOGGER.error('Unexpected error cleaning - %s', err)