functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack.create_flavor import OpenStackFlavor
  28 from snaps.openstack.tests import openstack_tests
  29 from snaps.openstack.utils import deploy_utils
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.opnfv_tests.openstack.snaps import snaps_utils
  34 from functest.opnfv_tests.openstack.tempest import conf_utils
  35 from functest.utils import config
  36 from functest.utils import env
  37
  38 LOGGER = logging.getLogger(__name__)
  39
  40
  41 class TempestCommon(testcase.TestCase):
  42     # pylint: disable=too-many-instance-attributes
  43     """TempestCommon testcases implementation class."""
  44
  45     TEMPEST_RESULTS_DIR = os.path.join(
  46         getattr(config.CONF, 'dir_results'), 'tempest')
  47
  48     def __init__(self, **kwargs):
  49         super(TempestCommon, self).__init__(**kwargs)
  50         self.resources = TempestResourcesManager(**kwargs)
  51         self.mode = ""
  52         self.option = []
  53         self.verifier_id = conf_utils.get_verifier_id()
  54         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  55             self.verifier_id)
  56         self.deployment_id = conf_utils.get_verifier_deployment_id()
  57         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  58             self.verifier_id, self.deployment_id)
  59         self.verification_id = None
  60         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  61         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  62         self.list = os.path.join(self.res_dir, 'test_list.txt')
  63         self.conf_file = None
  64
  65     @staticmethod
  66     def read_file(filename):
  67         """Read file and return content as a stripped list."""
  68         with open(filename) as src:
  69             return [line.strip() for line in src.readlines()]
  70
  71     @staticmethod
  72     def get_verifier_result(verif_id):
  73         """Retrieve verification results."""
  74         result = {
  75             'num_tests': 0,
  76             'num_success': 0,
  77             'num_failures': 0,
  78             'num_skipped': 0
  79         }
  80         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  81         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  82         proc = subprocess.Popen(cmd,
  83                                 stdout=subprocess.PIPE,
  84                                 stderr=subprocess.STDOUT)
  85         for line in proc.stdout:
  86             new_line = line.replace(' ', '').split('|')
  87             if 'Tests' in new_line:
  88                 break
  89             LOGGER.info(line)
  90             if 'Testscount' in new_line:
  91                 result['num_tests'] = int(new_line[2])
  92             elif 'Success' in new_line:
  93                 result['num_success'] = int(new_line[2])
  94             elif 'Skipped' in new_line:
  95                 result['num_skipped'] = int(new_line[2])
  96             elif 'Failures' in new_line:
  97                 result['num_failures'] = int(new_line[2])
  98         return result
  99
 100     def generate_test_list(self):
 101         """Generate test list based on the test mode."""
 102         LOGGER.debug("Generating test case list...")
 103         if self.mode == 'custom':
 104             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 105                 shutil.copyfile(
 106                     conf_utils.TEMPEST_CUSTOM, self.list)
 107             else:
 108                 raise Exception("Tempest test list file %s NOT found."
 109                                 % conf_utils.TEMPEST_CUSTOM)
 110         else:
 111             if self.mode == 'smoke':
 112                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 113             elif self.mode == 'full':
 114                 testr_mode = r"'^tempest\.'"
 115             else:
 116                 testr_mode = self.mode
 117             cmd = "(cd {0}; testr list-tests {1} >{2} 2>/dev/null)".format(
 118                 self.verifier_repo_dir, testr_mode, self.list)
 119             output = subprocess.check_output(cmd, shell=True)
 120             LOGGER.info("%s\n%s", cmd, output)
 121
 122     def apply_tempest_blacklist(self):
 123         """Exclude blacklisted test cases."""
 124         LOGGER.debug("Applying tempest blacklist...")
 125         if os.path.exists(self.raw_list):
 126             os.remove(self.raw_list)
 127         os.rename(self.list, self.raw_list)
 128         cases_file = self.read_file(self.raw_list)
 129         result_file = open(self.list, 'w')
 130         black_tests = []
 131         try:
 132             installer_type = env.get('INSTALLER_TYPE')
 133             deploy_scenario = env.get('DEPLOY_SCENARIO')
 134             if bool(installer_type) * bool(deploy_scenario):
 135                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 136                 # file
 137                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 138                 black_list_yaml = yaml.safe_load(black_list_file)
 139                 black_list_file.close()
 140                 for item in black_list_yaml:
 141                     scenarios = item['scenarios']
 142                     installers = item['installers']
 143                     if (deploy_scenario in scenarios and
 144                             installer_type in installers):
 145                         tests = item['tests']
 146                         for test in tests:
 147                             black_tests.append(test)
 148                         break
 149         except Exception:  # pylint: disable=broad-except
 150             black_tests = []
 151             LOGGER.debug("Tempest blacklist file does not exist.")
 152
 153         for cases_line in cases_file:
 154             for black_tests_line in black_tests:
 155                 if black_tests_line in cases_line:
 156                     break
 157             else:
 158                 result_file.write(str(cases_line) + '\n')
 159         result_file.close()
 160
 161     def run_verifier_tests(self):
 162         """Execute tempest test cases."""
 163         cmd = ["rally", "verify", "start", "--load-list",
 164                self.list]
 165         cmd.extend(self.option)
 166         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 167
 168         f_stdout = open(
 169             os.path.join(self.res_dir, "tempest.log"), 'w+')
 170         f_stderr = open(
 171             os.path.join(self.res_dir,
 172                          "tempest-error.log"), 'w+')
 173
 174         proc = subprocess.Popen(
 175             cmd,
 176             stdout=subprocess.PIPE,
 177             stderr=f_stderr,
 178             bufsize=1)
 179
 180         with proc.stdout:
 181             for line in iter(proc.stdout.readline, b''):
 182                 if re.search(r"\} tempest\.", line):
 183                     LOGGER.info(line.replace('\n', ''))
 184                 elif re.search('Starting verification', line):
 185                     LOGGER.info(line.replace('\n', ''))
 186                     first_pos = line.index("UUID=") + len("UUID=")
 187                     last_pos = line.index(") for deployment")
 188                     self.verification_id = line[first_pos:last_pos]
 189                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 190                 f_stdout.write(line)
 191         proc.wait()
 192
 193         f_stdout.close()
 194         f_stderr.close()
 195
 196         if self.verification_id is None:
 197             raise Exception('Verification UUID not found')
 198
 199     def parse_verifier_result(self):
 200         """Parse and save test results."""
 201         stat = self.get_verifier_result(self.verification_id)
 202         try:
 203             num_executed = stat['num_tests'] - stat['num_skipped']
 204             try:
 205                 self.result = 100 * stat['num_success'] / num_executed
 206             except ZeroDivisionError:
 207                 self.result = 0
 208                 if stat['num_tests'] > 0:
 209                     LOGGER.info("All tests have been skipped")
 210                 else:
 211                     LOGGER.error("No test has been executed")
 212                     return
 213
 214             with open(os.path.join(self.res_dir,
 215                                    "tempest.log"), 'r') as logfile:
 216                 output = logfile.read()
 217
 218             success_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 220                 success_testcases.append(match)
 221             failed_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail', output):
 223                 failed_testcases.append(match)
 224             skipped_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 226                 skipped_testcases.append(match)
 227
 228             self.details = {"tests_number": stat['num_tests'],
 229                             "success_number": stat['num_success'],
 230                             "skipped_number": stat['num_skipped'],
 231                             "failures_number": stat['num_failures'],
 232                             "success": success_testcases,
 233                             "skipped": skipped_testcases,
 234                             "failures": failed_testcases}
 235         except Exception:  # pylint: disable=broad-except
 236             self.result = 0
 237
 238         LOGGER.info("Tempest %s success_rate is %s%%",
 239                     self.case_name, self.result)
 240
 241     def generate_report(self):
 242         """Generate verification report."""
 243         html_file = os.path.join(self.res_dir,
 244                                  "tempest-report.html")
 245         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 246                self.verification_id, "--to", html_file]
 247         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 248                          stderr=subprocess.STDOUT)
 249
 250     def configure(self, **kwargs):  # pylint: disable=unused-argument
 251         """
 252         Create all openstack resources for tempest-based testcases and write
 253         tempest.conf.
 254         """
 255         if not os.path.exists(self.res_dir):
 256             os.makedirs(self.res_dir)
 257         resources = self.resources.create()
 258         compute_cnt = snaps_utils.get_active_compute_cnt(
 259             self.resources.os_creds)
 260         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 261         conf_utils.configure_tempest_update_params(
 262             self.conf_file, self.res_dir,
 263             network_name=resources.get("network_name"),
 264             image_id=resources.get("image_id"),
 265             flavor_id=resources.get("flavor_id"),
 266             compute_cnt=compute_cnt)
 267
 268     def run(self, **kwargs):
 269         self.start_time = time.time()
 270         try:
 271             self.configure()
 272             self.generate_test_list()
 273             self.apply_tempest_blacklist()
 274             self.run_verifier_tests()
 275             self.parse_verifier_result()
 276             self.generate_report()
 277             res = testcase.TestCase.EX_OK
 278         except Exception:  # pylint: disable=broad-except
 279             LOGGER.exception('Error with run')
 280             res = testcase.TestCase.EX_RUN_ERROR
 281         finally:
 282             self.resources.cleanup()
 283         self.stop_time = time.time()
 284         return res
 285
 286
 287 class TempestSmokeSerial(TempestCommon):
 288     """Tempest smoke serial testcase implementation."""
 289     def __init__(self, **kwargs):
 290         if "case_name" not in kwargs:
 291             kwargs["case_name"] = 'tempest_smoke_serial'
 292         TempestCommon.__init__(self, **kwargs)
 293         self.mode = "smoke"
 294         self.option = ["--concurrency", "1"]
 295
 296
 297 class TempestNeutronTrunk(TempestCommon):
 298     """Tempest neutron trunk testcase implementation."""
 299     def __init__(self, **kwargs):
 300         if "case_name" not in kwargs:
 301             kwargs["case_name"] = 'neutron_trunk'
 302         TempestCommon.__init__(self, **kwargs)
 303         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 304         self.res_dir = os.path.join(
 305             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 306         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 307         self.list = os.path.join(self.res_dir, 'test_list.txt')
 308
 309     def configure(self, **kwargs):
 310         super(TempestNeutronTrunk, self).configure(**kwargs)
 311         rconfig = conf_utils.ConfigParser.RawConfigParser()
 312         rconfig.read(self.conf_file)
 313         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 314         with open(self.conf_file, 'wb') as config_file:
 315             rconfig.write(config_file)
 316
 317
 318 class TempestSmokeParallel(TempestCommon):
 319     """Tempest smoke parallel testcase implementation."""
 320     def __init__(self, **kwargs):
 321         if "case_name" not in kwargs:
 322             kwargs["case_name"] = 'tempest_smoke_parallel'
 323         TempestCommon.__init__(self, **kwargs)
 324         self.mode = "smoke"
 325
 326
 327 class TempestFullParallel(TempestCommon):
 328     """Tempest full parallel testcase implementation."""
 329     def __init__(self, **kwargs):
 330         if "case_name" not in kwargs:
 331             kwargs["case_name"] = 'tempest_full_parallel'
 332         TempestCommon.__init__(self, **kwargs)
 333         self.mode = "full"
 334
 335
 336 class TempestCustom(TempestCommon):
 337     """Tempest custom testcase implementation."""
 338     def __init__(self, **kwargs):
 339         if "case_name" not in kwargs:
 340             kwargs["case_name"] = 'tempest_custom'
 341         TempestCommon.__init__(self, **kwargs)
 342         self.mode = "custom"
 343         self.option = ["--concurrency", "1"]
 344
 345
 346 class TempestDefcore(TempestCommon):
 347     """Tempest Defcore testcase implementation."""
 348     def __init__(self, **kwargs):
 349         if "case_name" not in kwargs:
 350             kwargs["case_name"] = 'tempest_defcore'
 351         TempestCommon.__init__(self, **kwargs)
 352         self.mode = "defcore"
 353         self.option = ["--concurrency", "1"]
 354
 355
 356 class TempestResourcesManager(object):
 357     """Tempest resource manager."""
 358     def __init__(self, **kwargs):
 359         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 360         self.guid = '-' + str(uuid.uuid4())
 361         self.creators = list()
 362         self.cirros_image_config = getattr(
 363             config.CONF, 'snaps_images_cirros', None)
 364
 365     def _create_project(self):
 366         """Create project for tests."""
 367         project_creator = deploy_utils.create_project(
 368             self.os_creds, ProjectConfig(
 369                 name=getattr(
 370                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 371                 description=getattr(
 372                     config.CONF, 'tempest_identity_tenant_description'),
 373                 domain=self.os_creds.project_domain_name))
 374         if project_creator is None or project_creator.get_project() is None:
 375             raise Exception("Failed to create tenant")
 376         self.creators.append(project_creator)
 377         return project_creator.get_project().id
 378
 379     def _create_user(self):
 380         """Create user for tests."""
 381         user_creator = deploy_utils.create_user(
 382             self.os_creds, UserConfig(
 383                 name=getattr(
 384                     config.CONF, 'tempest_identity_user_name') + self.guid,
 385                 password=getattr(
 386                     config.CONF, 'tempest_identity_user_password'),
 387                 project_name=getattr(
 388                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 389                 domain_name=self.os_creds.user_domain_name))
 390         if user_creator is None or user_creator.get_user() is None:
 391             raise Exception("Failed to create user")
 392         self.creators.append(user_creator)
 393         return user_creator.get_user().id
 394
 395     def _create_network(self, project_name):
 396         """Create network for tests."""
 397         tempest_network_type = None
 398         tempest_physical_network = None
 399         tempest_segmentation_id = None
 400
 401         tempest_network_type = getattr(
 402             config.CONF, 'tempest_network_type', None)
 403         tempest_physical_network = getattr(
 404             config.CONF, 'tempest_physical_network', None)
 405         tempest_segmentation_id = getattr(
 406             config.CONF, 'tempest_segmentation_id', None)
 407         tempest_net_name = getattr(
 408             config.CONF, 'tempest_private_net_name') + self.guid
 409
 410         network_creator = deploy_utils.create_network(
 411             self.os_creds, NetworkConfig(
 412                 name=tempest_net_name,
 413                 project_name=project_name,
 414                 network_type=tempest_network_type,
 415                 physical_network=tempest_physical_network,
 416                 segmentation_id=tempest_segmentation_id,
 417                 subnet_settings=[SubnetConfig(
 418                     name=getattr(
 419                         config.CONF,
 420                         'tempest_private_subnet_name') + self.guid,
 421                     project_name=project_name,
 422                     cidr=getattr(
 423                         config.CONF, 'tempest_private_subnet_cidr'),
 424                     dns_nameservers=[env.get('NAMESERVER')])]))
 425         if network_creator is None or network_creator.get_network() is None:
 426             raise Exception("Failed to create private network")
 427         self.creators.append(network_creator)
 428         return tempest_net_name
 429
 430     def _create_image(self, name):
 431         """Create image for tests"""
 432         os_image_settings = openstack_tests.cirros_image_settings(
 433             name, public=True,
 434             image_metadata=self.cirros_image_config)
 435         image_creator = deploy_utils.create_image(
 436             self.os_creds, os_image_settings)
 437         if image_creator is None:
 438             raise Exception('Failed to create image')
 439         self.creators.append(image_creator)
 440         return image_creator.get_image().id
 441
 442     def _create_flavor(self, name):
 443         """Create flavor for tests."""
 444         flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
 445         flavor_creator = OpenStackFlavor(
 446             self.os_creds, FlavorConfig(
 447                 name=name,
 448                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 449                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 450                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 451                 metadata=flavor_metadata))
 452         flavor = flavor_creator.create()
 453         if flavor is None:
 454             raise Exception('Failed to create flavor')
 455         self.creators.append(flavor_creator)
 456         return flavor.id
 457
 458     def create(self, create_project=False):
 459         """Create resources for Tempest test suite."""
 460         result = {
 461             'tempest_net_name': None,
 462             'image_id': None,
 463             'image_id_alt': None,
 464             'flavor_id': None,
 465             'flavor_id_alt': None
 466         }
 467         project_name = None
 468
 469         if create_project:
 470             LOGGER.debug("Creating project and user for Tempest suite")
 471             project_name = getattr(
 472                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 473             result['project_id'] = self._create_project()
 474             result['user_id'] = self._create_user()
 475             result['tenant_id'] = result['project_id']  # for compatibility
 476
 477         LOGGER.debug("Creating private network for Tempest suite")
 478         result['tempest_net_name'] = self._create_network(project_name)
 479
 480         LOGGER.debug("Creating two images for Tempest suite")
 481         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 482         result['image_id'] = self._create_image(image_name)
 483         image_name = getattr(
 484             config.CONF, 'openstack_image_name_alt') + self.guid
 485         result['image_id_alt'] = self._create_image(image_name)
 486
 487         LOGGER.info("Creating two flavors for Tempest suite")
 488         name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 489         result['flavor_id'] = self._create_flavor(name)
 490
 491         name = getattr(
 492             config.CONF, 'openstack_flavor_name_alt') + self.guid
 493         result['flavor_id_alt'] = self._create_flavor(name)
 494
 495         return result
 496
 497     def cleanup(self):
 498         """
 499         Cleanup all OpenStack objects. Should be called on completion.
 500         """
 501         for creator in reversed(self.creators):
 502             try:
 503                 creator.clean()
 504             except Exception as err:  # pylint: disable=broad-except
 505                 LOGGER.error('Unexpected error cleaning - %s', err)