functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import os_client_config
  24 from six.moves import configparser
  25 from xtesting.core import testcase
  26 import yaml
  27
  28 from functest.opnfv_tests.openstack.tempest import conf_utils
  29 from functest.utils import config
  30 from functest.utils import env
  31
  32 LOGGER = logging.getLogger(__name__)
  33
  34
  35 class TempestCommon(testcase.TestCase):
  36     # pylint: disable=too-many-instance-attributes
  37     """TempestCommon testcases implementation class."""
  38
  39     TEMPEST_RESULTS_DIR = os.path.join(
  40         getattr(config.CONF, 'dir_results'), 'tempest')
  41
  42     def __init__(self, **kwargs):
  43         super(TempestCommon, self).__init__(**kwargs)
  44         self.resources = TempestResourcesManager()
  45         self.mode = ""
  46         self.option = []
  47         self.verifier_id = conf_utils.get_verifier_id()
  48         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  49             self.verifier_id)
  50         self.deployment_id = conf_utils.get_verifier_deployment_id()
  51         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  52             self.verifier_id, self.deployment_id)
  53         self.verification_id = None
  54         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  55         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  56         self.list = os.path.join(self.res_dir, 'test_list.txt')
  57         self.conf_file = None
  58
  59     @staticmethod
  60     def read_file(filename):
  61         """Read file and return content as a stripped list."""
  62         with open(filename) as src:
  63             return [line.strip() for line in src.readlines()]
  64
  65     @staticmethod
  66     def get_verifier_result(verif_id):
  67         """Retrieve verification results."""
  68         result = {
  69             'num_tests': 0,
  70             'num_success': 0,
  71             'num_failures': 0,
  72             'num_skipped': 0
  73         }
  74         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  75         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  76         proc = subprocess.Popen(cmd,
  77                                 stdout=subprocess.PIPE,
  78                                 stderr=subprocess.STDOUT)
  79         for line in proc.stdout:
  80             new_line = line.replace(' ', '').split('|')
  81             if 'Tests' in new_line:
  82                 break
  83             LOGGER.info(line)
  84             if 'Testscount' in new_line:
  85                 result['num_tests'] = int(new_line[2])
  86             elif 'Success' in new_line:
  87                 result['num_success'] = int(new_line[2])
  88             elif 'Skipped' in new_line:
  89                 result['num_skipped'] = int(new_line[2])
  90             elif 'Failures' in new_line:
  91                 result['num_failures'] = int(new_line[2])
  92         return result
  93
  94     @staticmethod
  95     def backup_tempest_config(conf_file, res_dir):
  96         """
  97         Copy config file to tempest results directory
  98         """
  99         if not os.path.exists(res_dir):
 100             os.makedirs(res_dir)
 101         shutil.copyfile(conf_file,
 102                         os.path.join(res_dir, 'tempest.conf'))
 103
 104     def generate_test_list(self):
 105         """Generate test list based on the test mode."""
 106         LOGGER.debug("Generating test case list...")
 107         if self.mode == 'custom':
 108             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 109                 shutil.copyfile(
 110                     conf_utils.TEMPEST_CUSTOM, self.list)
 111             else:
 112                 raise Exception("Tempest test list file %s NOT found."
 113                                 % conf_utils.TEMPEST_CUSTOM)
 114         else:
 115             if self.mode == 'smoke':
 116                 testr_mode = r"'^tempest\.(api|scenario).*\[.*\bsmoke\b.*\]$'"
 117             elif self.mode == 'full':
 118                 testr_mode = r"'^tempest\.'"
 119             else:
 120                 testr_mode = self.mode
 121             cmd = "(cd {0}; stestr list {1} >{2} 2>/dev/null)".format(
 122                 self.verifier_repo_dir, testr_mode, self.list)
 123             output = subprocess.check_output(cmd, shell=True)
 124             LOGGER.info("%s\n%s", cmd, output)
 125
 126     def apply_tempest_blacklist(self):
 127         """Exclude blacklisted test cases."""
 128         LOGGER.debug("Applying tempest blacklist...")
 129         if os.path.exists(self.raw_list):
 130             os.remove(self.raw_list)
 131         os.rename(self.list, self.raw_list)
 132         cases_file = self.read_file(self.raw_list)
 133         result_file = open(self.list, 'w')
 134         black_tests = []
 135         try:
 136             installer_type = env.get('INSTALLER_TYPE')
 137             deploy_scenario = env.get('DEPLOY_SCENARIO')
 138             if bool(installer_type) * bool(deploy_scenario):
 139                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 140                 # file
 141                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 142                 black_list_yaml = yaml.safe_load(black_list_file)
 143                 black_list_file.close()
 144                 for item in black_list_yaml:
 145                     scenarios = item['scenarios']
 146                     installers = item['installers']
 147                     if (deploy_scenario in scenarios and
 148                             installer_type in installers):
 149                         tests = item['tests']
 150                         for test in tests:
 151                             black_tests.append(test)
 152                         break
 153         except Exception:  # pylint: disable=broad-except
 154             black_tests = []
 155             LOGGER.debug("Tempest blacklist file does not exist.")
 156
 157         for cases_line in cases_file:
 158             for black_tests_line in black_tests:
 159                 if black_tests_line in cases_line:
 160                     break
 161             else:
 162                 result_file.write(str(cases_line) + '\n')
 163         result_file.close()
 164
 165     def run_verifier_tests(self):
 166         """Execute tempest test cases."""
 167         cmd = ["rally", "verify", "start", "--load-list",
 168                self.list]
 169         cmd.extend(self.option)
 170         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 171
 172         f_stdout = open(
 173             os.path.join(self.res_dir, "tempest.log"), 'w+')
 174         f_stderr = open(
 175             os.path.join(self.res_dir,
 176                          "tempest-error.log"), 'w+')
 177
 178         proc = subprocess.Popen(
 179             cmd,
 180             stdout=subprocess.PIPE,
 181             stderr=f_stderr,
 182             bufsize=1)
 183
 184         with proc.stdout:
 185             for line in iter(proc.stdout.readline, b''):
 186                 if re.search(r"\} tempest\.", line):
 187                     LOGGER.info(line.replace('\n', ''))
 188                 elif re.search(r'(?=\(UUID=(.*)\))', line):
 189                     self.verification_id = re.search(
 190                         r'(?=\(UUID=(.*)\))', line).group(1)
 191                     LOGGER.info('Verification UUID: %s', self.verification_id)
 192                 f_stdout.write(line)
 193         proc.wait()
 194
 195         f_stdout.close()
 196         f_stderr.close()
 197
 198         if self.verification_id is None:
 199             raise Exception('Verification UUID not found')
 200
 201     def parse_verifier_result(self):
 202         """Parse and save test results."""
 203         stat = self.get_verifier_result(self.verification_id)
 204         try:
 205             num_executed = stat['num_tests'] - stat['num_skipped']
 206             try:
 207                 self.result = 100 * stat['num_success'] / num_executed
 208             except ZeroDivisionError:
 209                 self.result = 0
 210                 if stat['num_tests'] > 0:
 211                     LOGGER.info("All tests have been skipped")
 212                 else:
 213                     LOGGER.error("No test has been executed")
 214                     return
 215
 216             with open(os.path.join(self.res_dir,
 217                                    "tempest-error.log"), 'r') as logfile:
 218                 output = logfile.read()
 219
 220             success_testcases = []
 221             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} success ',
 222                                     output):
 223                 success_testcases.append(match)
 224             failed_testcases = []
 225             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} fail',
 226                                     output):
 227                 failed_testcases.append(match)
 228             skipped_testcases = []
 229             for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:',
 230                                     output):
 231                 skipped_testcases.append(match)
 232
 233             self.details = {"tests_number": stat['num_tests'],
 234                             "success_number": stat['num_success'],
 235                             "skipped_number": stat['num_skipped'],
 236                             "failures_number": stat['num_failures'],
 237                             "success": success_testcases,
 238                             "skipped": skipped_testcases,
 239                             "failures": failed_testcases}
 240         except Exception:  # pylint: disable=broad-except
 241             self.result = 0
 242
 243         LOGGER.info("Tempest %s success_rate is %s%%",
 244                     self.case_name, self.result)
 245
 246     def generate_report(self):
 247         """Generate verification report."""
 248         html_file = os.path.join(self.res_dir,
 249                                  "tempest-report.html")
 250         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 251                self.verification_id, "--to", html_file]
 252         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 253                          stderr=subprocess.STDOUT)
 254
 255     def configure(self, **kwargs):  # pylint: disable=unused-argument
 256         """
 257         Create all openstack resources for tempest-based testcases and write
 258         tempest.conf.
 259         """
 260         if not os.path.exists(self.res_dir):
 261             os.makedirs(self.res_dir)
 262         self.resources.create()
 263         compute_cnt = len(self.resources.cloud.list_hypervisors())
 264         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 265         conf_utils.configure_tempest_update_params(
 266             self.conf_file, network_name=self.resources.network.id,
 267             image_id=self.resources.image.id,
 268             flavor_id=self.resources.flavor.id,
 269             compute_cnt=compute_cnt,
 270             image_alt_id=self.resources.image_alt.id,
 271             flavor_alt_id=self.resources.flavor_alt.id)
 272         self.backup_tempest_config(self.conf_file, self.res_dir)
 273
 274     def run(self, **kwargs):
 275         self.start_time = time.time()
 276         try:
 277             self.configure(**kwargs)
 278             self.generate_test_list()
 279             self.apply_tempest_blacklist()
 280             self.run_verifier_tests()
 281             self.parse_verifier_result()
 282             self.generate_report()
 283             res = testcase.TestCase.EX_OK
 284         except Exception:  # pylint: disable=broad-except
 285             LOGGER.exception('Error with run')
 286             res = testcase.TestCase.EX_RUN_ERROR
 287         finally:
 288             self.resources.cleanup()
 289         self.stop_time = time.time()
 290         return res
 291
 292
 293 class TempestSmokeSerial(TempestCommon):
 294     """Tempest smoke serial testcase implementation."""
 295     def __init__(self, **kwargs):
 296         if "case_name" not in kwargs:
 297             kwargs["case_name"] = 'tempest_smoke_serial'
 298         TempestCommon.__init__(self, **kwargs)
 299         self.mode = "smoke"
 300         self.option = ["--concurrency", "1"]
 301
 302
 303 class TempestNeutronTrunk(TempestCommon):
 304     """Tempest neutron trunk testcase implementation."""
 305     def __init__(self, **kwargs):
 306         if "case_name" not in kwargs:
 307             kwargs["case_name"] = 'neutron_trunk'
 308         TempestCommon.__init__(self, **kwargs)
 309         self.mode = "'neutron_tempest_plugin.(api|scenario).test_trunk'"
 310         self.res_dir = os.path.join(
 311             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 312         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 313         self.list = os.path.join(self.res_dir, 'test_list.txt')
 314
 315     def configure(self, **kwargs):
 316         super(TempestNeutronTrunk, self).configure(**kwargs)
 317         rconfig = configparser.RawConfigParser()
 318         rconfig.read(self.conf_file)
 319         rconfig.set('network-feature-enabled', 'api_extensions', 'all')
 320         with open(self.conf_file, 'wb') as config_file:
 321             rconfig.write(config_file)
 322
 323
 324 class TempestSmokeParallel(TempestCommon):
 325     """Tempest smoke parallel testcase implementation."""
 326     def __init__(self, **kwargs):
 327         if "case_name" not in kwargs:
 328             kwargs["case_name"] = 'tempest_smoke_parallel'
 329         TempestCommon.__init__(self, **kwargs)
 330         self.mode = "smoke"
 331
 332
 333 class TempestFullParallel(TempestCommon):
 334     """Tempest full parallel testcase implementation."""
 335     def __init__(self, **kwargs):
 336         if "case_name" not in kwargs:
 337             kwargs["case_name"] = 'tempest_full_parallel'
 338         TempestCommon.__init__(self, **kwargs)
 339         self.mode = "full"
 340
 341
 342 class TempestCustom(TempestCommon):
 343     """Tempest custom testcase implementation."""
 344     def __init__(self, **kwargs):
 345         if "case_name" not in kwargs:
 346             kwargs["case_name"] = 'tempest_custom'
 347         TempestCommon.__init__(self, **kwargs)
 348         self.mode = "custom"
 349         self.option = ["--concurrency", "1"]
 350
 351
 352 class TempestDefcore(TempestCommon):
 353     """Tempest Defcore testcase implementation."""
 354     def __init__(self, **kwargs):
 355         if "case_name" not in kwargs:
 356             kwargs["case_name"] = 'tempest_defcore'
 357         TempestCommon.__init__(self, **kwargs)
 358         self.mode = "defcore"
 359         self.option = ["--concurrency", "1"]
 360
 361
 362 class TempestResourcesManager(object):
 363     # pylint: disable=too-many-instance-attributes
 364     """Tempest resource manager."""
 365     def __init__(self):
 366         self.guid = '-' + str(uuid.uuid4())
 367         self.cloud = os_client_config.make_shade()
 368         LOGGER.debug("cloud: %s", self.cloud)
 369         self.domain = self.cloud.get_domain(
 370             name_or_id=self.cloud.auth.get(
 371                 "project_domain_name", "Default"))
 372         LOGGER.debug("domain: %s", self.domain)
 373         self.project = None
 374         self.user = None
 375         self.network = None
 376         self.subnet = None
 377         self.image = None
 378         self.image_alt = None
 379         self.flavor = None
 380         self.flavor_alt = None
 381
 382     def _create_project(self):
 383         """Create project for tests."""
 384         self.project = self.cloud.create_project(
 385             getattr(config.CONF, 'tempest_identity_tenant_name') + self.guid,
 386             description=getattr(
 387                 config.CONF, 'tempest_identity_tenant_description'),
 388             domain_id=self.domain.id)
 389         LOGGER.debug("project: %s", self.project)
 390
 391     def _create_user(self):
 392         """Create user for tests."""
 393         self.user = self.cloud.create_user(
 394             name=getattr(
 395                 config.CONF, 'tempest_identity_user_name') + self.guid,
 396             password=getattr(config.CONF, 'tempest_identity_user_password'),
 397             default_project=getattr(
 398                 config.CONF, 'tempest_identity_tenant_name') + self.guid,
 399             domain_id=self.domain.id)
 400         LOGGER.debug("user: %s", self.user)
 401
 402     def _create_network(self):
 403         """Create network for tests."""
 404         tempest_net_name = getattr(
 405             config.CONF, 'tempest_private_net_name') + self.guid
 406         provider = {}
 407         if hasattr(config.CONF, 'tempest_network_type'):
 408             provider["network_type"] = getattr(
 409                 config.CONF, 'tempest_network_type')
 410         if hasattr(config.CONF, 'tempest_physical_network'):
 411             provider["physical_network"] = getattr(
 412                 config.CONF, 'tempest_physical_network')
 413         if hasattr(config.CONF, 'tempest_segmentation_id'):
 414             provider["segmentation_id"] = getattr(
 415                 config.CONF, 'tempest_segmentation_id')
 416         LOGGER.info(
 417             "Creating network with name: '%s'", tempest_net_name)
 418         self.network = self.cloud.create_network(
 419             tempest_net_name, provider=provider)
 420         LOGGER.debug("network: %s", self.network)
 421
 422         self.subnet = self.cloud.create_subnet(
 423             self.network.id,
 424             subnet_name=getattr(
 425                 config.CONF, 'tempest_private_subnet_name') + self.guid,
 426             cidr=getattr(config.CONF, 'tempest_private_subnet_cidr'),
 427             enable_dhcp=True,
 428             dns_nameservers=[env.get('NAMESERVER')])
 429         LOGGER.debug("subnet: %s", self.subnet)
 430
 431     def _create_image(self, name):
 432         """Create image for tests"""
 433         LOGGER.info("Creating image with name: '%s'", name)
 434         meta = getattr(config.CONF, 'openstack_extra_properties', None)
 435         image = self.cloud.create_image(
 436             name, filename=getattr(config.CONF, 'openstack_image_url'),
 437             is_public=True, meta=meta)
 438         LOGGER.debug("image: %s", image)
 439         return image
 440
 441     def _create_flavor(self, name):
 442         """Create flavor for tests."""
 443         flavor = self.cloud.create_flavor(
 444             name, getattr(config.CONF, 'openstack_flavor_ram'),
 445             getattr(config.CONF, 'openstack_flavor_vcpus'),
 446             getattr(config.CONF, 'openstack_flavor_disk'))
 447         self.cloud.set_flavor_specs(
 448             flavor.id, getattr(config.CONF, 'flavor_extra_specs', {}))
 449         LOGGER.debug("flavor: %s", flavor)
 450         return flavor
 451
 452     def create(self, create_project=False):
 453         """Create resources for Tempest test suite."""
 454         if create_project:
 455             self._create_project()
 456             self._create_user()
 457         self._create_network()
 458
 459         LOGGER.debug("Creating two images for Tempest suite")
 460         self.image = self._create_image(
 461             getattr(config.CONF, 'openstack_image_name') + self.guid)
 462         self.image_alt = self._create_image(
 463             getattr(config.CONF, 'openstack_image_name_alt') + self.guid)
 464
 465         LOGGER.info("Creating two flavors for Tempest suite")
 466         self.flavor = self._create_flavor(
 467             getattr(config.CONF, 'openstack_flavor_name') + self.guid)
 468         self.flavor_alt = self._create_flavor(
 469             getattr(config.CONF, 'openstack_flavor_name_alt') + self.guid)
 470
 471     def cleanup(self):
 472         """
 473         Cleanup all OpenStack objects. Should be called on completion.
 474         """
 475         self.cloud.delete_image(self.image)
 476         self.cloud.delete_image(self.image_alt)
 477         self.cloud.delete_network(self.network.id)
 478         self.cloud.delete_flavor(self.flavor.id)
 479         self.cloud.delete_flavor(self.flavor_alt.id)
 480         if self.project:
 481             self.cloud.delete_user(self.user.id)
 482             self.cloud.delete_project(self.project.id)