functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack.create_flavor import OpenStackFlavor
  28 from snaps.openstack.tests import openstack_tests
  29 from snaps.openstack.utils import deploy_utils
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.opnfv_tests.openstack.snaps import snaps_utils
  34 from functest.opnfv_tests.openstack.tempest import conf_utils
  35 from functest.utils import config
  36 from functest.utils import env
  37 from functest.utils import functest_utils
  38
  39 LOGGER = logging.getLogger(__name__)
  40
  41
  42 class TempestCommon(testcase.TestCase):
  43     # pylint: disable=too-many-instance-attributes
  44     """TempestCommon testcases implementation class."""
  45
  46     TEMPEST_RESULTS_DIR = os.path.join(
  47         getattr(config.CONF, 'dir_results'), 'tempest')
  48
  49     def __init__(self, **kwargs):
  50         super(TempestCommon, self).__init__(**kwargs)
  51         self.resources = TempestResourcesManager(**kwargs)
  52         self.mode = ""
  53         self.option = []
  54         self.verifier_id = conf_utils.get_verifier_id()
  55         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  56             self.verifier_id)
  57         self.deployment_id = conf_utils.get_verifier_deployment_id()
  58         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  59             self.verifier_id, self.deployment_id)
  60         self.verification_id = None
  61         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  62         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  63         self.list = os.path.join(self.res_dir, 'test_list.txt')
  64         self.conf_file = None
  65
  66     @staticmethod
  67     def read_file(filename):
  68         """Read file and return content as a stripped list."""
  69         with open(filename) as src:
  70             return [line.strip() for line in src.readlines()]
  71
  72     @staticmethod
  73     def get_verifier_result(verif_id):
  74         """Retrieve verification results."""
  75         result = {
  76             'num_tests': 0,
  77             'num_success': 0,
  78             'num_failures': 0,
  79             'num_skipped': 0
  80         }
  81         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  82         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  83         proc = subprocess.Popen(cmd,
  84                                 stdout=subprocess.PIPE,
  85                                 stderr=subprocess.STDOUT)
  86         for line in proc.stdout:
  87             new_line = line.replace(' ', '').split('|')
  88             if 'Tests' in new_line:
  89                 break
  90             LOGGER.info(line)
  91             if 'Testscount' in new_line:
  92                 result['num_tests'] = int(new_line[2])
  93             elif 'Success' in new_line:
  94                 result['num_success'] = int(new_line[2])
  95             elif 'Skipped' in new_line:
  96                 result['num_skipped'] = int(new_line[2])
  97             elif 'Failures' in new_line:
  98                 result['num_failures'] = int(new_line[2])
  99         return result
 100
 101     def generate_test_list(self):
 102         """Generate test list based on the test mode."""
 103         LOGGER.debug("Generating test case list...")
 104         if self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, self.list)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 114             elif self.mode == 'full':
 115                 testr_mode = r"'^tempest\.'"
 116             else:
 117                 testr_mode = self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(self.verifier_repo_dir,
 121                                   testr_mode,
 122                                   self.list))
 123             functest_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search('Starting verification', line):
 188                     LOGGER.info(line.replace('\n', ''))
 189                     first_pos = line.index("UUID=") + len("UUID=")
 190                     last_pos = line.index(") for deployment")
 191                     self.verification_id = line[first_pos:last_pos]
 192                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 193                 f_stdout.write(line)
 194         proc.wait()
 195
 196         f_stdout.close()
 197         f_stderr.close()
 198
 199         if self.verification_id is None:
 200             raise Exception('Verification UUID not found')
 201
 202     def parse_verifier_result(self):
 203         """Parse and save test results."""
 204         stat = self.get_verifier_result(self.verification_id)
 205         try:
 206             num_executed = stat['num_tests'] - stat['num_skipped']
 207             try:
 208                 self.result = 100 * stat['num_success'] / num_executed
 209             except ZeroDivisionError:
 210                 self.result = 0
 211                 if stat['num_tests'] > 0:
 212                     LOGGER.info("All tests have been skipped")
 213                 else:
 214                     LOGGER.error("No test has been executed")
 215                     return
 216
 217             with open(os.path.join(self.res_dir,
 218                                    "tempest.log"), 'r') as logfile:
 219                 output = logfile.read()
 220
 221             success_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 223                 success_testcases.append(match)
 224             failed_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 226                 failed_testcases.append(match)
 227             skipped_testcases = []
 228             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests": stat['num_tests'],
 232                             "failures": stat['num_failures'],
 233                             "success": success_testcases,
 234                             "skipped": skipped_testcases,
 235                             "errors": failed_testcases}
 236         except Exception:  # pylint: disable=broad-except
 237             self.result = 0
 238
 239         LOGGER.info("Tempest %s success_rate is %s%%",
 240                     self.case_name, self.result)
 241
 242     def generate_report(self):
 243         """Generate verification report."""
 244         html_file = os.path.join(self.res_dir,
 245                                  "tempest-report.html")
 246         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 247                self.verification_id, "--to", html_file]
 248         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 249                          stderr=subprocess.STDOUT)
 250
 251     def configure(self):
 252         """
 253         Create all openstack resources for tempest-based testcases and write
 254         tempest.conf.
 255         """
 256         if not os.path.exists(self.res_dir):
 257             os.makedirs(self.res_dir)
 258         resources = self.resources.create()
 259         compute_cnt = snaps_utils.get_active_compute_cnt(
 260             self.resources.os_creds)
 261         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 262         conf_utils.configure_tempest_update_params(
 263             self.conf_file, self.res_dir,
 264             network_name=resources.get("network_name"),
 265             image_id=resources.get("image_id"),
 266             flavor_id=resources.get("flavor_id"),
 267             compute_cnt=compute_cnt)
 268
 269     def run(self, **kwargs):
 270         self.start_time = time.time()
 271         try:
 272             self.configure()
 273             self.generate_test_list()
 274             self.apply_tempest_blacklist()
 275             self.run_verifier_tests()
 276             self.parse_verifier_result()
 277             self.generate_report()
 278             res = testcase.TestCase.EX_OK
 279         except Exception:  # pylint: disable=broad-except
 280             LOGGER.exception('Error with run')
 281             res = testcase.TestCase.EX_RUN_ERROR
 282         finally:
 283             self.resources.cleanup()
 284         self.stop_time = time.time()
 285         return res
 286
 287
 288 class TempestSmokeSerial(TempestCommon):
 289     """Tempest smoke serial testcase implementation."""
 290     def __init__(self, **kwargs):
 291         if "case_name" not in kwargs:
 292             kwargs["case_name"] = 'tempest_smoke_serial'
 293         TempestCommon.__init__(self, **kwargs)
 294         self.mode = "smoke"
 295         self.option = ["--concurrency", "1"]
 296
 297
 298 class TempestNeutronTrunk(TempestCommon):
 299     """Tempest neutron trunk testcase implementation."""
 300     def __init__(self, **kwargs):
 301         if "case_name" not in kwargs:
 302             kwargs["case_name"] = 'neutron_trunk'
 303         TempestCommon.__init__(self, **kwargs)
 304         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 305         self.res_dir = os.path.join(
 306             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 307         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 308         self.list = os.path.join(self.res_dir, 'test_list.txt')
 309
 310
 311 class TempestSmokeParallel(TempestCommon):
 312     """Tempest smoke parallel testcase implementation."""
 313     def __init__(self, **kwargs):
 314         if "case_name" not in kwargs:
 315             kwargs["case_name"] = 'tempest_smoke_parallel'
 316         TempestCommon.__init__(self, **kwargs)
 317         self.mode = "smoke"
 318
 319
 320 class TempestFullParallel(TempestCommon):
 321     """Tempest full parallel testcase implementation."""
 322     def __init__(self, **kwargs):
 323         if "case_name" not in kwargs:
 324             kwargs["case_name"] = 'tempest_full_parallel'
 325         TempestCommon.__init__(self, **kwargs)
 326         self.mode = "full"
 327
 328
 329 class TempestCustom(TempestCommon):
 330     """Tempest custom testcase implementation."""
 331     def __init__(self, **kwargs):
 332         if "case_name" not in kwargs:
 333             kwargs["case_name"] = 'tempest_custom'
 334         TempestCommon.__init__(self, **kwargs)
 335         self.mode = "custom"
 336         self.option = ["--concurrency", "1"]
 337
 338
 339 class TempestDefcore(TempestCommon):
 340     """Tempest Defcore testcase implementation."""
 341     def __init__(self, **kwargs):
 342         if "case_name" not in kwargs:
 343             kwargs["case_name"] = 'tempest_defcore'
 344         TempestCommon.__init__(self, **kwargs)
 345         self.mode = "defcore"
 346         self.option = ["--concurrency", "1"]
 347
 348
 349 class TempestResourcesManager(object):
 350     """Tempest resource manager."""
 351     def __init__(self, **kwargs):
 352         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 353         self.guid = '-' + str(uuid.uuid4())
 354         self.creators = list()
 355         self.cirros_image_config = getattr(
 356             config.CONF, 'snaps_images_cirros', None)
 357
 358     def _create_project(self):
 359         """Create project for tests."""
 360         project_creator = deploy_utils.create_project(
 361             self.os_creds, ProjectConfig(
 362                 name=getattr(
 363                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 364                 description=getattr(
 365                     config.CONF, 'tempest_identity_tenant_description'),
 366                 domain=self.os_creds.project_domain_name))
 367         if project_creator is None or project_creator.get_project() is None:
 368             raise Exception("Failed to create tenant")
 369         self.creators.append(project_creator)
 370         return project_creator.get_project().id
 371
 372     def _create_user(self):
 373         """Create user for tests."""
 374         user_creator = deploy_utils.create_user(
 375             self.os_creds, UserConfig(
 376                 name=getattr(
 377                     config.CONF, 'tempest_identity_user_name') + self.guid,
 378                 password=getattr(
 379                     config.CONF, 'tempest_identity_user_password'),
 380                 project_name=getattr(
 381                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 382                 domain_name=self.os_creds.user_domain_name))
 383         if user_creator is None or user_creator.get_user() is None:
 384             raise Exception("Failed to create user")
 385         self.creators.append(user_creator)
 386         return user_creator.get_user().id
 387
 388     def _create_network(self, project_name):
 389         """Create network for tests."""
 390         tempest_network_type = None
 391         tempest_physical_network = None
 392         tempest_segmentation_id = None
 393
 394         tempest_network_type = getattr(
 395             config.CONF, 'tempest_network_type', None)
 396         tempest_physical_network = getattr(
 397             config.CONF, 'tempest_physical_network', None)
 398         tempest_segmentation_id = getattr(
 399             config.CONF, 'tempest_segmentation_id', None)
 400         tempest_net_name = getattr(
 401             config.CONF, 'tempest_private_net_name') + self.guid
 402
 403         network_creator = deploy_utils.create_network(
 404             self.os_creds, NetworkConfig(
 405                 name=tempest_net_name,
 406                 project_name=project_name,
 407                 network_type=tempest_network_type,
 408                 physical_network=tempest_physical_network,
 409                 segmentation_id=tempest_segmentation_id,
 410                 subnet_settings=[SubnetConfig(
 411                     name=getattr(
 412                         config.CONF,
 413                         'tempest_private_subnet_name') + self.guid,
 414                     project_name=project_name,
 415                     cidr=getattr(
 416                         config.CONF, 'tempest_private_subnet_cidr'),
 417                     dns_nameservers=[env.get('NAMESERVER')])]))
 418         if network_creator is None or network_creator.get_network() is None:
 419             raise Exception("Failed to create private network")
 420         self.creators.append(network_creator)
 421         return tempest_net_name
 422
 423     def _create_image(self, name):
 424         """Create image for tests"""
 425         os_image_settings = openstack_tests.cirros_image_settings(
 426             name, public=True,
 427             image_metadata=self.cirros_image_config)
 428         image_creator = deploy_utils.create_image(
 429             self.os_creds, os_image_settings)
 430         if image_creator is None:
 431             raise Exception('Failed to create image')
 432         self.creators.append(image_creator)
 433         return image_creator.get_image().id
 434
 435     def _create_flavor(self, name):
 436         """Create flavor for tests."""
 437         flavor_metadata = getattr(config.CONF, 'flavor_extra_specs', None)
 438         flavor_creator = OpenStackFlavor(
 439             self.os_creds, FlavorConfig(
 440                 name=name,
 441                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 442                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 443                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 444                 metadata=flavor_metadata))
 445         flavor = flavor_creator.create()
 446         if flavor is None:
 447             raise Exception('Failed to create flavor')
 448         self.creators.append(flavor_creator)
 449         return flavor.id
 450
 451     def create(self, create_project=False):
 452         """Create resources for Tempest test suite."""
 453         result = {
 454             'tempest_net_name': None,
 455             'image_id': None,
 456             'image_id_alt': None,
 457             'flavor_id': None,
 458             'flavor_id_alt': None
 459         }
 460         project_name = None
 461
 462         if create_project:
 463             LOGGER.debug("Creating project and user for Tempest suite")
 464             project_name = getattr(
 465                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 466             result['project_id'] = self._create_project()
 467             result['user_id'] = self._create_user()
 468             result['tenant_id'] = result['project_id']  # for compatibility
 469
 470         LOGGER.debug("Creating private network for Tempest suite")
 471         result['tempest_net_name'] = self._create_network(project_name)
 472
 473         LOGGER.debug("Creating two images for Tempest suite")
 474         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 475         result['image_id'] = self._create_image(image_name)
 476         image_name = getattr(
 477             config.CONF, 'openstack_image_name_alt') + self.guid
 478         result['image_id_alt'] = self._create_image(image_name)
 479
 480         LOGGER.info("Creating two flavors for Tempest suite")
 481         name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 482         result['flavor_id'] = self._create_flavor(name)
 483
 484         name = getattr(
 485             config.CONF, 'openstack_flavor_name_alt') + self.guid
 486         result['flavor_id_alt'] = self._create_flavor(name)
 487
 488         return result
 489
 490     def cleanup(self):
 491         """
 492         Cleanup all OpenStack objects. Should be called on completion.
 493         """
 494         for creator in reversed(self.creators):
 495             try:
 496                 creator.clean()
 497             except Exception as err:  # pylint: disable=broad-except
 498                 LOGGER.error('Unexpected error cleaning - %s', err)