functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     TEMPEST_RESULTS_DIR = os.path.join(
  48         getattr(config.CONF, 'dir_results'), 'tempest')
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  63         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  64         self.list = os.path.join(self.res_dir, 'test_list.txt')
  65
  66     @staticmethod
  67     def read_file(filename):
  68         """Read file and return content as a stripped list."""
  69         with open(filename) as src:
  70             return [line.strip() for line in src.readlines()]
  71
  72     @staticmethod
  73     def get_verifier_result(verif_id):
  74         """Retrieve verification results."""
  75         result = {
  76             'num_tests': 0,
  77             'num_success': 0,
  78             'num_failures': 0,
  79             'num_skipped': 0
  80         }
  81         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  82         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  83         proc = subprocess.Popen(cmd,
  84                                 stdout=subprocess.PIPE,
  85                                 stderr=subprocess.STDOUT)
  86         for line in proc.stdout:
  87             new_line = line.replace(' ', '').split('|')
  88             if 'Tests' in new_line:
  89                 break
  90             LOGGER.info(line)
  91             if 'Testscount' in new_line:
  92                 result['num_tests'] = int(new_line[2])
  93             elif 'Success' in new_line:
  94                 result['num_success'] = int(new_line[2])
  95             elif 'Skipped' in new_line:
  96                 result['num_skipped'] = int(new_line[2])
  97             elif 'Failures' in new_line:
  98                 result['num_failures'] = int(new_line[2])
  99         return result
 100
 101     def generate_test_list(self, verifier_repo_dir):
 102         """Generate test list based on the test mode."""
 103         LOGGER.debug("Generating test case list...")
 104         if self.mode == 'custom':
 105             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 106                 shutil.copyfile(
 107                     conf_utils.TEMPEST_CUSTOM, self.list)
 108             else:
 109                 raise Exception("Tempest test list file %s NOT found."
 110                                 % conf_utils.TEMPEST_CUSTOM)
 111         else:
 112             if self.mode == 'smoke':
 113                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 114             elif self.mode == 'full':
 115                 testr_mode = r"'^tempest\.'"
 116             else:
 117                 testr_mode = self.mode
 118             cmd = ("cd {0};"
 119                    "testr list-tests {1} > {2};"
 120                    "cd -;".format(verifier_repo_dir,
 121                                   testr_mode,
 122                                   self.list))
 123             functest_utils.execute_command(cmd)
 124
 125     def apply_tempest_blacklist(self):
 126         """Exclude blacklisted test cases."""
 127         LOGGER.debug("Applying tempest blacklist...")
 128         if os.path.exists(self.raw_list):
 129             os.remove(self.raw_list)
 130         os.rename(self.list, self.raw_list)
 131         cases_file = self.read_file(self.raw_list)
 132         result_file = open(self.list, 'w')
 133         black_tests = []
 134         try:
 135             installer_type = env.get('INSTALLER_TYPE')
 136             deploy_scenario = env.get('DEPLOY_SCENARIO')
 137             if bool(installer_type) * bool(deploy_scenario):
 138                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 139                 # file
 140                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 141                 black_list_yaml = yaml.safe_load(black_list_file)
 142                 black_list_file.close()
 143                 for item in black_list_yaml:
 144                     scenarios = item['scenarios']
 145                     installers = item['installers']
 146                     if (deploy_scenario in scenarios and
 147                             installer_type in installers):
 148                         tests = item['tests']
 149                         for test in tests:
 150                             black_tests.append(test)
 151                         break
 152         except Exception:  # pylint: disable=broad-except
 153             black_tests = []
 154             LOGGER.debug("Tempest blacklist file does not exist.")
 155
 156         for cases_line in cases_file:
 157             for black_tests_line in black_tests:
 158                 if black_tests_line in cases_line:
 159                     break
 160             else:
 161                 result_file.write(str(cases_line) + '\n')
 162         result_file.close()
 163
 164     def run_verifier_tests(self):
 165         """Execute tempest test cases."""
 166         cmd = ["rally", "verify", "start", "--load-list",
 167                self.list]
 168         cmd.extend(self.option)
 169         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 170
 171         f_stdout = open(
 172             os.path.join(self.res_dir, "tempest.log"), 'w+')
 173         f_stderr = open(
 174             os.path.join(self.res_dir,
 175                          "tempest-error.log"), 'w+')
 176
 177         proc = subprocess.Popen(
 178             cmd,
 179             stdout=subprocess.PIPE,
 180             stderr=f_stderr,
 181             bufsize=1)
 182
 183         with proc.stdout:
 184             for line in iter(proc.stdout.readline, b''):
 185                 if re.search(r"\} tempest\.", line):
 186                     LOGGER.info(line.replace('\n', ''))
 187                 elif re.search('Starting verification', line):
 188                     LOGGER.info(line.replace('\n', ''))
 189                     first_pos = line.index("UUID=") + len("UUID=")
 190                     last_pos = line.index(") for deployment")
 191                     self.verification_id = line[first_pos:last_pos]
 192                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 193                 f_stdout.write(line)
 194         proc.wait()
 195
 196         f_stdout.close()
 197         f_stderr.close()
 198
 199         if self.verification_id is None:
 200             raise Exception('Verification UUID not found')
 201
 202     def parse_verifier_result(self):
 203         """Parse and save test results."""
 204         stat = self.get_verifier_result(self.verification_id)
 205         try:
 206             num_executed = stat['num_tests'] - stat['num_skipped']
 207             try:
 208                 self.result = 100 * stat['num_success'] / num_executed
 209             except ZeroDivisionError:
 210                 self.result = 0
 211                 if stat['num_tests'] > 0:
 212                     LOGGER.info("All tests have been skipped")
 213                 else:
 214                     LOGGER.error("No test has been executed")
 215                     return
 216
 217             with open(os.path.join(self.res_dir,
 218                                    "tempest.log"), 'r') as logfile:
 219                 output = logfile.read()
 220
 221             success_testcases = []
 222             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 223                 success_testcases.append(match)
 224             failed_testcases = []
 225             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 226                 failed_testcases.append(match)
 227             skipped_testcases = []
 228             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 229                 skipped_testcases.append(match)
 230
 231             self.details = {"tests": stat['num_tests'],
 232                             "failures": stat['num_failures'],
 233                             "success": success_testcases,
 234                             "skipped": skipped_testcases,
 235                             "errors": failed_testcases}
 236         except Exception:  # pylint: disable=broad-except
 237             self.result = 0
 238
 239         LOGGER.info("Tempest %s success_rate is %s%%",
 240                     self.case_name, self.result)
 241
 242     def generate_report(self):
 243         """Generate verification report."""
 244         html_file = os.path.join(self.res_dir,
 245                                  "tempest-report.html")
 246         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 247                self.verification_id, "--to", html_file]
 248         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 249                          stderr=subprocess.STDOUT)
 250
 251     def run(self, **kwargs):
 252
 253         self.start_time = time.time()
 254         try:
 255             if not os.path.exists(self.res_dir):
 256                 os.makedirs(self.res_dir)
 257             resources = self.resources.create()
 258             compute_cnt = snaps_utils.get_active_compute_cnt(
 259                 self.resources.os_creds)
 260             conf_utils.configure_tempest(
 261                 self.deployment_dir, self.res_dir,
 262                 network_name=resources.get("network_name"),
 263                 image_id=resources.get("image_id"),
 264                 flavor_id=resources.get("flavor_id"),
 265                 compute_cnt=compute_cnt)
 266             self.generate_test_list(self.verifier_repo_dir)
 267             self.apply_tempest_blacklist()
 268             self.run_verifier_tests()
 269             self.parse_verifier_result()
 270             self.generate_report()
 271             res = testcase.TestCase.EX_OK
 272         except Exception as err:  # pylint: disable=broad-except
 273             LOGGER.error('Error with run: %s', err)
 274             res = testcase.TestCase.EX_RUN_ERROR
 275         finally:
 276             self.resources.cleanup()
 277
 278         self.stop_time = time.time()
 279         return res
 280
 281
 282 class TempestSmokeSerial(TempestCommon):
 283     """Tempest smoke serial testcase implementation."""
 284     def __init__(self, **kwargs):
 285         if "case_name" not in kwargs:
 286             kwargs["case_name"] = 'tempest_smoke_serial'
 287         TempestCommon.__init__(self, **kwargs)
 288         self.mode = "smoke"
 289         self.option = ["--concurrency", "1"]
 290
 291
 292 class TempestNeutronTrunk(TempestCommon):
 293     """Tempest neutron trunk testcase implementation."""
 294     def __init__(self, **kwargs):
 295         if "case_name" not in kwargs:
 296             kwargs["case_name"] = 'neutron_trunk'
 297         TempestCommon.__init__(self, **kwargs)
 298         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 299         self.res_dir = os.path.join(
 300             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 301         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 302         self.list = os.path.join(self.res_dir, 'test_list.txt')
 303
 304
 305 class TempestSmokeParallel(TempestCommon):
 306     """Tempest smoke parallel testcase implementation."""
 307     def __init__(self, **kwargs):
 308         if "case_name" not in kwargs:
 309             kwargs["case_name"] = 'tempest_smoke_parallel'
 310         TempestCommon.__init__(self, **kwargs)
 311         self.mode = "smoke"
 312
 313
 314 class TempestFullParallel(TempestCommon):
 315     """Tempest full parallel testcase implementation."""
 316     def __init__(self, **kwargs):
 317         if "case_name" not in kwargs:
 318             kwargs["case_name"] = 'tempest_full_parallel'
 319         TempestCommon.__init__(self, **kwargs)
 320         self.mode = "full"
 321
 322
 323 class TempestCustom(TempestCommon):
 324     """Tempest custom testcase implementation."""
 325     def __init__(self, **kwargs):
 326         if "case_name" not in kwargs:
 327             kwargs["case_name"] = 'tempest_custom'
 328         TempestCommon.__init__(self, **kwargs)
 329         self.mode = "custom"
 330         self.option = ["--concurrency", "1"]
 331
 332
 333 class TempestDefcore(TempestCommon):
 334     """Tempest Defcore testcase implementation."""
 335     def __init__(self, **kwargs):
 336         if "case_name" not in kwargs:
 337             kwargs["case_name"] = 'tempest_defcore'
 338         TempestCommon.__init__(self, **kwargs)
 339         self.mode = "defcore"
 340         self.option = ["--concurrency", "1"]
 341
 342
 343 class TempestResourcesManager(object):
 344     """Tempest resource manager."""
 345     def __init__(self, **kwargs):
 346         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 347         self.guid = '-' + str(uuid.uuid4())
 348         self.creators = list()
 349         self.cirros_image_config = getattr(
 350             config.CONF, 'snaps_images_cirros', None)
 351
 352     def _create_project(self):
 353         """Create project for tests."""
 354         project_creator = deploy_utils.create_project(
 355             self.os_creds, ProjectConfig(
 356                 name=getattr(
 357                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 358                 description=getattr(
 359                     config.CONF, 'tempest_identity_tenant_description'),
 360                 domain=self.os_creds.project_domain_name))
 361         if project_creator is None or project_creator.get_project() is None:
 362             raise Exception("Failed to create tenant")
 363         self.creators.append(project_creator)
 364         return project_creator.get_project().id
 365
 366     def _create_user(self):
 367         """Create user for tests."""
 368         user_creator = deploy_utils.create_user(
 369             self.os_creds, UserConfig(
 370                 name=getattr(
 371                     config.CONF, 'tempest_identity_user_name') + self.guid,
 372                 password=getattr(
 373                     config.CONF, 'tempest_identity_user_password'),
 374                 project_name=getattr(
 375                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 376                 domain_name=self.os_creds.user_domain_name))
 377         if user_creator is None or user_creator.get_user() is None:
 378             raise Exception("Failed to create user")
 379         self.creators.append(user_creator)
 380         return user_creator.get_user().id
 381
 382     def _create_network(self, project_name):
 383         """Create network for tests."""
 384         tempest_network_type = None
 385         tempest_physical_network = None
 386         tempest_segmentation_id = None
 387
 388         tempest_network_type = getattr(
 389             config.CONF, 'tempest_network_type', None)
 390         tempest_physical_network = getattr(
 391             config.CONF, 'tempest_physical_network', None)
 392         tempest_segmentation_id = getattr(
 393             config.CONF, 'tempest_segmentation_id', None)
 394         tempest_net_name = getattr(
 395             config.CONF, 'tempest_private_net_name') + self.guid
 396
 397         network_creator = deploy_utils.create_network(
 398             self.os_creds, NetworkConfig(
 399                 name=tempest_net_name,
 400                 project_name=project_name,
 401                 network_type=tempest_network_type,
 402                 physical_network=tempest_physical_network,
 403                 segmentation_id=tempest_segmentation_id,
 404                 subnet_settings=[SubnetConfig(
 405                     name=getattr(
 406                         config.CONF,
 407                         'tempest_private_subnet_name') + self.guid,
 408                     project_name=project_name,
 409                     cidr=getattr(
 410                         config.CONF, 'tempest_private_subnet_cidr'),
 411                     dns_nameservers=[env.get('NAMESERVER')])]))
 412         if network_creator is None or network_creator.get_network() is None:
 413             raise Exception("Failed to create private network")
 414         self.creators.append(network_creator)
 415         return tempest_net_name
 416
 417     def _create_image(self, name):
 418         """Create image for tests"""
 419         os_image_settings = openstack_tests.cirros_image_settings(
 420             name, public=True,
 421             image_metadata=self.cirros_image_config)
 422         image_creator = deploy_utils.create_image(
 423             self.os_creds, os_image_settings)
 424         if image_creator is None:
 425             raise Exception('Failed to create image')
 426         self.creators.append(image_creator)
 427         return image_creator.get_image().id
 428
 429     def _create_flavor(self, name):
 430         """Create flavor for tests."""
 431         scenario = env.get('DEPLOY_SCENARIO')
 432         flavor_metadata = None
 433         if 'ovs' in scenario or 'fdio' in scenario:
 434             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 435         flavor_creator = OpenStackFlavor(
 436             self.os_creds, FlavorConfig(
 437                 name=name,
 438                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 439                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 440                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 441                 metadata=flavor_metadata))
 442         flavor = flavor_creator.create()
 443         if flavor is None:
 444             raise Exception('Failed to create flavor')
 445         self.creators.append(flavor_creator)
 446         return flavor.id
 447
 448     def create(self, use_custom_images=False, use_custom_flavors=False,
 449                create_project=False):
 450         """Create resources for Tempest test suite."""
 451         result = {
 452             'tempest_net_name': None,
 453             'image_id': None,
 454             'image_id_alt': None,
 455             'flavor_id': None,
 456             'flavor_id_alt': None
 457         }
 458         project_name = None
 459
 460         if create_project:
 461             LOGGER.debug("Creating project and user for Tempest suite")
 462             project_name = getattr(
 463                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 464             result['project_id'] = self._create_project()
 465             result['user_id'] = self._create_user()
 466             result['tenant_id'] = result['project_id']  # for compatibility
 467
 468         LOGGER.debug("Creating private network for Tempest suite")
 469         result['tempest_net_name'] = self._create_network(project_name)
 470
 471         LOGGER.debug("Creating image for Tempest suite")
 472         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 473         result['image_id'] = self._create_image(image_name)
 474
 475         if use_custom_images:
 476             LOGGER.debug("Creating 2nd image for Tempest suite")
 477             image_name = getattr(
 478                 config.CONF, 'openstack_image_name_alt') + self.guid
 479             result['image_id_alt'] = self._create_image(image_name)
 480
 481         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 482                 use_custom_flavors):
 483             LOGGER.info("Creating flavor for Tempest suite")
 484             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 485             result['flavor_id'] = self._create_flavor(name)
 486
 487         if use_custom_flavors:
 488             LOGGER.info("Creating 2nd flavor for Tempest suite")
 489             scenario = env.get('DEPLOY_SCENARIO')
 490             if 'ovs' in scenario or 'fdio' in scenario:
 491                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 492             name = getattr(
 493                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 494             result['flavor_id_alt'] = self._create_flavor(name)
 495
 496         return result
 497
 498     def cleanup(self):
 499         """
 500         Cleanup all OpenStack objects. Should be called on completion.
 501         """
 502         for creator in reversed(self.creators):
 503             try:
 504                 creator.clean()
 505             except Exception as err:  # pylint: disable=broad-except
 506                 LOGGER.error('Unexpected error cleaning - %s', err)