functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     def __init__(self, **kwargs):
  48         super(TempestCommon, self).__init__(**kwargs)
  49         self.resources = TempestResourcesManager(**kwargs)
  50         self.mode = ""
  51         self.option = []
  52         self.verifier_id = conf_utils.get_verifier_id()
  53         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  54             self.verifier_id)
  55         self.deployment_id = conf_utils.get_verifier_deployment_id()
  56         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  57             self.verifier_id, self.deployment_id)
  58         self.verification_id = None
  59
  60     @staticmethod
  61     def read_file(filename):
  62         """Read file and return content as a stripped list."""
  63         with open(filename) as src:
  64             return [line.strip() for line in src.readlines()]
  65
  66     @staticmethod
  67     def get_verifier_result(verif_id):
  68         """Retrieve verification results."""
  69         result = {
  70             'num_tests': 0,
  71             'num_success': 0,
  72             'num_failures': 0,
  73             'num_skipped': 0
  74         }
  75         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  76         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  77         proc = subprocess.Popen(cmd,
  78                                 stdout=subprocess.PIPE,
  79                                 stderr=subprocess.STDOUT)
  80         for line in proc.stdout:
  81             new_line = line.replace(' ', '').split('|')
  82             if 'Tests' in new_line:
  83                 break
  84             LOGGER.info(line)
  85             if 'Testscount' in new_line:
  86                 result['num_tests'] = int(new_line[2])
  87             elif 'Success' in new_line:
  88                 result['num_success'] = int(new_line[2])
  89             elif 'Skipped' in new_line:
  90                 result['num_skipped'] = int(new_line[2])
  91             elif 'Failures' in new_line:
  92                 result['num_failures'] = int(new_line[2])
  93         return result
  94
  95     def generate_test_list(self, verifier_repo_dir):
  96         """Generate test list based on the test mode."""
  97         LOGGER.debug("Generating test case list...")
  98         if self.mode == 'custom':
  99             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 100                 shutil.copyfile(
 101                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 102             else:
 103                 raise Exception("Tempest test list file %s NOT found."
 104                                 % conf_utils.TEMPEST_CUSTOM)
 105         else:
 106             if self.mode == 'smoke':
 107                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 108             elif self.mode == 'full':
 109                 testr_mode = r"'^tempest\.'"
 110             else:
 111                 testr_mode = self.mode
 112             cmd = ("cd {0};"
 113                    "testr list-tests {1} > {2};"
 114                    "cd -;".format(verifier_repo_dir,
 115                                   testr_mode,
 116                                   conf_utils.TEMPEST_RAW_LIST))
 117             functest_utils.execute_command(cmd)
 118
 119     def apply_tempest_blacklist(self):
 120         """Exclude blacklisted test cases."""
 121         LOGGER.debug("Applying tempest blacklist...")
 122         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 123         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 124         black_tests = []
 125         try:
 126             installer_type = env.get('INSTALLER_TYPE')
 127             deploy_scenario = env.get('DEPLOY_SCENARIO')
 128             if bool(installer_type) * bool(deploy_scenario):
 129                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 130                 # file
 131                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 132                 black_list_yaml = yaml.safe_load(black_list_file)
 133                 black_list_file.close()
 134                 for item in black_list_yaml:
 135                     scenarios = item['scenarios']
 136                     installers = item['installers']
 137                     if (deploy_scenario in scenarios and
 138                             installer_type in installers):
 139                         tests = item['tests']
 140                         for test in tests:
 141                             black_tests.append(test)
 142                         break
 143         except Exception:  # pylint: disable=broad-except
 144             black_tests = []
 145             LOGGER.debug("Tempest blacklist file does not exist.")
 146
 147         for cases_line in cases_file:
 148             for black_tests_line in black_tests:
 149                 if black_tests_line in cases_line:
 150                     break
 151             else:
 152                 result_file.write(str(cases_line) + '\n')
 153         result_file.close()
 154
 155     def run_verifier_tests(self):
 156         """Execute tempest test cases."""
 157         cmd = ["rally", "verify", "start", "--load-list",
 158                conf_utils.TEMPEST_LIST]
 159         cmd.extend(self.option)
 160         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 161
 162         f_stdout = open(
 163             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 164         f_stderr = open(
 165             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 166                          "tempest-error.log"), 'w+')
 167
 168         proc = subprocess.Popen(
 169             cmd,
 170             stdout=subprocess.PIPE,
 171             stderr=f_stderr,
 172             bufsize=1)
 173
 174         with proc.stdout:
 175             for line in iter(proc.stdout.readline, b''):
 176                 if re.search(r"\} tempest\.", line):
 177                     LOGGER.info(line.replace('\n', ''))
 178                 elif re.search('Starting verification', line):
 179                     LOGGER.info(line.replace('\n', ''))
 180                     first_pos = line.index("UUID=") + len("UUID=")
 181                     last_pos = line.index(") for deployment")
 182                     self.verification_id = line[first_pos:last_pos]
 183                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 184                 f_stdout.write(line)
 185         proc.wait()
 186
 187         f_stdout.close()
 188         f_stderr.close()
 189
 190         if self.verification_id is None:
 191             raise Exception('Verification UUID not found')
 192
 193     def parse_verifier_result(self):
 194         """Parse and save test results."""
 195         stat = self.get_verifier_result(self.verification_id)
 196         try:
 197             num_executed = stat['num_tests'] - stat['num_skipped']
 198             try:
 199                 self.result = 100 * stat['num_success'] / num_executed
 200             except ZeroDivisionError:
 201                 self.result = 0
 202                 if stat['num_tests'] > 0:
 203                     LOGGER.info("All tests have been skipped")
 204                 else:
 205                     LOGGER.error("No test has been executed")
 206                     return
 207
 208             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 209                                    "tempest.log"), 'r') as logfile:
 210                 output = logfile.read()
 211
 212             success_testcases = []
 213             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 214                 success_testcases.append(match)
 215             failed_testcases = []
 216             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 217                 failed_testcases.append(match)
 218             skipped_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 220                 skipped_testcases.append(match)
 221
 222             self.details = {"tests": stat['num_tests'],
 223                             "failures": stat['num_failures'],
 224                             "success": success_testcases,
 225                             "skipped": skipped_testcases,
 226                             "errors": failed_testcases}
 227         except Exception:  # pylint: disable=broad-except
 228             self.result = 0
 229
 230         LOGGER.info("Tempest %s success_rate is %s%%",
 231                     self.case_name, self.result)
 232
 233     def generate_report(self):
 234         """Generate verification report."""
 235         html_file = os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 236                                  "tempest-report.html")
 237         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 238                self.verification_id, "--to", html_file]
 239         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 240                          stderr=subprocess.STDOUT)
 241
 242     def run(self, **kwargs):
 243
 244         self.start_time = time.time()
 245         try:
 246             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 247                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 248             resources = self.resources.create()
 249             compute_cnt = snaps_utils.get_active_compute_cnt(
 250                 self.resources.os_creds)
 251             conf_utils.configure_tempest(
 252                 self.deployment_dir,
 253                 network_name=resources.get("network_name"),
 254                 image_id=resources.get("image_id"),
 255                 flavor_id=resources.get("flavor_id"),
 256                 compute_cnt=compute_cnt)
 257             self.generate_test_list(self.verifier_repo_dir)
 258             self.apply_tempest_blacklist()
 259             self.run_verifier_tests()
 260             self.parse_verifier_result()
 261             self.generate_report()
 262             res = testcase.TestCase.EX_OK
 263         except Exception as err:  # pylint: disable=broad-except
 264             LOGGER.error('Error with run: %s', err)
 265             res = testcase.TestCase.EX_RUN_ERROR
 266         finally:
 267             self.resources.cleanup()
 268
 269         self.stop_time = time.time()
 270         return res
 271
 272
 273 class TempestSmokeSerial(TempestCommon):
 274     """Tempest smoke serial testcase implementation."""
 275     def __init__(self, **kwargs):
 276         if "case_name" not in kwargs:
 277             kwargs["case_name"] = 'tempest_smoke_serial'
 278         TempestCommon.__init__(self, **kwargs)
 279         self.mode = "smoke"
 280         self.option = ["--concurrency", "1"]
 281
 282
 283 class TempestSmokeParallel(TempestCommon):
 284     """Tempest smoke parallel testcase implementation."""
 285     def __init__(self, **kwargs):
 286         if "case_name" not in kwargs:
 287             kwargs["case_name"] = 'tempest_smoke_parallel'
 288         TempestCommon.__init__(self, **kwargs)
 289         self.mode = "smoke"
 290
 291
 292 class TempestFullParallel(TempestCommon):
 293     """Tempest full parallel testcase implementation."""
 294     def __init__(self, **kwargs):
 295         if "case_name" not in kwargs:
 296             kwargs["case_name"] = 'tempest_full_parallel'
 297         TempestCommon.__init__(self, **kwargs)
 298         self.mode = "full"
 299
 300
 301 class TempestCustom(TempestCommon):
 302     """Tempest custom testcase implementation."""
 303     def __init__(self, **kwargs):
 304         if "case_name" not in kwargs:
 305             kwargs["case_name"] = 'tempest_custom'
 306         TempestCommon.__init__(self, **kwargs)
 307         self.mode = "custom"
 308         self.option = ["--concurrency", "1"]
 309
 310
 311 class TempestDefcore(TempestCommon):
 312     """Tempest Defcore testcase implementation."""
 313     def __init__(self, **kwargs):
 314         if "case_name" not in kwargs:
 315             kwargs["case_name"] = 'tempest_defcore'
 316         TempestCommon.__init__(self, **kwargs)
 317         self.mode = "defcore"
 318         self.option = ["--concurrency", "1"]
 319
 320
 321 class TempestResourcesManager(object):
 322     """Tempest resource manager."""
 323     def __init__(self, **kwargs):
 324         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 325         self.guid = '-' + str(uuid.uuid4())
 326         self.creators = list()
 327         self.cirros_image_config = getattr(
 328             config.CONF, 'snaps_images_cirros', None)
 329
 330     def _create_project(self):
 331         """Create project for tests."""
 332         project_creator = deploy_utils.create_project(
 333             self.os_creds, ProjectConfig(
 334                 name=getattr(
 335                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 336                 description=getattr(
 337                     config.CONF, 'tempest_identity_tenant_description'),
 338                 domain=self.os_creds.project_domain_name))
 339         if project_creator is None or project_creator.get_project() is None:
 340             raise Exception("Failed to create tenant")
 341         self.creators.append(project_creator)
 342         return project_creator.get_project().id
 343
 344     def _create_user(self):
 345         """Create user for tests."""
 346         user_creator = deploy_utils.create_user(
 347             self.os_creds, UserConfig(
 348                 name=getattr(
 349                     config.CONF, 'tempest_identity_user_name') + self.guid,
 350                 password=getattr(
 351                     config.CONF, 'tempest_identity_user_password'),
 352                 project_name=getattr(
 353                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 354                 domain_name=self.os_creds.user_domain_name))
 355         if user_creator is None or user_creator.get_user() is None:
 356             raise Exception("Failed to create user")
 357         self.creators.append(user_creator)
 358         return user_creator.get_user().id
 359
 360     def _create_network(self, project_name):
 361         """Create network for tests."""
 362         tempest_network_type = None
 363         tempest_physical_network = None
 364         tempest_segmentation_id = None
 365
 366         tempest_network_type = getattr(
 367             config.CONF, 'tempest_network_type', None)
 368         tempest_physical_network = getattr(
 369             config.CONF, 'tempest_physical_network', None)
 370         tempest_segmentation_id = getattr(
 371             config.CONF, 'tempest_segmentation_id', None)
 372         tempest_net_name = getattr(
 373             config.CONF, 'tempest_private_net_name') + self.guid
 374
 375         network_creator = deploy_utils.create_network(
 376             self.os_creds, NetworkConfig(
 377                 name=tempest_net_name,
 378                 project_name=project_name,
 379                 network_type=tempest_network_type,
 380                 physical_network=tempest_physical_network,
 381                 segmentation_id=tempest_segmentation_id,
 382                 subnet_settings=[SubnetConfig(
 383                     name=getattr(
 384                         config.CONF,
 385                         'tempest_private_subnet_name') + self.guid,
 386                     project_name=project_name,
 387                     cidr=getattr(
 388                         config.CONF, 'tempest_private_subnet_cidr'))]))
 389         if network_creator is None or network_creator.get_network() is None:
 390             raise Exception("Failed to create private network")
 391         self.creators.append(network_creator)
 392         return tempest_net_name
 393
 394     def _create_image(self, name):
 395         """Create image for tests"""
 396         os_image_settings = openstack_tests.cirros_image_settings(
 397             name, public=True,
 398             image_metadata=self.cirros_image_config)
 399         image_creator = deploy_utils.create_image(
 400             self.os_creds, os_image_settings)
 401         if image_creator is None:
 402             raise Exception('Failed to create image')
 403         self.creators.append(image_creator)
 404         return image_creator.get_image().id
 405
 406     def _create_flavor(self, name):
 407         """Create flavor for tests."""
 408         scenario = env.get('DEPLOY_SCENARIO')
 409         flavor_metadata = None
 410         if 'ovs' in scenario or 'fdio' in scenario:
 411             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 412         flavor_creator = OpenStackFlavor(
 413             self.os_creds, FlavorConfig(
 414                 name=name,
 415                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 416                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 417                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 418                 metadata=flavor_metadata))
 419         flavor = flavor_creator.create()
 420         if flavor is None:
 421             raise Exception('Failed to create flavor')
 422         self.creators.append(flavor_creator)
 423         return flavor.id
 424
 425     def create(self, use_custom_images=False, use_custom_flavors=False,
 426                create_project=False):
 427         """Create resources for Tempest test suite."""
 428         result = {
 429             'tempest_net_name': None,
 430             'image_id': None,
 431             'image_id_alt': None,
 432             'flavor_id': None,
 433             'flavor_id_alt': None
 434         }
 435         project_name = None
 436
 437         if create_project:
 438             LOGGER.debug("Creating project and user for Tempest suite")
 439             project_name = getattr(
 440                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 441             result['project_id'] = self._create_project()
 442             result['user_id'] = self._create_user()
 443             result['tenant_id'] = result['project_id']  # for compatibility
 444
 445         LOGGER.debug("Creating private network for Tempest suite")
 446         result['tempest_net_name'] = self._create_network(project_name)
 447
 448         LOGGER.debug("Creating image for Tempest suite")
 449         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 450         result['image_id'] = self._create_image(image_name)
 451
 452         if use_custom_images:
 453             LOGGER.debug("Creating 2nd image for Tempest suite")
 454             image_name = getattr(
 455                 config.CONF, 'openstack_image_name_alt') + self.guid
 456             result['image_id_alt'] = self._create_image(image_name)
 457
 458         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 459                 use_custom_flavors):
 460             LOGGER.info("Creating flavor for Tempest suite")
 461             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 462             result['flavor_id'] = self._create_flavor(name)
 463
 464         if use_custom_flavors:
 465             LOGGER.info("Creating 2nd flavor for Tempest suite")
 466             scenario = env.get('DEPLOY_SCENARIO')
 467             if 'ovs' in scenario or 'fdio' in scenario:
 468                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 469             name = getattr(
 470                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 471             result['flavor_id_alt'] = self._create_flavor(name)
 472
 473         return result
 474
 475     def cleanup(self):
 476         """
 477         Cleanup all OpenStack objects. Should be called on completion.
 478         """
 479         for creator in reversed(self.creators):
 480             try:
 481                 creator.clean()
 482             except Exception as err:  # pylint: disable=broad-except
 483                 LOGGER.error('Unexpected error cleaning - %s', err)