functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     def __init__(self, **kwargs):
  48         super(TempestCommon, self).__init__(**kwargs)
  49         self.resources = TempestResourcesManager(**kwargs)
  50         self.mode = ""
  51         self.option = []
  52         self.verifier_id = conf_utils.get_verifier_id()
  53         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  54             self.verifier_id)
  55         self.deployment_id = conf_utils.get_verifier_deployment_id()
  56         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  57             self.verifier_id, self.deployment_id)
  58         self.verification_id = None
  59
  60     @staticmethod
  61     def read_file(filename):
  62         """Read file and return content as a stripped list."""
  63         with open(filename) as src:
  64             return [line.strip() for line in src.readlines()]
  65
  66     @staticmethod
  67     def get_verifier_result(verif_id):
  68         """Retrieve verification results."""
  69         result = {
  70             'num_tests': 0,
  71             'num_success': 0,
  72             'num_failures': 0,
  73             'num_skipped': 0
  74         }
  75         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  76         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  77         proc = subprocess.Popen(cmd,
  78                                 stdout=subprocess.PIPE,
  79                                 stderr=subprocess.STDOUT)
  80         for line in proc.stdout:
  81             new_line = line.replace(' ', '').split('|')
  82             if 'Tests' in new_line:
  83                 break
  84             LOGGER.info(line)
  85             if 'Testscount' in new_line:
  86                 result['num_tests'] = int(new_line[2])
  87             elif 'Success' in new_line:
  88                 result['num_success'] = int(new_line[2])
  89             elif 'Skipped' in new_line:
  90                 result['num_skipped'] = int(new_line[2])
  91             elif 'Failures' in new_line:
  92                 result['num_failures'] = int(new_line[2])
  93         return result
  94
  95     def generate_test_list(self, verifier_repo_dir):
  96         """Generate test list based on the test mode."""
  97         LOGGER.debug("Generating test case list...")
  98         if self.mode == 'custom':
  99             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 100                 shutil.copyfile(
 101                     conf_utils.TEMPEST_CUSTOM, conf_utils.TEMPEST_RAW_LIST)
 102             else:
 103                 raise Exception("Tempest test list file %s NOT found."
 104                                 % conf_utils.TEMPEST_CUSTOM)
 105         else:
 106             if self.mode == 'smoke':
 107                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 108             elif self.mode == 'full':
 109                 testr_mode = r"'^tempest\.'"
 110             else:
 111                 testr_mode = self.mode
 112             cmd = ("cd {0};"
 113                    "testr list-tests {1} > {2};"
 114                    "cd -;".format(verifier_repo_dir,
 115                                   testr_mode,
 116                                   conf_utils.TEMPEST_RAW_LIST))
 117             functest_utils.execute_command(cmd)
 118
 119     def apply_tempest_blacklist(self):
 120         """Exclude blacklisted test cases."""
 121         LOGGER.debug("Applying tempest blacklist...")
 122         cases_file = self.read_file(conf_utils.TEMPEST_RAW_LIST)
 123         result_file = open(conf_utils.TEMPEST_LIST, 'w')
 124         black_tests = []
 125         try:
 126             installer_type = env.get('INSTALLER_TYPE')
 127             deploy_scenario = env.get('DEPLOY_SCENARIO')
 128             if bool(installer_type) * bool(deploy_scenario):
 129                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 130                 # file
 131                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 132                 black_list_yaml = yaml.safe_load(black_list_file)
 133                 black_list_file.close()
 134                 for item in black_list_yaml:
 135                     scenarios = item['scenarios']
 136                     installers = item['installers']
 137                     if (deploy_scenario in scenarios and
 138                             installer_type in installers):
 139                         tests = item['tests']
 140                         for test in tests:
 141                             black_tests.append(test)
 142                         break
 143         except Exception:  # pylint: disable=broad-except
 144             black_tests = []
 145             LOGGER.debug("Tempest blacklist file does not exist.")
 146
 147         for cases_line in cases_file:
 148             for black_tests_line in black_tests:
 149                 if black_tests_line in cases_line:
 150                     break
 151             else:
 152                 result_file.write(str(cases_line) + '\n')
 153         result_file.close()
 154
 155     def run_verifier_tests(self):
 156         """Execute tempest test cases."""
 157         cmd = ["rally", "verify", "start", "--load-list",
 158                conf_utils.TEMPEST_LIST]
 159         cmd.extend(self.option)
 160         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 161
 162         f_stdout = open(
 163             os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+')
 164         f_stderr = open(
 165             os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 166                          "tempest-error.log"), 'w+')
 167
 168         proc = subprocess.Popen(
 169             cmd,
 170             stdout=subprocess.PIPE,
 171             stderr=f_stderr,
 172             bufsize=1)
 173
 174         with proc.stdout:
 175             for line in iter(proc.stdout.readline, b''):
 176                 if re.search(r"\} tempest\.", line):
 177                     LOGGER.info(line.replace('\n', ''))
 178                 elif re.search('Starting verification', line):
 179                     LOGGER.info(line.replace('\n', ''))
 180                     first_pos = line.index("UUID=") + len("UUID=")
 181                     last_pos = line.index(") for deployment")
 182                     self.verification_id = line[first_pos:last_pos]
 183                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 184                 f_stdout.write(line)
 185         proc.wait()
 186
 187         f_stdout.close()
 188         f_stderr.close()
 189
 190         if self.verification_id is None:
 191             raise Exception('Verification UUID not found')
 192
 193     def parse_verifier_result(self):
 194         """Parse and save test results."""
 195         stat = self.get_verifier_result(self.verification_id)
 196         try:
 197             num_executed = stat['num_tests'] - stat['num_skipped']
 198             try:
 199                 self.result = 100 * stat['num_success'] / num_executed
 200             except ZeroDivisionError:
 201                 self.result = 0
 202                 if stat['num_tests'] > 0:
 203                     LOGGER.info("All tests have been skipped")
 204                 else:
 205                     LOGGER.error("No test has been executed")
 206                     return
 207
 208             with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 209                                    "tempest.log"), 'r') as logfile:
 210                 output = logfile.read()
 211
 212             success_testcases = []
 213             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 214                 success_testcases.append(match)
 215             failed_testcases = []
 216             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 217                 failed_testcases.append(match)
 218             skipped_testcases = []
 219             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 220                 skipped_testcases.append(match)
 221
 222             self.details = {"tests": stat['num_tests'],
 223                             "failures": stat['num_failures'],
 224                             "success": success_testcases,
 225                             "skipped": skipped_testcases,
 226                             "errors": failed_testcases}
 227         except Exception:  # pylint: disable=broad-except
 228             self.result = 0
 229
 230         LOGGER.info("Tempest %s success_rate is %s%%",
 231                     self.case_name, self.result)
 232
 233     def generate_report(self):
 234         """Generate verification report."""
 235         html_file = os.path.join(conf_utils.TEMPEST_RESULTS_DIR,
 236                                  "tempest-report.html")
 237         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 238                self.verification_id, "--to", html_file]
 239         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 240                          stderr=subprocess.STDOUT)
 241
 242     def run(self, **kwargs):
 243
 244         self.start_time = time.time()
 245         try:
 246             if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR):
 247                 os.makedirs(conf_utils.TEMPEST_RESULTS_DIR)
 248             resources = self.resources.create()
 249             compute_cnt = snaps_utils.get_active_compute_cnt(
 250                 self.resources.os_creds)
 251             conf_utils.configure_tempest(
 252                 self.deployment_dir,
 253                 network_name=resources.get("network_name"),
 254                 image_id=resources.get("image_id"),
 255                 flavor_id=resources.get("flavor_id"),
 256                 compute_cnt=compute_cnt)
 257             self.generate_test_list(self.verifier_repo_dir)
 258             self.apply_tempest_blacklist()
 259             self.run_verifier_tests()
 260             self.parse_verifier_result()
 261             self.generate_report()
 262             res = testcase.TestCase.EX_OK
 263         except Exception as err:  # pylint: disable=broad-except
 264             LOGGER.error('Error with run: %s', err)
 265             res = testcase.TestCase.EX_RUN_ERROR
 266         finally:
 267             self.resources.cleanup()
 268
 269         self.stop_time = time.time()
 270         return res
 271
 272
 273 class TempestSmokeSerial(TempestCommon):
 274     """Tempest smoke serial testcase implementation."""
 275     def __init__(self, **kwargs):
 276         if "case_name" not in kwargs:
 277             kwargs["case_name"] = 'tempest_smoke_serial'
 278         TempestCommon.__init__(self, **kwargs)
 279         self.mode = "smoke"
 280         self.option = ["--concurrency", "1"]
 281
 282
 283 class TempestSmokeParallel(TempestCommon):
 284     """Tempest smoke parallel testcase implementation."""
 285     def __init__(self, **kwargs):
 286         if "case_name" not in kwargs:
 287             kwargs["case_name"] = 'tempest_smoke_parallel'
 288         TempestCommon.__init__(self, **kwargs)
 289         self.mode = "smoke"
 290
 291
 292 class TempestFullParallel(TempestCommon):
 293     """Tempest full parallel testcase implementation."""
 294     def __init__(self, **kwargs):
 295         if "case_name" not in kwargs:
 296             kwargs["case_name"] = 'tempest_full_parallel'
 297         TempestCommon.__init__(self, **kwargs)
 298         self.mode = "full"
 299
 300
 301 class TempestCustom(TempestCommon):
 302     """Tempest custom testcase implementation."""
 303     def __init__(self, **kwargs):
 304         if "case_name" not in kwargs:
 305             kwargs["case_name"] = 'tempest_custom'
 306         TempestCommon.__init__(self, **kwargs)
 307         self.mode = "custom"
 308         self.option = ["--concurrency", "1"]
 309
 310
 311 class TempestDefcore(TempestCommon):
 312     """Tempest Defcore testcase implementation."""
 313     def __init__(self, **kwargs):
 314         if "case_name" not in kwargs:
 315             kwargs["case_name"] = 'tempest_defcore'
 316         TempestCommon.__init__(self, **kwargs)
 317         self.mode = "defcore"
 318         self.option = ["--concurrency", "1"]
 319
 320
 321 class TempestResourcesManager(object):
 322     """Tempest resource manager."""
 323     def __init__(self, **kwargs):
 324         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 325         self.guid = '-' + str(uuid.uuid4())
 326         self.creators = list()
 327         self.cirros_image_config = getattr(
 328             config.CONF, 'snaps_images_cirros', None)
 329
 330     def _create_project(self):
 331         """Create project for tests."""
 332         project_creator = deploy_utils.create_project(
 333             self.os_creds, ProjectConfig(
 334                 name=getattr(
 335                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 336                 description=getattr(
 337                     config.CONF, 'tempest_identity_tenant_description')))
 338         if project_creator is None or project_creator.get_project() is None:
 339             raise Exception("Failed to create tenant")
 340         self.creators.append(project_creator)
 341         return project_creator.get_project().id
 342
 343     def _create_user(self):
 344         """Create user for tests."""
 345         user_creator = deploy_utils.create_user(
 346             self.os_creds, UserConfig(
 347                 name=getattr(
 348                     config.CONF, 'tempest_identity_user_name') + self.guid,
 349                 password=getattr(
 350                     config.CONF, 'tempest_identity_user_password'),
 351                 project_name=getattr(
 352                     config.CONF, 'tempest_identity_tenant_name') + self.guid))
 353         if user_creator is None or user_creator.get_user() is None:
 354             raise Exception("Failed to create user")
 355         self.creators.append(user_creator)
 356         return user_creator.get_user().id
 357
 358     def _create_network(self, project_name):
 359         """Create network for tests."""
 360         tempest_network_type = None
 361         tempest_physical_network = None
 362         tempest_segmentation_id = None
 363
 364         tempest_network_type = getattr(
 365             config.CONF, 'tempest_network_type', None)
 366         tempest_physical_network = getattr(
 367             config.CONF, 'tempest_physical_network', None)
 368         tempest_segmentation_id = getattr(
 369             config.CONF, 'tempest_segmentation_id', None)
 370         tempest_net_name = getattr(
 371             config.CONF, 'tempest_private_net_name') + self.guid
 372
 373         network_creator = deploy_utils.create_network(
 374             self.os_creds, NetworkConfig(
 375                 name=tempest_net_name,
 376                 project_name=project_name,
 377                 network_type=tempest_network_type,
 378                 physical_network=tempest_physical_network,
 379                 segmentation_id=tempest_segmentation_id,
 380                 subnet_settings=[SubnetConfig(
 381                     name=getattr(
 382                         config.CONF,
 383                         'tempest_private_subnet_name') + self.guid,
 384                     project_name=project_name,
 385                     cidr=getattr(
 386                         config.CONF, 'tempest_private_subnet_cidr'))]))
 387         if network_creator is None or network_creator.get_network() is None:
 388             raise Exception("Failed to create private network")
 389         self.creators.append(network_creator)
 390         return tempest_net_name
 391
 392     def _create_image(self, name):
 393         """Create image for tests"""
 394         os_image_settings = openstack_tests.cirros_image_settings(
 395             name, public=True,
 396             image_metadata=self.cirros_image_config)
 397         image_creator = deploy_utils.create_image(
 398             self.os_creds, os_image_settings)
 399         if image_creator is None:
 400             raise Exception('Failed to create image')
 401         self.creators.append(image_creator)
 402         return image_creator.get_image().id
 403
 404     def _create_flavor(self, name):
 405         """Create flavor for tests."""
 406         scenario = env.get('DEPLOY_SCENARIO')
 407         flavor_metadata = None
 408         if 'ovs' in scenario or 'fdio' in scenario:
 409             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 410         flavor_creator = OpenStackFlavor(
 411             self.os_creds, FlavorConfig(
 412                 name=name,
 413                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 414                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 415                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 416                 metadata=flavor_metadata))
 417         flavor = flavor_creator.create()
 418         if flavor is None:
 419             raise Exception('Failed to create flavor')
 420         self.creators.append(flavor_creator)
 421         return flavor.id
 422
 423     def create(self, use_custom_images=False, use_custom_flavors=False,
 424                create_project=False):
 425         """Create resources for Tempest test suite."""
 426         result = {
 427             'tempest_net_name': None,
 428             'image_id': None,
 429             'image_id_alt': None,
 430             'flavor_id': None,
 431             'flavor_id_alt': None
 432         }
 433         project_name = None
 434
 435         if create_project:
 436             LOGGER.debug("Creating project and user for Tempest suite")
 437             project_name = getattr(
 438                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 439             result['project_id'] = self._create_project()
 440             result['user_id'] = self._create_user()
 441             result['tenant_id'] = result['project_id']  # for compatibility
 442
 443         LOGGER.debug("Creating private network for Tempest suite")
 444         result['tempest_net_name'] = self._create_network(project_name)
 445
 446         LOGGER.debug("Creating image for Tempest suite")
 447         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 448         result['image_id'] = self._create_image(image_name)
 449
 450         if use_custom_images:
 451             LOGGER.debug("Creating 2nd image for Tempest suite")
 452             image_name = getattr(
 453                 config.CONF, 'openstack_image_name_alt') + self.guid
 454             result['image_id_alt'] = self._create_image(image_name)
 455
 456         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 457                 use_custom_flavors):
 458             LOGGER.info("Creating flavor for Tempest suite")
 459             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 460             result['flavor_id'] = self._create_flavor(name)
 461
 462         if use_custom_flavors:
 463             LOGGER.info("Creating 2nd flavor for Tempest suite")
 464             scenario = env.get('DEPLOY_SCENARIO')
 465             if 'ovs' in scenario or 'fdio' in scenario:
 466                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 467             name = getattr(
 468                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 469             result['flavor_id_alt'] = self._create_flavor(name)
 470
 471         return result
 472
 473     def cleanup(self):
 474         """
 475         Cleanup all OpenStack objects. Should be called on completion.
 476         """
 477         for creator in reversed(self.creators):
 478             try:
 479                 creator.clean()
 480             except Exception as err:  # pylint: disable=broad-except
 481                 LOGGER.error('Unexpected error cleaning - %s', err)