functest/opnfv_tests/openstack/tempest/tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import logging
  16 import os
  17 import re
  18 import shutil
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 from snaps.config.flavor import FlavorConfig
  24 from snaps.config.network import NetworkConfig, SubnetConfig
  25 from snaps.config.project import ProjectConfig
  26 from snaps.config.user import UserConfig
  27 from snaps.openstack import create_flavor
  28 from snaps.openstack.create_flavor import OpenStackFlavor
  29 from snaps.openstack.tests import openstack_tests
  30 from snaps.openstack.utils import deploy_utils
  31 from xtesting.core import testcase
  32 import yaml
  33
  34 from functest.opnfv_tests.openstack.snaps import snaps_utils
  35 from functest.opnfv_tests.openstack.tempest import conf_utils
  36 from functest.utils import config
  37 from functest.utils import env
  38 from functest.utils import functest_utils
  39
  40 LOGGER = logging.getLogger(__name__)
  41
  42
  43 class TempestCommon(testcase.TestCase):
  44     # pylint: disable=too-many-instance-attributes
  45     """TempestCommon testcases implementation class."""
  46
  47     TEMPEST_RESULTS_DIR = os.path.join(
  48         getattr(config.CONF, 'dir_results'), 'tempest')
  49
  50     def __init__(self, **kwargs):
  51         super(TempestCommon, self).__init__(**kwargs)
  52         self.resources = TempestResourcesManager(**kwargs)
  53         self.mode = ""
  54         self.option = []
  55         self.verifier_id = conf_utils.get_verifier_id()
  56         self.verifier_repo_dir = conf_utils.get_verifier_repo_dir(
  57             self.verifier_id)
  58         self.deployment_id = conf_utils.get_verifier_deployment_id()
  59         self.deployment_dir = conf_utils.get_verifier_deployment_dir(
  60             self.verifier_id, self.deployment_id)
  61         self.verification_id = None
  62         self.res_dir = TempestCommon.TEMPEST_RESULTS_DIR
  63         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
  64         self.list = os.path.join(self.res_dir, 'test_list.txt')
  65         self.conf_file = None
  66
  67     @staticmethod
  68     def read_file(filename):
  69         """Read file and return content as a stripped list."""
  70         with open(filename) as src:
  71             return [line.strip() for line in src.readlines()]
  72
  73     @staticmethod
  74     def get_verifier_result(verif_id):
  75         """Retrieve verification results."""
  76         result = {
  77             'num_tests': 0,
  78             'num_success': 0,
  79             'num_failures': 0,
  80             'num_skipped': 0
  81         }
  82         cmd = ["rally", "verify", "show", "--uuid", verif_id]
  83         LOGGER.info("Showing result for a verification: '%s'.", cmd)
  84         proc = subprocess.Popen(cmd,
  85                                 stdout=subprocess.PIPE,
  86                                 stderr=subprocess.STDOUT)
  87         for line in proc.stdout:
  88             new_line = line.replace(' ', '').split('|')
  89             if 'Tests' in new_line:
  90                 break
  91             LOGGER.info(line)
  92             if 'Testscount' in new_line:
  93                 result['num_tests'] = int(new_line[2])
  94             elif 'Success' in new_line:
  95                 result['num_success'] = int(new_line[2])
  96             elif 'Skipped' in new_line:
  97                 result['num_skipped'] = int(new_line[2])
  98             elif 'Failures' in new_line:
  99                 result['num_failures'] = int(new_line[2])
 100         return result
 101
 102     def generate_test_list(self):
 103         """Generate test list based on the test mode."""
 104         LOGGER.debug("Generating test case list...")
 105         if self.mode == 'custom':
 106             if os.path.isfile(conf_utils.TEMPEST_CUSTOM):
 107                 shutil.copyfile(
 108                     conf_utils.TEMPEST_CUSTOM, self.list)
 109             else:
 110                 raise Exception("Tempest test list file %s NOT found."
 111                                 % conf_utils.TEMPEST_CUSTOM)
 112         else:
 113             if self.mode == 'smoke':
 114                 testr_mode = r"'tempest\.(api|scenario).*\[.*\bsmoke\b.*\]'"
 115             elif self.mode == 'full':
 116                 testr_mode = r"'^tempest\.'"
 117             else:
 118                 testr_mode = self.mode
 119             cmd = ("cd {0};"
 120                    "testr list-tests {1} > {2};"
 121                    "cd -;".format(self.verifier_repo_dir,
 122                                   testr_mode,
 123                                   self.list))
 124             functest_utils.execute_command(cmd)
 125
 126     def apply_tempest_blacklist(self):
 127         """Exclude blacklisted test cases."""
 128         LOGGER.debug("Applying tempest blacklist...")
 129         if os.path.exists(self.raw_list):
 130             os.remove(self.raw_list)
 131         os.rename(self.list, self.raw_list)
 132         cases_file = self.read_file(self.raw_list)
 133         result_file = open(self.list, 'w')
 134         black_tests = []
 135         try:
 136             installer_type = env.get('INSTALLER_TYPE')
 137             deploy_scenario = env.get('DEPLOY_SCENARIO')
 138             if bool(installer_type) * bool(deploy_scenario):
 139                 # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the
 140                 # file
 141                 black_list_file = open(conf_utils.TEMPEST_BLACKLIST)
 142                 black_list_yaml = yaml.safe_load(black_list_file)
 143                 black_list_file.close()
 144                 for item in black_list_yaml:
 145                     scenarios = item['scenarios']
 146                     installers = item['installers']
 147                     if (deploy_scenario in scenarios and
 148                             installer_type in installers):
 149                         tests = item['tests']
 150                         for test in tests:
 151                             black_tests.append(test)
 152                         break
 153         except Exception:  # pylint: disable=broad-except
 154             black_tests = []
 155             LOGGER.debug("Tempest blacklist file does not exist.")
 156
 157         for cases_line in cases_file:
 158             for black_tests_line in black_tests:
 159                 if black_tests_line in cases_line:
 160                     break
 161             else:
 162                 result_file.write(str(cases_line) + '\n')
 163         result_file.close()
 164
 165     def run_verifier_tests(self):
 166         """Execute tempest test cases."""
 167         cmd = ["rally", "verify", "start", "--load-list",
 168                self.list]
 169         cmd.extend(self.option)
 170         LOGGER.info("Starting Tempest test suite: '%s'.", cmd)
 171
 172         f_stdout = open(
 173             os.path.join(self.res_dir, "tempest.log"), 'w+')
 174         f_stderr = open(
 175             os.path.join(self.res_dir,
 176                          "tempest-error.log"), 'w+')
 177
 178         proc = subprocess.Popen(
 179             cmd,
 180             stdout=subprocess.PIPE,
 181             stderr=f_stderr,
 182             bufsize=1)
 183
 184         with proc.stdout:
 185             for line in iter(proc.stdout.readline, b''):
 186                 if re.search(r"\} tempest\.", line):
 187                     LOGGER.info(line.replace('\n', ''))
 188                 elif re.search('Starting verification', line):
 189                     LOGGER.info(line.replace('\n', ''))
 190                     first_pos = line.index("UUID=") + len("UUID=")
 191                     last_pos = line.index(") for deployment")
 192                     self.verification_id = line[first_pos:last_pos]
 193                     LOGGER.debug('Verification UUID: %s', self.verification_id)
 194                 f_stdout.write(line)
 195         proc.wait()
 196
 197         f_stdout.close()
 198         f_stderr.close()
 199
 200         if self.verification_id is None:
 201             raise Exception('Verification UUID not found')
 202
 203     def parse_verifier_result(self):
 204         """Parse and save test results."""
 205         stat = self.get_verifier_result(self.verification_id)
 206         try:
 207             num_executed = stat['num_tests'] - stat['num_skipped']
 208             try:
 209                 self.result = 100 * stat['num_success'] / num_executed
 210             except ZeroDivisionError:
 211                 self.result = 0
 212                 if stat['num_tests'] > 0:
 213                     LOGGER.info("All tests have been skipped")
 214                 else:
 215                     LOGGER.error("No test has been executed")
 216                     return
 217
 218             with open(os.path.join(self.res_dir,
 219                                    "tempest.log"), 'r') as logfile:
 220                 output = logfile.read()
 221
 222             success_testcases = []
 223             for match in re.findall(r'.*\{0\} (.*?)[. ]*success ', output):
 224                 success_testcases.append(match)
 225             failed_testcases = []
 226             for match in re.findall(r'.*\{0\} (.*?)[. ]*fail ', output):
 227                 failed_testcases.append(match)
 228             skipped_testcases = []
 229             for match in re.findall(r'.*\{0\} (.*?)[. ]*skip:', output):
 230                 skipped_testcases.append(match)
 231
 232             self.details = {"tests": stat['num_tests'],
 233                             "failures": stat['num_failures'],
 234                             "success": success_testcases,
 235                             "skipped": skipped_testcases,
 236                             "errors": failed_testcases}
 237         except Exception:  # pylint: disable=broad-except
 238             self.result = 0
 239
 240         LOGGER.info("Tempest %s success_rate is %s%%",
 241                     self.case_name, self.result)
 242
 243     def generate_report(self):
 244         """Generate verification report."""
 245         html_file = os.path.join(self.res_dir,
 246                                  "tempest-report.html")
 247         cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
 248                self.verification_id, "--to", html_file]
 249         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 250                          stderr=subprocess.STDOUT)
 251
 252     def configure(self):
 253         """
 254         Create all openstack resources for tempest-based testcases and write
 255         tempest.conf.
 256         """
 257         if not os.path.exists(self.res_dir):
 258             os.makedirs(self.res_dir)
 259         resources = self.resources.create()
 260         compute_cnt = snaps_utils.get_active_compute_cnt(
 261             self.resources.os_creds)
 262         self.conf_file = conf_utils.configure_verifier(self.deployment_dir)
 263         conf_utils.configure_tempest_update_params(
 264             self.conf_file, self.res_dir,
 265             network_name=resources.get("network_name"),
 266             image_id=resources.get("image_id"),
 267             flavor_id=resources.get("flavor_id"),
 268             compute_cnt=compute_cnt)
 269
 270     def run(self, **kwargs):
 271         self.start_time = time.time()
 272         try:
 273             self.configure()
 274             self.generate_test_list()
 275             self.apply_tempest_blacklist()
 276             self.run_verifier_tests()
 277             self.parse_verifier_result()
 278             self.generate_report()
 279             res = testcase.TestCase.EX_OK
 280         except Exception:  # pylint: disable=broad-except
 281             LOGGER.exception('Error with run')
 282             res = testcase.TestCase.EX_RUN_ERROR
 283         finally:
 284             self.resources.cleanup()
 285         self.stop_time = time.time()
 286         return res
 287
 288
 289 class TempestSmokeSerial(TempestCommon):
 290     """Tempest smoke serial testcase implementation."""
 291     def __init__(self, **kwargs):
 292         if "case_name" not in kwargs:
 293             kwargs["case_name"] = 'tempest_smoke_serial'
 294         TempestCommon.__init__(self, **kwargs)
 295         self.mode = "smoke"
 296         self.option = ["--concurrency", "1"]
 297
 298
 299 class TempestNeutronTrunk(TempestCommon):
 300     """Tempest neutron trunk testcase implementation."""
 301     def __init__(self, **kwargs):
 302         if "case_name" not in kwargs:
 303             kwargs["case_name"] = 'neutron_trunk'
 304         TempestCommon.__init__(self, **kwargs)
 305         self.mode = "'neutron.tests.tempest.(api|scenario).test_trunk'"
 306         self.res_dir = os.path.join(
 307             getattr(config.CONF, 'dir_results'), 'neutron_trunk')
 308         self.raw_list = os.path.join(self.res_dir, 'test_raw_list.txt')
 309         self.list = os.path.join(self.res_dir, 'test_list.txt')
 310
 311
 312 class TempestSmokeParallel(TempestCommon):
 313     """Tempest smoke parallel testcase implementation."""
 314     def __init__(self, **kwargs):
 315         if "case_name" not in kwargs:
 316             kwargs["case_name"] = 'tempest_smoke_parallel'
 317         TempestCommon.__init__(self, **kwargs)
 318         self.mode = "smoke"
 319
 320
 321 class TempestFullParallel(TempestCommon):
 322     """Tempest full parallel testcase implementation."""
 323     def __init__(self, **kwargs):
 324         if "case_name" not in kwargs:
 325             kwargs["case_name"] = 'tempest_full_parallel'
 326         TempestCommon.__init__(self, **kwargs)
 327         self.mode = "full"
 328
 329
 330 class TempestCustom(TempestCommon):
 331     """Tempest custom testcase implementation."""
 332     def __init__(self, **kwargs):
 333         if "case_name" not in kwargs:
 334             kwargs["case_name"] = 'tempest_custom'
 335         TempestCommon.__init__(self, **kwargs)
 336         self.mode = "custom"
 337         self.option = ["--concurrency", "1"]
 338
 339
 340 class TempestDefcore(TempestCommon):
 341     """Tempest Defcore testcase implementation."""
 342     def __init__(self, **kwargs):
 343         if "case_name" not in kwargs:
 344             kwargs["case_name"] = 'tempest_defcore'
 345         TempestCommon.__init__(self, **kwargs)
 346         self.mode = "defcore"
 347         self.option = ["--concurrency", "1"]
 348
 349
 350 class TempestResourcesManager(object):
 351     """Tempest resource manager."""
 352     def __init__(self, **kwargs):
 353         self.os_creds = kwargs.get('os_creds') or snaps_utils.get_credentials()
 354         self.guid = '-' + str(uuid.uuid4())
 355         self.creators = list()
 356         self.cirros_image_config = getattr(
 357             config.CONF, 'snaps_images_cirros', None)
 358
 359     def _create_project(self):
 360         """Create project for tests."""
 361         project_creator = deploy_utils.create_project(
 362             self.os_creds, ProjectConfig(
 363                 name=getattr(
 364                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 365                 description=getattr(
 366                     config.CONF, 'tempest_identity_tenant_description'),
 367                 domain=self.os_creds.project_domain_name))
 368         if project_creator is None or project_creator.get_project() is None:
 369             raise Exception("Failed to create tenant")
 370         self.creators.append(project_creator)
 371         return project_creator.get_project().id
 372
 373     def _create_user(self):
 374         """Create user for tests."""
 375         user_creator = deploy_utils.create_user(
 376             self.os_creds, UserConfig(
 377                 name=getattr(
 378                     config.CONF, 'tempest_identity_user_name') + self.guid,
 379                 password=getattr(
 380                     config.CONF, 'tempest_identity_user_password'),
 381                 project_name=getattr(
 382                     config.CONF, 'tempest_identity_tenant_name') + self.guid,
 383                 domain_name=self.os_creds.user_domain_name))
 384         if user_creator is None or user_creator.get_user() is None:
 385             raise Exception("Failed to create user")
 386         self.creators.append(user_creator)
 387         return user_creator.get_user().id
 388
 389     def _create_network(self, project_name):
 390         """Create network for tests."""
 391         tempest_network_type = None
 392         tempest_physical_network = None
 393         tempest_segmentation_id = None
 394
 395         tempest_network_type = getattr(
 396             config.CONF, 'tempest_network_type', None)
 397         tempest_physical_network = getattr(
 398             config.CONF, 'tempest_physical_network', None)
 399         tempest_segmentation_id = getattr(
 400             config.CONF, 'tempest_segmentation_id', None)
 401         tempest_net_name = getattr(
 402             config.CONF, 'tempest_private_net_name') + self.guid
 403
 404         network_creator = deploy_utils.create_network(
 405             self.os_creds, NetworkConfig(
 406                 name=tempest_net_name,
 407                 project_name=project_name,
 408                 network_type=tempest_network_type,
 409                 physical_network=tempest_physical_network,
 410                 segmentation_id=tempest_segmentation_id,
 411                 subnet_settings=[SubnetConfig(
 412                     name=getattr(
 413                         config.CONF,
 414                         'tempest_private_subnet_name') + self.guid,
 415                     project_name=project_name,
 416                     cidr=getattr(
 417                         config.CONF, 'tempest_private_subnet_cidr'),
 418                     dns_nameservers=[env.get('NAMESERVER')])]))
 419         if network_creator is None or network_creator.get_network() is None:
 420             raise Exception("Failed to create private network")
 421         self.creators.append(network_creator)
 422         return tempest_net_name
 423
 424     def _create_image(self, name):
 425         """Create image for tests"""
 426         os_image_settings = openstack_tests.cirros_image_settings(
 427             name, public=True,
 428             image_metadata=self.cirros_image_config)
 429         image_creator = deploy_utils.create_image(
 430             self.os_creds, os_image_settings)
 431         if image_creator is None:
 432             raise Exception('Failed to create image')
 433         self.creators.append(image_creator)
 434         return image_creator.get_image().id
 435
 436     def _create_flavor(self, name):
 437         """Create flavor for tests."""
 438         scenario = env.get('DEPLOY_SCENARIO')
 439         flavor_metadata = None
 440         if 'ovs' in scenario or 'fdio' in scenario:
 441             flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE
 442         flavor_creator = OpenStackFlavor(
 443             self.os_creds, FlavorConfig(
 444                 name=name,
 445                 ram=getattr(config.CONF, 'openstack_flavor_ram'),
 446                 disk=getattr(config.CONF, 'openstack_flavor_disk'),
 447                 vcpus=getattr(config.CONF, 'openstack_flavor_vcpus'),
 448                 metadata=flavor_metadata))
 449         flavor = flavor_creator.create()
 450         if flavor is None:
 451             raise Exception('Failed to create flavor')
 452         self.creators.append(flavor_creator)
 453         return flavor.id
 454
 455     def create(self, use_custom_images=False, use_custom_flavors=False,
 456                create_project=False):
 457         """Create resources for Tempest test suite."""
 458         result = {
 459             'tempest_net_name': None,
 460             'image_id': None,
 461             'image_id_alt': None,
 462             'flavor_id': None,
 463             'flavor_id_alt': None
 464         }
 465         project_name = None
 466
 467         if create_project:
 468             LOGGER.debug("Creating project and user for Tempest suite")
 469             project_name = getattr(
 470                 config.CONF, 'tempest_identity_tenant_name') + self.guid
 471             result['project_id'] = self._create_project()
 472             result['user_id'] = self._create_user()
 473             result['tenant_id'] = result['project_id']  # for compatibility
 474
 475         LOGGER.debug("Creating private network for Tempest suite")
 476         result['tempest_net_name'] = self._create_network(project_name)
 477
 478         LOGGER.debug("Creating image for Tempest suite")
 479         image_name = getattr(config.CONF, 'openstack_image_name') + self.guid
 480         result['image_id'] = self._create_image(image_name)
 481
 482         if use_custom_images:
 483             LOGGER.debug("Creating 2nd image for Tempest suite")
 484             image_name = getattr(
 485                 config.CONF, 'openstack_image_name_alt') + self.guid
 486             result['image_id_alt'] = self._create_image(image_name)
 487
 488         if (getattr(config.CONF, 'tempest_use_custom_flavors') == 'True' or
 489                 use_custom_flavors):
 490             LOGGER.info("Creating flavor for Tempest suite")
 491             name = getattr(config.CONF, 'openstack_flavor_name') + self.guid
 492             result['flavor_id'] = self._create_flavor(name)
 493
 494         if use_custom_flavors:
 495             LOGGER.info("Creating 2nd flavor for Tempest suite")
 496             scenario = env.get('DEPLOY_SCENARIO')
 497             if 'ovs' in scenario or 'fdio' in scenario:
 498                 setattr(config.CONF, 'openstack_flavor_ram', 1024)
 499             name = getattr(
 500                 config.CONF, 'openstack_flavor_name_alt') + self.guid
 501             result['flavor_id_alt'] = self._create_flavor(name)
 502
 503         return result
 504
 505     def cleanup(self):
 506         """
 507         Cleanup all OpenStack objects. Should be called on completion.
 508         """
 509         for creator in reversed(self.creators):
 510             try:
 511                 creator.clean()
 512             except Exception as err:  # pylint: disable=broad-except
 513                 LOGGER.error('Unexpected error cleaning - %s', err)