functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican']
  45
  46     rally_conf_path = "/etc/rally/rally.conf"
  47     rally_aar4_patch_path = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     rally_dir = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     rally_scenario_dir = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     template_dir = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     support_dir = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     users_amount = 2
  58     tenants_amount = 3
  59     iterations_amount = 10
  60     concurrency = 4
  61     volume_version = 3
  62     volume_service_type = "volumev3"
  63     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  64     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     temp_dir = os.path.join(task_dir, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69     allow_no_fip = True
  70
  71     def __init__(self, **kwargs):
  72         """Initialize RallyBase object."""
  73         super(RallyBase, self).__init__(**kwargs)
  74         assert self.orig_cloud
  75         assert self.project
  76         if self.orig_cloud.get_role("admin"):
  77             role_name = "admin"
  78         elif self.orig_cloud.get_role("Admin"):
  79             role_name = "Admin"
  80         else:
  81             raise Exception("Cannot detect neither admin nor Admin")
  82         self.orig_cloud.grant_role(
  83             role_name, user=self.project.user.id,
  84             project=self.project.project.id,
  85             domain=self.project.domain.id)
  86         self.results_dir = os.path.join(
  87             getattr(config.CONF, 'dir_results'), self.case_name)
  88         self.task_file = ''
  89         self.creators = []
  90         self.summary = []
  91         self.scenario_dir = ''
  92         self.smoke = None
  93         self.start_time = None
  94         self.result = None
  95         self.details = None
  96         self.compute_cnt = 0
  97         self.flavor_alt = None
  98         self.tests = []
  99         self.run_cmd = ''
 100         self.network_extensions = []
 101         self.services = []
 102
 103     def build_task_args(self, test_name):
 104         """Build arguments for the Rally task."""
 105         task_args = {'service_list': [test_name]}
 106         task_args['image_name'] = str(self.image.name)
 107         task_args['flavor_name'] = str(self.flavor.name)
 108         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 109         task_args['glance_image_location'] = str(self.filename)
 110         task_args['glance_image_format'] = str(self.image_format)
 111         task_args['tmpl_dir'] = str(self.template_dir)
 112         task_args['sup_dir'] = str(self.support_dir)
 113         task_args['users_amount'] = self.users_amount
 114         task_args['tenants_amount'] = self.tenants_amount
 115         task_args['use_existing_users'] = False
 116         task_args['iterations'] = self.iterations_amount
 117         task_args['concurrency'] = self.concurrency
 118         task_args['smoke'] = self.smoke
 119         task_args['volume_version'] = self.volume_version
 120         task_args['volume_service_type'] = self.volume_service_type
 121         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 122
 123         if self.ext_net:
 124             task_args['floating_network'] = str(self.ext_net.name)
 125         else:
 126             task_args['floating_network'] = ''
 127
 128         if self.network:
 129             task_args['netid'] = str(self.network.id)
 130         else:
 131             task_args['netid'] = ''
 132
 133         return task_args
 134
 135     def _prepare_test_list(self, test_name):
 136         """Build the list of test cases to be executed."""
 137         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 138         scenario_file_name = os.path.join(self.rally_scenario_dir,
 139                                           test_yaml_file_name)
 140
 141         if not os.path.exists(scenario_file_name):
 142             scenario_file_name = os.path.join(self.scenario_dir,
 143                                               test_yaml_file_name)
 144
 145             if not os.path.exists(scenario_file_name):
 146                 raise Exception("The scenario '%s' does not exist."
 147                                 % scenario_file_name)
 148
 149         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 150         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 151
 152         if not os.path.exists(self.temp_dir):
 153             os.makedirs(self.temp_dir)
 154
 155         self.apply_blacklist(scenario_file_name, test_file_name)
 156         return test_file_name
 157
 158     @staticmethod
 159     def get_verifier_deployment_id():
 160         """
 161         Returns deployment id for active Rally deployment
 162         """
 163         cmd = ("rally deployment list | awk '/" +
 164                getattr(config.CONF, 'rally_deployment_name') +
 165                "/ {print $2}'")
 166         proc = subprocess.Popen(cmd, shell=True,
 167                                 stdout=subprocess.PIPE,
 168                                 stderr=subprocess.STDOUT)
 169         deployment_uuid = proc.stdout.readline().rstrip()
 170         return deployment_uuid.decode("utf-8")
 171
 172     @staticmethod
 173     def create_rally_deployment(environ=None):
 174         """Create new rally deployment"""
 175         # set the architecture to default
 176         pod_arch = env.get("POD_ARCH")
 177         arch_filter = ['aarch64']
 178
 179         if pod_arch and pod_arch in arch_filter:
 180             LOGGER.info("Apply aarch64 specific to rally config...")
 181             with open(RallyBase.rally_aar4_patch_path, "r") as pfile:
 182                 rally_patch_conf = pfile.read()
 183
 184             for line in fileinput.input(RallyBase.rally_conf_path):
 185                 print(line, end=' ')
 186                 if "cirros|testvm" in line:
 187                     print(rally_patch_conf)
 188
 189         LOGGER.info("Creating Rally environment...")
 190         try:
 191             cmd = ['rally', 'deployment', 'destroy',
 192                    '--deployment',
 193                    str(getattr(config.CONF, 'rally_deployment_name'))]
 194             output = subprocess.check_output(cmd)
 195             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 196         except subprocess.CalledProcessError:
 197             pass
 198
 199         cmd = ['rally', 'deployment', 'create', '--fromenv',
 200                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 201         output = subprocess.check_output(cmd, env=environ)
 202         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 203
 204         cmd = ['rally', 'deployment', 'check']
 205         output = subprocess.check_output(cmd)
 206         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 207         return RallyBase.get_verifier_deployment_id()
 208
 209     @staticmethod
 210     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 211         """Set keystone_default_role in rally.conf"""
 212         if env.get("NEW_USER_ROLE").lower() != "member":
 213             rconfig = configparser.RawConfigParser()
 214             rconfig.read(rally_conf)
 215             if not rconfig.has_section('openstack'):
 216                 rconfig.add_section('openstack')
 217             rconfig.set(
 218                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 219             with open(rally_conf, 'w') as config_file:
 220                 rconfig.write(config_file)
 221
 222     @staticmethod
 223     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 224         """Clean Rally config"""
 225         if env.get("NEW_USER_ROLE").lower() != "member":
 226             rconfig = configparser.RawConfigParser()
 227             rconfig.read(rally_conf)
 228             if rconfig.has_option('openstack', 'keystone_default_role'):
 229                 rconfig.remove_option('openstack', 'keystone_default_role')
 230             with open(rally_conf, 'w') as config_file:
 231                 rconfig.write(config_file)
 232
 233     @staticmethod
 234     def get_task_id(cmd_raw):
 235         """
 236         Get task id from command rally result.
 237
 238         :param cmd_raw:
 239         :return: task_id as string
 240         """
 241         taskid_re = re.compile('^Task +(.*): started$')
 242         for line in cmd_raw.splitlines(True):
 243             line = line.strip()
 244             match = taskid_re.match(line.decode("utf-8"))
 245             if match:
 246                 return match.group(1)
 247         return None
 248
 249     @staticmethod
 250     def task_succeed(json_raw):
 251         """
 252         Parse JSON from rally JSON results.
 253
 254         :param json_raw:
 255         :return: Bool
 256         """
 257         rally_report = json.loads(json_raw)
 258         tasks = rally_report.get('tasks')
 259         if tasks:
 260             for task in tasks:
 261                 if task.get('status') != 'finished' or \
 262                    task.get('pass_sla') is not True:
 263                     return False
 264         else:
 265             return False
 266         return True
 267
 268     def _migration_supported(self):
 269         """Determine if migration is supported."""
 270         if self.compute_cnt > 1:
 271             return True
 272         return False
 273
 274     def _network_trunk_supported(self):
 275         """Determine if network trunk service is available"""
 276         if 'trunk' in self.network_extensions:
 277             return True
 278         return False
 279
 280     @staticmethod
 281     def excl_scenario():
 282         """Exclude scenario."""
 283         black_tests = []
 284         try:
 285             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 286                 black_list_yaml = yaml.safe_load(black_list_file)
 287
 288             deploy_scenario = env.get('DEPLOY_SCENARIO')
 289             if (bool(deploy_scenario) and
 290                     'scenario' in black_list_yaml.keys()):
 291                 for item in black_list_yaml['scenario']:
 292                     scenarios = item['scenarios']
 293                     in_it = RallyBase.in_iterable_re
 294                     if in_it(deploy_scenario, scenarios):
 295                         tests = item['tests']
 296                         black_tests.extend(tests)
 297         except Exception:  # pylint: disable=broad-except
 298             LOGGER.debug("Scenario exclusion not applied.")
 299
 300         return black_tests
 301
 302     @staticmethod
 303     def in_iterable_re(needle, haystack):
 304         """
 305         Check if given needle is in the iterable haystack, using regex.
 306
 307         :param needle: string to be matched
 308         :param haystack: iterable of strings (optionally regex patterns)
 309         :return: True if needle is eqial to any of the elements in haystack,
 310                  or if a nonempty regex pattern in haystack is found in needle.
 311         """
 312         # match without regex
 313         if needle in haystack:
 314             return True
 315
 316         for pattern in haystack:
 317             # match if regex pattern is set and found in the needle
 318             if pattern and re.search(pattern, needle) is not None:
 319                 return True
 320
 321         return False
 322
 323     def excl_func(self):
 324         """Exclude functionalities."""
 325         black_tests = []
 326         func_list = []
 327
 328         try:
 329             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 330                 black_list_yaml = yaml.safe_load(black_list_file)
 331
 332             if env.get('BLOCK_MIGRATION').lower() == 'true':
 333                 func_list.append("block_migration")
 334             if not self._migration_supported():
 335                 func_list.append("no_migration")
 336             if not self._network_trunk_supported():
 337                 func_list.append("no_net_trunk_service")
 338             if not self.ext_net:
 339                 func_list.append("no_floating_ip")
 340
 341             if 'functionality' in black_list_yaml.keys():
 342                 for item in black_list_yaml['functionality']:
 343                     functions = item['functions']
 344                     for func in func_list:
 345                         if func in functions:
 346                             tests = item['tests']
 347                             black_tests.extend(tests)
 348         except Exception:  # pylint: disable=broad-except
 349             LOGGER.debug("Functionality exclusion not applied.")
 350
 351         return black_tests
 352
 353     def apply_blacklist(self, case_file_name, result_file_name):
 354         """Apply blacklist."""
 355         LOGGER.debug("Applying blacklist...")
 356         cases_file = open(case_file_name, 'r')
 357         result_file = open(result_file_name, 'w')
 358
 359         black_tests = list(set(self.excl_func() +
 360                                self.excl_scenario()))
 361
 362         if black_tests:
 363             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 364
 365         include = True
 366         for cases_line in cases_file:
 367             if include:
 368                 for black_tests_line in black_tests:
 369                     if re.search(black_tests_line,
 370                                  cases_line.strip().rstrip(':')):
 371                         include = False
 372                         break
 373                 else:
 374                     result_file.write(str(cases_line))
 375             else:
 376                 if cases_line.isspace():
 377                     include = True
 378
 379         cases_file.close()
 380         result_file.close()
 381
 382     @staticmethod
 383     def file_is_empty(file_name):
 384         """Determine is a file is empty."""
 385         try:
 386             if os.stat(file_name).st_size > 0:
 387                 return False
 388         except Exception:  # pylint: disable=broad-except
 389             pass
 390
 391         return True
 392
 393     def _save_results(self, test_name, task_id):
 394         """ Generate and save task execution results"""
 395         # check for result directory and create it otherwise
 396         if not os.path.exists(self.results_dir):
 397             LOGGER.debug('%s does not exist, we create it.',
 398                          self.results_dir)
 399             os.makedirs(self.results_dir)
 400
 401         # put detailed result to log
 402         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 403         LOGGER.debug('running command: %s', cmd)
 404         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 405         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 406
 407         # save report as JSON
 408         report_json_name = '{}.json'.format(test_name)
 409         report_json_dir = os.path.join(self.results_dir, report_json_name)
 410         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 411                 "--out", report_json_dir])
 412         LOGGER.debug('running command: %s', cmd)
 413         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 414         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 415
 416         json_results = open(report_json_dir).read()
 417         self._append_summary(json_results, test_name)
 418
 419         # parse JSON operation result
 420         if self.task_succeed(json_results):
 421             LOGGER.info('Test scenario: "%s" OK.', test_name)
 422         else:
 423             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 424
 425     def run_task(self, test_name):
 426         """Run a task."""
 427         LOGGER.info('Starting test scenario "%s" ...', test_name)
 428         LOGGER.debug('running command: %s', self.run_cmd)
 429         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 430                                 stderr=subprocess.STDOUT)
 431         output = proc.communicate()[0]
 432
 433         task_id = self.get_task_id(output)
 434         LOGGER.debug('task_id : %s', task_id)
 435         if task_id is None:
 436             LOGGER.error("Failed to retrieve task_id")
 437             LOGGER.error("Result:\n%s", output.decode("utf-8"))
 438             raise Exception("Failed to retrieve task id")
 439         self._save_results(test_name, task_id)
 440
 441     def _append_summary(self, json_raw, test_name):
 442         # pylint: disable=too-many-locals
 443         """Update statistics summary info."""
 444         nb_tests = 0
 445         nb_success = 0
 446         overall_duration = 0.0
 447         success = []
 448         failures = []
 449
 450         rally_report = json.loads(json_raw)
 451         for task in rally_report.get('tasks'):
 452             for subtask in task.get('subtasks'):
 453                 has_errors = False
 454                 for workload in subtask.get('workloads'):
 455                     if workload.get('full_duration'):
 456                         overall_duration += workload.get('full_duration')
 457
 458                     if workload.get('data'):
 459                         nb_tests += len(workload.get('data'))
 460
 461                     for result in workload.get('data'):
 462                         if not result.get('error'):
 463                             nb_success += 1
 464                         else:
 465                             has_errors = True
 466
 467                 if has_errors:
 468                     failures.append(subtask['title'])
 469                 else:
 470                     success.append(subtask['title'])
 471
 472         scenario_summary = {'test_name': test_name,
 473                             'overall_duration': overall_duration,
 474                             'nb_tests': nb_tests,
 475                             'nb_success': nb_success,
 476                             'success': success,
 477                             'failures': failures,
 478                             'task_status': self.task_succeed(json_raw)}
 479         self.summary.append(scenario_summary)
 480
 481     def prepare_run(self, **kwargs):
 482         """Prepare resources needed by test scenarios."""
 483         assert self.cloud
 484         LOGGER.debug('Validating run tests...')
 485         for test in kwargs.get('tests', self.stests):
 486             if test in self.stests:
 487                 self.tests.append(test)
 488             else:
 489                 raise Exception("Test name '%s' is invalid" % test)
 490
 491         if not os.path.exists(self.task_dir):
 492             os.makedirs(self.task_dir)
 493
 494         task = os.path.join(self.rally_dir, 'task.yaml')
 495         if not os.path.exists(task):
 496             LOGGER.error("Task file '%s' does not exist.", task)
 497             raise Exception("Task file '{}' does not exist.".
 498                             format(task))
 499         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 500         shutil.copyfile(task, self.task_file)
 501
 502         task_macro = os.path.join(self.rally_dir, 'macro')
 503         if not os.path.exists(task_macro):
 504             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 505             raise Exception("Task macro dir '{}' does not exist.".
 506                             format(task_macro))
 507         macro_dir = os.path.join(self.task_dir, 'macro')
 508         if os.path.exists(macro_dir):
 509             shutil.rmtree(macro_dir)
 510         shutil.copytree(task_macro, macro_dir)
 511
 512         self.update_keystone_default_role()
 513         self.compute_cnt = len(self.cloud.list_hypervisors())
 514         self.network_extensions = self.cloud.get_network_extensions()
 515         self.flavor_alt = self.create_flavor_alt()
 516         self.services = [service.name for service in
 517                          self.cloud.list_services()]
 518
 519         LOGGER.debug("flavor: %s", self.flavor_alt)
 520
 521     def prepare_task(self, test_name):
 522         """Prepare resources for test run."""
 523         file_name = self._prepare_test_list(test_name)
 524         if self.file_is_empty(file_name):
 525             LOGGER.info('No tests for scenario "%s"', test_name)
 526             return False
 527         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 528                          "--task", self.task_file, "--task-args",
 529                          str(self.build_task_args(test_name))])
 530         return True
 531
 532     def run_tests(self, **kwargs):
 533         """Execute tests."""
 534         optional = kwargs.get('optional', [])
 535         for test in self.tests:
 536             if test in self.services or test not in optional:
 537                 if self.prepare_task(test):
 538                     self.run_task(test)
 539
 540     def _generate_report(self):
 541         """Generate test execution summary report."""
 542         total_duration = 0.0
 543         total_nb_tests = 0
 544         total_nb_success = 0
 545         nb_modules = 0
 546         payload = []
 547
 548         res_table = prettytable.PrettyTable(
 549             padding_width=2,
 550             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 551         res_table.align['Module'] = "l"
 552         res_table.align['Duration'] = "r"
 553         res_table.align['Success'] = "r"
 554
 555         # for each scenario we draw a row for the table
 556         for item in self.summary:
 557             if item['task_status'] is True:
 558                 nb_modules += 1
 559             total_duration += item['overall_duration']
 560             total_nb_tests += item['nb_tests']
 561             total_nb_success += item['nb_success']
 562             try:
 563                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 564             except ZeroDivisionError:
 565                 success_avg = 0
 566             success_str = str("{:0.2f}".format(success_avg)) + '%'
 567             duration_str = time.strftime("%H:%M:%S",
 568                                          time.gmtime(item['overall_duration']))
 569             res_table.add_row([item['test_name'], duration_str,
 570                                item['nb_tests'], success_str])
 571             payload.append({'module': item['test_name'],
 572                             'details': {'duration': item['overall_duration'],
 573                                         'nb tests': item['nb_tests'],
 574                                         'success rate': success_str,
 575                                         'success': item['success'],
 576                                         'failures': item['failures']}})
 577
 578         total_duration_str = time.strftime("%H:%M:%S",
 579                                            time.gmtime(total_duration))
 580         try:
 581             self.result = 100 * total_nb_success / total_nb_tests
 582         except ZeroDivisionError:
 583             self.result = 100
 584         success_rate = "{:0.2f}".format(self.result)
 585         success_rate_str = str(success_rate) + '%'
 586         res_table.add_row(["", "", "", ""])
 587         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 588                            success_rate_str])
 589
 590         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 591         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 592                     self.case_name, success_rate, nb_modules,
 593                     len(self.summary))
 594         payload.append({'summary': {'duration': total_duration,
 595                                     'nb tests': total_nb_tests,
 596                                     'nb success': success_rate}})
 597         self.details = payload
 598
 599     @staticmethod
 600     def export_task(file_name, export_type="html"):
 601         """Export all task results (e.g. html or xunit report)
 602
 603         Raises:
 604             subprocess.CalledProcessError: if Rally doesn't return 0
 605
 606         Returns:
 607             None
 608         """
 609         cmd = ["rally", "task", "export", "--type", export_type,
 610                "--deployment",
 611                str(getattr(config.CONF, 'rally_deployment_name')),
 612                "--to", file_name]
 613         LOGGER.debug('running command: %s', cmd)
 614         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 615         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 616
 617     @staticmethod
 618     def verify_report(file_name, uuid, export_type="html"):
 619         """Generate the verifier report (e.g. html or xunit report)
 620
 621         Raises:
 622             subprocess.CalledProcessError: if Rally doesn't return 0
 623
 624         Returns:
 625             None
 626         """
 627         cmd = ["rally", "verify", "report", "--type", export_type,
 628                "--uuid", uuid, "--to", file_name]
 629         LOGGER.debug('running command: %s', cmd)
 630         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 631         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 632
 633     def clean(self):
 634         """Cleanup of OpenStack resources. Should be called on completion."""
 635         self.clean_rally_conf()
 636         self.clean_rally_logs()
 637         if self.flavor_alt:
 638             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 639         super(RallyBase, self).clean()
 640
 641     def is_successful(self):
 642         """The overall result of the test."""
 643         for item in self.summary:
 644             if item['task_status'] is False:
 645                 return testcase.TestCase.EX_TESTCASE_FAILED
 646
 647         return super(RallyBase, self).is_successful()
 648
 649     @staticmethod
 650     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 651         """Print rally logs in res dir"""
 652         if not os.path.exists(res_dir):
 653             os.makedirs(res_dir)
 654         rconfig = configparser.RawConfigParser()
 655         rconfig.read(rally_conf)
 656         rconfig.set('DEFAULT', 'debug', True)
 657         rconfig.set('DEFAULT', 'use_stderr', False)
 658         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 659         rconfig.set('DEFAULT', 'log_dir', res_dir)
 660         with open(rally_conf, 'w') as config_file:
 661             rconfig.write(config_file)
 662
 663     @staticmethod
 664     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 665         """Clean Rally config"""
 666         rconfig = configparser.RawConfigParser()
 667         rconfig.read(rally_conf)
 668         if rconfig.has_option('DEFAULT', 'use_stderr'):
 669             rconfig.remove_option('DEFAULT', 'use_stderr')
 670         if rconfig.has_option('DEFAULT', 'debug'):
 671             rconfig.remove_option('DEFAULT', 'debug')
 672         if rconfig.has_option('DEFAULT', 'log-file'):
 673             rconfig.remove_option('DEFAULT', 'log-file')
 674         if rconfig.has_option('DEFAULT', 'log_dir'):
 675             rconfig.remove_option('DEFAULT', 'log_dir')
 676         with open(rally_conf, 'w') as config_file:
 677             rconfig.write(config_file)
 678
 679     def run(self, **kwargs):
 680         """Run testcase."""
 681         self.start_time = time.time()
 682         try:
 683             assert super(RallyBase, self).run(
 684                 **kwargs) == testcase.TestCase.EX_OK
 685             self.update_rally_logs(self.res_dir)
 686             self.create_rally_deployment(environ=self.project.get_environ())
 687             self.prepare_run(**kwargs)
 688             self.run_tests(**kwargs)
 689             self._generate_report()
 690             self.export_task(
 691                 "{}/{}.html".format(self.results_dir, self.case_name))
 692             self.export_task(
 693                 "{}/{}.xml".format(self.results_dir, self.case_name),
 694                 export_type="junit-xml")
 695             res = testcase.TestCase.EX_OK
 696         except Exception:   # pylint: disable=broad-except
 697             LOGGER.exception('Error with run:')
 698             self.result = 0
 699             res = testcase.TestCase.EX_RUN_ERROR
 700         self.stop_time = time.time()
 701         return res
 702
 703
 704 class RallySanity(RallyBase):
 705     """Rally sanity testcase implementation."""
 706
 707     def __init__(self, **kwargs):
 708         """Initialize RallySanity object."""
 709         if "case_name" not in kwargs:
 710             kwargs["case_name"] = "rally_sanity"
 711         super(RallySanity, self).__init__(**kwargs)
 712         self.smoke = True
 713         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 714
 715
 716 class RallyFull(RallyBase):
 717     """Rally full testcase implementation."""
 718
 719     def __init__(self, **kwargs):
 720         """Initialize RallyFull object."""
 721         if "case_name" not in kwargs:
 722             kwargs["case_name"] = "rally_full"
 723         super(RallyFull, self).__init__(**kwargs)
 724         self.smoke = False
 725         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 726
 727
 728 class RallyJobs(RallyBase):
 729     """Rally OpenStack CI testcase implementation."""
 730
 731     stests = ["neutron"]
 732
 733     def __init__(self, **kwargs):
 734         """Initialize RallyJobs object."""
 735         if "case_name" not in kwargs:
 736             kwargs["case_name"] = "rally_jobs"
 737         super(RallyJobs, self).__init__(**kwargs)
 738         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 739         self.task_yaml = None
 740
 741     def prepare_run(self, **kwargs):
 742         """Create resources needed by test scenarios."""
 743         super(RallyJobs, self).prepare_run(**kwargs)
 744         with open(os.path.join(self.rally_dir,
 745                                'rally_jobs.yaml'), 'r') as task_file:
 746             self.task_yaml = yaml.safe_load(task_file)
 747
 748         for task in self.task_yaml:
 749             if task not in self.tests:
 750                 raise Exception("Test '%s' not in '%s'" %
 751                                 (task, self.tests))
 752
 753     def apply_blacklist(self, case_file_name, result_file_name):
 754         # pylint: disable=too-many-branches
 755         """Apply blacklist."""
 756         LOGGER.debug("Applying blacklist...")
 757         black_tests = list(set(self.excl_func() +
 758                                self.excl_scenario()))
 759         if black_tests:
 760             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 761
 762         template = YAML(typ='jinja2')
 763         with open(case_file_name, 'r') as fname:
 764             cases = template.load(fname)
 765         if cases.get("version", 1) == 1:
 766             # scenarios in dictionary
 767             for name in cases.keys():
 768                 if self.in_iterable_re(name, black_tests):
 769                     cases.pop(name)
 770         else:
 771             # workloads in subtasks
 772             for sind, subtask in reversed(list(
 773                     enumerate(cases.get('subtasks', [])))):
 774                 for wind, workload in reversed(list(
 775                         enumerate(subtask.get('workloads', [])))):
 776                     scenario = workload.get('scenario', {})
 777                     for name in scenario.keys():
 778                         if self.in_iterable_re(name, black_tests):
 779                             cases['subtasks'][sind]['workloads'].pop(wind)
 780                             break
 781                 if 'workloads' in cases['subtasks'][sind]:
 782                     if not cases['subtasks'][sind]['workloads']:
 783                         cases['subtasks'].pop(sind)
 784             # scenarios in subtasks
 785             for sind, subtask in reversed(list(
 786                     enumerate(cases.get('subtasks', [])))):
 787                 scenario = subtask.get('scenario', {})
 788                 for name in scenario.keys():
 789                     if self.in_iterable_re(name, black_tests):
 790                         cases['subtasks'].pop(sind)
 791                         break
 792
 793         with open(result_file_name, 'w') as fname:
 794             template.dump(cases, fname)
 795
 796     def build_task_args(self, test_name):
 797         """Build arguments for the Rally task."""
 798         task_args = {}
 799         if self.ext_net:
 800             task_args['floating_network'] = str(self.ext_net.name)
 801         else:
 802             task_args['floating_network'] = ''
 803         task_args['image_name'] = str(self.image.name)
 804         task_args['flavor_name'] = str(self.flavor.name)
 805         return task_args
 806
 807     @staticmethod
 808     def _remove_plugins_extra():
 809         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 810         try:
 811             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 812             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 813         except Exception:  # pylint: disable=broad-except
 814             pass
 815
 816     def prepare_task(self, test_name):
 817         """Prepare resources for test run."""
 818         self._remove_plugins_extra()
 819         jobs_dir = os.path.join(
 820             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 821         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 822         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 823                         os.path.join(inst_dir, 'plugins'))
 824         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 825                         os.path.join(inst_dir, 'extra'))
 826
 827         task_name = self.task_yaml.get(test_name).get("task")
 828         task = os.path.join(jobs_dir, task_name)
 829         if not os.path.exists(task):
 830             raise Exception("The scenario '%s' does not exist." % task)
 831         LOGGER.debug('Scenario fetched from : %s', task)
 832
 833         if not os.path.exists(self.temp_dir):
 834             os.makedirs(self.temp_dir)
 835         task_file_name = os.path.join(self.temp_dir, task_name)
 836         self.apply_blacklist(task, task_file_name)
 837         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 838                          "--task-args", str(self.build_task_args(test_name))])
 839         return True
 840
 841     def clean(self):
 842         self._remove_plugins_extra()
 843         super(RallyJobs, self).clean()