functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 import six
  29 from six.moves import configparser
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.core import singlevm
  34 from functest.utils import config
  35 from functest.utils import env
  36 from functest.utils import functest_utils
  37
  38 LOGGER = logging.getLogger(__name__)
  39
  40
  41 class RallyBase(singlevm.VmReady2):
  42     """Base class form Rally testcases implementation."""
  43
  44     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  45     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  46               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican']
  47
  48     rally_conf_path = "/etc/rally/rally.conf"
  49     rally_aar4_patch_path = pkg_resources.resource_filename(
  50         'functest', 'ci/rally_aarch64_patch.conf')
  51     rally_dir = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally')
  53     rally_scenario_dir = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario')
  55     template_dir = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  57     support_dir = pkg_resources.resource_filename(
  58         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  59     users_amount = 2
  60     tenants_amount = 3
  61     iterations_amount = 10
  62     concurrency = 4
  63     volume_version = 3
  64     volume_service_type = "volumev3"
  65     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  66     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  67     temp_dir = os.path.join(task_dir, 'var')
  68
  69     visibility = 'public'
  70     shared_network = True
  71     allow_no_fip = True
  72     task_timeout = 3600
  73
  74     def __init__(self, **kwargs):
  75         """Initialize RallyBase object."""
  76         super(RallyBase, self).__init__(**kwargs)
  77         assert self.orig_cloud
  78         assert self.project
  79         if self.orig_cloud.get_role("admin"):
  80             role_name = "admin"
  81         elif self.orig_cloud.get_role("Admin"):
  82             role_name = "Admin"
  83         else:
  84             raise Exception("Cannot detect neither admin nor Admin")
  85         self.orig_cloud.grant_role(
  86             role_name, user=self.project.user.id,
  87             project=self.project.project.id,
  88             domain=self.project.domain.id)
  89         self.results_dir = os.path.join(
  90             getattr(config.CONF, 'dir_results'), self.case_name)
  91         self.task_file = ''
  92         self.creators = []
  93         self.summary = []
  94         self.scenario_dir = ''
  95         self.smoke = None
  96         self.start_time = None
  97         self.result = None
  98         self.compute_cnt = 0
  99         self.flavor_alt = None
 100         self.tests = []
 101         self.run_cmd = ''
 102         self.network_extensions = []
 103         self.services = []
 104
 105     def build_task_args(self, test_name):
 106         """Build arguments for the Rally task."""
 107         task_args = {'service_list': [test_name]}
 108         task_args['image_name'] = str(self.image.name)
 109         task_args['flavor_name'] = str(self.flavor.name)
 110         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 111         task_args['glance_image_location'] = str(self.filename)
 112         task_args['glance_image_format'] = str(self.image_format)
 113         task_args['tmpl_dir'] = str(self.template_dir)
 114         task_args['sup_dir'] = str(self.support_dir)
 115         task_args['users_amount'] = self.users_amount
 116         task_args['tenants_amount'] = self.tenants_amount
 117         task_args['use_existing_users'] = False
 118         task_args['iterations'] = self.iterations_amount
 119         task_args['concurrency'] = self.concurrency
 120         task_args['smoke'] = self.smoke
 121         task_args['volume_version'] = self.volume_version
 122         task_args['volume_service_type'] = self.volume_service_type
 123         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 124
 125         if self.ext_net:
 126             task_args['floating_network'] = str(self.ext_net.name)
 127         else:
 128             task_args['floating_network'] = ''
 129
 130         if self.network:
 131             task_args['netid'] = str(self.network.id)
 132         else:
 133             task_args['netid'] = ''
 134
 135         return task_args
 136
 137     def _prepare_test_list(self, test_name):
 138         """Build the list of test cases to be executed."""
 139         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 140         scenario_file_name = os.path.join(self.rally_scenario_dir,
 141                                           test_yaml_file_name)
 142
 143         if not os.path.exists(scenario_file_name):
 144             scenario_file_name = os.path.join(self.scenario_dir,
 145                                               test_yaml_file_name)
 146
 147             if not os.path.exists(scenario_file_name):
 148                 raise Exception("The scenario '%s' does not exist."
 149                                 % scenario_file_name)
 150
 151         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 152         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 153
 154         if not os.path.exists(self.temp_dir):
 155             os.makedirs(self.temp_dir)
 156
 157         self.apply_blacklist(scenario_file_name, test_file_name)
 158         return test_file_name
 159
 160     @staticmethod
 161     def get_verifier_deployment_id():
 162         """
 163         Returns deployment id for active Rally deployment
 164         """
 165         cmd = ("rally deployment list | awk '/" +
 166                getattr(config.CONF, 'rally_deployment_name') +
 167                "/ {print $2}'")
 168         proc = subprocess.Popen(cmd, shell=True,
 169                                 stdout=subprocess.PIPE,
 170                                 stderr=subprocess.STDOUT)
 171         deployment_uuid = proc.stdout.readline().rstrip()
 172         return deployment_uuid.decode("utf-8")
 173
 174     @staticmethod
 175     def create_rally_deployment(environ=None):
 176         # pylint: disable=unexpected-keyword-arg
 177         """Create new rally deployment"""
 178         # set the architecture to default
 179         pod_arch = env.get("POD_ARCH")
 180         arch_filter = ['aarch64']
 181
 182         if pod_arch and pod_arch in arch_filter:
 183             LOGGER.info("Apply aarch64 specific to rally config...")
 184             with open(RallyBase.rally_aar4_patch_path, "r") as pfile:
 185                 rally_patch_conf = pfile.read()
 186
 187             for line in fileinput.input(RallyBase.rally_conf_path):
 188                 print(line, end=' ')
 189                 if "cirros|testvm" in line:
 190                     print(rally_patch_conf)
 191
 192         LOGGER.info("Creating Rally environment...")
 193         try:
 194             cmd = ['rally', 'deployment', 'destroy',
 195                    '--deployment',
 196                    str(getattr(config.CONF, 'rally_deployment_name'))]
 197             output = subprocess.check_output(cmd)
 198             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 199         except subprocess.CalledProcessError:
 200             pass
 201
 202         cmd = ['rally', 'deployment', 'create', '--fromenv',
 203                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 204         output = subprocess.check_output(cmd, env=environ)
 205         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 206
 207         cmd = ['rally', 'deployment', 'check']
 208         output = subprocess.check_output(cmd)
 209         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 210         return RallyBase.get_verifier_deployment_id()
 211
 212     @staticmethod
 213     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 214         """Set keystone_default_role in rally.conf"""
 215         if env.get("NEW_USER_ROLE").lower() != "member":
 216             rconfig = configparser.RawConfigParser()
 217             rconfig.read(rally_conf)
 218             if not rconfig.has_section('openstack'):
 219                 rconfig.add_section('openstack')
 220             rconfig.set(
 221                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 222             with open(rally_conf, 'w') as config_file:
 223                 rconfig.write(config_file)
 224
 225     @staticmethod
 226     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 227         """Clean Rally config"""
 228         if env.get("NEW_USER_ROLE").lower() != "member":
 229             rconfig = configparser.RawConfigParser()
 230             rconfig.read(rally_conf)
 231             if rconfig.has_option('openstack', 'keystone_default_role'):
 232                 rconfig.remove_option('openstack', 'keystone_default_role')
 233             with open(rally_conf, 'w') as config_file:
 234                 rconfig.write(config_file)
 235
 236     @staticmethod
 237     def get_task_id(tag):
 238         """
 239         Get task id from command rally result.
 240
 241         :param tag:
 242         :return: task_id as string
 243         """
 244         cmd = ["rally", "task", "list", "--tag", tag, "--uuids-only"]
 245         output = subprocess.check_output(cmd).decode("utf-8").rstrip()
 246         LOGGER.info("%s: %s", " ".join(cmd), output)
 247         return output
 248
 249     @staticmethod
 250     def task_succeed(json_raw):
 251         """
 252         Parse JSON from rally JSON results.
 253
 254         :param json_raw:
 255         :return: Bool
 256         """
 257         rally_report = json.loads(json_raw)
 258         tasks = rally_report.get('tasks')
 259         if tasks:
 260             for task in tasks:
 261                 if task.get('status') != 'finished' or \
 262                    task.get('pass_sla') is not True:
 263                     return False
 264         else:
 265             return False
 266         return True
 267
 268     def _migration_supported(self):
 269         """Determine if migration is supported."""
 270         if self.compute_cnt > 1:
 271             return True
 272         return False
 273
 274     def _network_trunk_supported(self):
 275         """Determine if network trunk service is available"""
 276         if 'trunk' in self.network_extensions:
 277             return True
 278         return False
 279
 280     @staticmethod
 281     def excl_scenario():
 282         """Exclude scenario."""
 283         black_tests = []
 284         try:
 285             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 286                 black_list_yaml = yaml.safe_load(black_list_file)
 287
 288             deploy_scenario = env.get('DEPLOY_SCENARIO')
 289             if (bool(deploy_scenario) and
 290                     'scenario' in black_list_yaml.keys()):
 291                 for item in black_list_yaml['scenario']:
 292                     scenarios = item['scenarios']
 293                     in_it = RallyBase.in_iterable_re
 294                     if in_it(deploy_scenario, scenarios):
 295                         tests = item['tests']
 296                         black_tests.extend(tests)
 297         except Exception:  # pylint: disable=broad-except
 298             LOGGER.debug("Scenario exclusion not applied.")
 299
 300         return black_tests
 301
 302     @staticmethod
 303     def in_iterable_re(needle, haystack):
 304         """
 305         Check if given needle is in the iterable haystack, using regex.
 306
 307         :param needle: string to be matched
 308         :param haystack: iterable of strings (optionally regex patterns)
 309         :return: True if needle is eqial to any of the elements in haystack,
 310                  or if a nonempty regex pattern in haystack is found in needle.
 311         """
 312         # match without regex
 313         if needle in haystack:
 314             return True
 315
 316         for pattern in haystack:
 317             # match if regex pattern is set and found in the needle
 318             if pattern and re.search(pattern, needle) is not None:
 319                 return True
 320
 321         return False
 322
 323     def excl_func(self):
 324         """Exclude functionalities."""
 325         black_tests = []
 326         func_list = []
 327
 328         try:
 329             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 330                 black_list_yaml = yaml.safe_load(black_list_file)
 331
 332             if env.get('BLOCK_MIGRATION').lower() == 'true':
 333                 func_list.append("block_migration")
 334             if not self._migration_supported():
 335                 func_list.append("no_migration")
 336             if not self._network_trunk_supported():
 337                 func_list.append("no_net_trunk_service")
 338             if not self.ext_net:
 339                 func_list.append("no_floating_ip")
 340
 341             if 'functionality' in black_list_yaml.keys():
 342                 for item in black_list_yaml['functionality']:
 343                     functions = item['functions']
 344                     for func in func_list:
 345                         if func in functions:
 346                             tests = item['tests']
 347                             black_tests.extend(tests)
 348         except Exception:  # pylint: disable=broad-except
 349             LOGGER.debug("Functionality exclusion not applied.")
 350
 351         return black_tests
 352
 353     def apply_blacklist(self, case_file_name, result_file_name):
 354         """Apply blacklist."""
 355         LOGGER.debug("Applying blacklist...")
 356         cases_file = open(case_file_name, 'r')
 357         result_file = open(result_file_name, 'w')
 358
 359         black_tests = list(set(self.excl_func() +
 360                                self.excl_scenario()))
 361
 362         if black_tests:
 363             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 364
 365         include = True
 366         for cases_line in cases_file:
 367             if include:
 368                 for black_tests_line in black_tests:
 369                     if re.search(black_tests_line,
 370                                  cases_line.strip().rstrip(':')):
 371                         include = False
 372                         break
 373                 else:
 374                     result_file.write(str(cases_line))
 375             else:
 376                 if cases_line.isspace():
 377                     include = True
 378
 379         cases_file.close()
 380         result_file.close()
 381
 382     @staticmethod
 383     def file_is_empty(file_name):
 384         """Determine is a file is empty."""
 385         try:
 386             if os.stat(file_name).st_size > 0:
 387                 return False
 388         except Exception:  # pylint: disable=broad-except
 389             pass
 390
 391         return True
 392
 393     def _save_results(self, test_name, task_id):
 394         """ Generate and save task execution results"""
 395         # check for result directory and create it otherwise
 396         if not os.path.exists(self.results_dir):
 397             LOGGER.debug('%s does not exist, we create it.',
 398                          self.results_dir)
 399             os.makedirs(self.results_dir)
 400
 401         # put detailed result to log
 402         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 403         LOGGER.debug('running command: %s', cmd)
 404         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 405         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 406
 407         # save report as JSON
 408         report_json_name = '{}.json'.format(test_name)
 409         report_json_dir = os.path.join(self.results_dir, report_json_name)
 410         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 411                 "--out", report_json_dir])
 412         LOGGER.debug('running command: %s', cmd)
 413         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 414         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 415
 416         json_results = open(report_json_dir).read()
 417         self._append_summary(json_results, test_name)
 418
 419         # parse JSON operation result
 420         if self.task_succeed(json_results):
 421             LOGGER.info('Test scenario: "%s" OK.', test_name)
 422         else:
 423             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 424
 425     def run_task(self, test_name):
 426         """Run a task."""
 427         LOGGER.info('Starting test scenario "%s" ...', test_name)
 428         LOGGER.debug('running command: %s', self.run_cmd)
 429         if six.PY3:
 430             subprocess.call(
 431                 self.run_cmd, timeout=self.task_timeout,
 432                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 433         else:
 434             with open(os.devnull, 'wb') as devnull:
 435                 subprocess.call(self.run_cmd, stdout=devnull, stderr=devnull)
 436         task_id = self.get_task_id(test_name)
 437         LOGGER.debug('task_id : %s', task_id)
 438         if not task_id:
 439             LOGGER.error("Failed to retrieve task_id")
 440             raise Exception("Failed to retrieve task id")
 441         self._save_results(test_name, task_id)
 442
 443     def _append_summary(self, json_raw, test_name):
 444         # pylint: disable=too-many-locals
 445         """Update statistics summary info."""
 446         nb_tests = 0
 447         nb_success = 0
 448         overall_duration = 0.0
 449         success = []
 450         failures = []
 451
 452         rally_report = json.loads(json_raw)
 453         for task in rally_report.get('tasks'):
 454             for subtask in task.get('subtasks'):
 455                 has_errors = False
 456                 for workload in subtask.get('workloads'):
 457                     if workload.get('full_duration'):
 458                         overall_duration += workload.get('full_duration')
 459
 460                     if workload.get('data'):
 461                         nb_tests += len(workload.get('data'))
 462
 463                     for result in workload.get('data'):
 464                         if not result.get('error'):
 465                             nb_success += 1
 466                         else:
 467                             has_errors = True
 468
 469                 if has_errors:
 470                     failures.append(subtask['title'])
 471                 else:
 472                     success.append(subtask['title'])
 473
 474         scenario_summary = {'test_name': test_name,
 475                             'overall_duration': overall_duration,
 476                             'nb_tests': nb_tests,
 477                             'nb_success': nb_success,
 478                             'success': success,
 479                             'failures': failures,
 480                             'task_status': self.task_succeed(json_raw)}
 481         self.summary.append(scenario_summary)
 482
 483     def prepare_run(self, **kwargs):
 484         """Prepare resources needed by test scenarios."""
 485         assert self.cloud
 486         LOGGER.debug('Validating run tests...')
 487         for test in kwargs.get('tests', self.stests):
 488             if test in self.stests:
 489                 self.tests.append(test)
 490             else:
 491                 raise Exception("Test name '%s' is invalid" % test)
 492
 493         if not os.path.exists(self.task_dir):
 494             os.makedirs(self.task_dir)
 495
 496         task = os.path.join(self.rally_dir, 'task.yaml')
 497         if not os.path.exists(task):
 498             LOGGER.error("Task file '%s' does not exist.", task)
 499             raise Exception("Task file '{}' does not exist.".
 500                             format(task))
 501         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 502         shutil.copyfile(task, self.task_file)
 503
 504         task_macro = os.path.join(self.rally_dir, 'macro')
 505         if not os.path.exists(task_macro):
 506             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 507             raise Exception("Task macro dir '{}' does not exist.".
 508                             format(task_macro))
 509         macro_dir = os.path.join(self.task_dir, 'macro')
 510         if os.path.exists(macro_dir):
 511             shutil.rmtree(macro_dir)
 512         shutil.copytree(task_macro, macro_dir)
 513
 514         self.update_keystone_default_role()
 515         self.compute_cnt = self.count_hypervisors()
 516         self.network_extensions = self.cloud.get_network_extensions()
 517         self.flavor_alt = self.create_flavor_alt()
 518         self.services = [service.name for service in
 519                          functest_utils.list_services(self.cloud)]
 520
 521         LOGGER.debug("flavor: %s", self.flavor_alt)
 522
 523     def prepare_task(self, test_name):
 524         """Prepare resources for test run."""
 525         file_name = self._prepare_test_list(test_name)
 526         if self.file_is_empty(file_name):
 527             LOGGER.info('No tests for scenario "%s"', test_name)
 528             return False
 529         self.run_cmd = (["rally", "task", "start", "--tag", test_name,
 530                          "--abort-on-sla-failure",
 531                          "--task", self.task_file, "--task-args",
 532                          str(self.build_task_args(test_name))])
 533         return True
 534
 535     def run_tests(self, **kwargs):
 536         """Execute tests."""
 537         optional = kwargs.get('optional', [])
 538         for test in self.tests:
 539             if test in self.services or test not in optional:
 540                 if self.prepare_task(test):
 541                     self.run_task(test)
 542
 543     def _generate_report(self):
 544         """Generate test execution summary report."""
 545         total_duration = 0.0
 546         total_nb_tests = 0
 547         total_nb_success = 0
 548         nb_modules = 0
 549         payload = []
 550
 551         res_table = prettytable.PrettyTable(
 552             padding_width=2,
 553             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 554         res_table.align['Module'] = "l"
 555         res_table.align['Duration'] = "r"
 556         res_table.align['Success'] = "r"
 557
 558         # for each scenario we draw a row for the table
 559         for item in self.summary:
 560             if item['task_status'] is True:
 561                 nb_modules += 1
 562             total_duration += item['overall_duration']
 563             total_nb_tests += item['nb_tests']
 564             total_nb_success += item['nb_success']
 565             try:
 566                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 567             except ZeroDivisionError:
 568                 success_avg = 0
 569             success_str = str("{:0.2f}".format(success_avg)) + '%'
 570             duration_str = time.strftime("%H:%M:%S",
 571                                          time.gmtime(item['overall_duration']))
 572             res_table.add_row([item['test_name'], duration_str,
 573                                item['nb_tests'], success_str])
 574             payload.append({'module': item['test_name'],
 575                             'details': {'duration': item['overall_duration'],
 576                                         'nb tests': item['nb_tests'],
 577                                         'success rate': success_str,
 578                                         'success': item['success'],
 579                                         'failures': item['failures']}})
 580
 581         total_duration_str = time.strftime("%H:%M:%S",
 582                                            time.gmtime(total_duration))
 583         try:
 584             self.result = 100 * total_nb_success / total_nb_tests
 585         except ZeroDivisionError:
 586             self.result = 100
 587         success_rate = "{:0.2f}".format(self.result)
 588         success_rate_str = str(success_rate) + '%'
 589         res_table.add_row(["", "", "", ""])
 590         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 591                            success_rate_str])
 592
 593         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 594         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 595                     self.case_name, success_rate, nb_modules,
 596                     len(self.summary))
 597         self.details['summary'] = {'duration': total_duration,
 598                                    'nb tests': total_nb_tests,
 599                                    'nb success': success_rate}
 600         self.details["modules"] = payload
 601
 602     @staticmethod
 603     def export_task(file_name, export_type="html"):
 604         """Export all task results (e.g. html or xunit report)
 605
 606         Raises:
 607             subprocess.CalledProcessError: if Rally doesn't return 0
 608
 609         Returns:
 610             None
 611         """
 612         cmd = ["rally", "task", "export", "--type", export_type,
 613                "--deployment",
 614                str(getattr(config.CONF, 'rally_deployment_name')),
 615                "--to", file_name]
 616         LOGGER.debug('running command: %s', cmd)
 617         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 618         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 619
 620     @staticmethod
 621     def verify_report(file_name, uuid, export_type="html"):
 622         """Generate the verifier report (e.g. html or xunit report)
 623
 624         Raises:
 625             subprocess.CalledProcessError: if Rally doesn't return 0
 626
 627         Returns:
 628             None
 629         """
 630         cmd = ["rally", "verify", "report", "--type", export_type,
 631                "--uuid", uuid, "--to", file_name]
 632         LOGGER.debug('running command: %s', cmd)
 633         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 634         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 635
 636     def clean(self):
 637         """Cleanup of OpenStack resources. Should be called on completion."""
 638         self.clean_rally_conf()
 639         self.clean_rally_logs()
 640         if self.flavor_alt:
 641             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 642         super(RallyBase, self).clean()
 643
 644     def is_successful(self):
 645         """The overall result of the test."""
 646         for item in self.summary:
 647             if item['task_status'] is False:
 648                 return testcase.TestCase.EX_TESTCASE_FAILED
 649
 650         return super(RallyBase, self).is_successful()
 651
 652     @staticmethod
 653     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 654         """Print rally logs in res dir"""
 655         if not os.path.exists(res_dir):
 656             os.makedirs(res_dir)
 657         rconfig = configparser.RawConfigParser()
 658         rconfig.read(rally_conf)
 659         rconfig.set('DEFAULT', 'debug', True)
 660         rconfig.set('DEFAULT', 'use_stderr', False)
 661         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 662         rconfig.set('DEFAULT', 'log_dir', res_dir)
 663         with open(rally_conf, 'w') as config_file:
 664             rconfig.write(config_file)
 665
 666     @staticmethod
 667     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 668         """Clean Rally config"""
 669         rconfig = configparser.RawConfigParser()
 670         rconfig.read(rally_conf)
 671         if rconfig.has_option('DEFAULT', 'use_stderr'):
 672             rconfig.remove_option('DEFAULT', 'use_stderr')
 673         if rconfig.has_option('DEFAULT', 'debug'):
 674             rconfig.remove_option('DEFAULT', 'debug')
 675         if rconfig.has_option('DEFAULT', 'log-file'):
 676             rconfig.remove_option('DEFAULT', 'log-file')
 677         if rconfig.has_option('DEFAULT', 'log_dir'):
 678             rconfig.remove_option('DEFAULT', 'log_dir')
 679         with open(rally_conf, 'w') as config_file:
 680             rconfig.write(config_file)
 681
 682     def run(self, **kwargs):
 683         """Run testcase."""
 684         self.start_time = time.time()
 685         try:
 686             assert super(RallyBase, self).run(
 687                 **kwargs) == testcase.TestCase.EX_OK
 688             self.update_rally_logs(self.res_dir)
 689             self.create_rally_deployment(environ=self.project.get_environ())
 690             self.prepare_run(**kwargs)
 691             self.run_tests(**kwargs)
 692             self._generate_report()
 693             self.export_task(
 694                 "{}/{}.html".format(self.results_dir, self.case_name))
 695             self.export_task(
 696                 "{}/{}.xml".format(self.results_dir, self.case_name),
 697                 export_type="junit-xml")
 698             res = testcase.TestCase.EX_OK
 699         except Exception:   # pylint: disable=broad-except
 700             LOGGER.exception('Error with run:')
 701             self.result = 0
 702             res = testcase.TestCase.EX_RUN_ERROR
 703         self.stop_time = time.time()
 704         return res
 705
 706
 707 class RallySanity(RallyBase):
 708     """Rally sanity testcase implementation."""
 709
 710     def __init__(self, **kwargs):
 711         """Initialize RallySanity object."""
 712         if "case_name" not in kwargs:
 713             kwargs["case_name"] = "rally_sanity"
 714         super(RallySanity, self).__init__(**kwargs)
 715         self.smoke = True
 716         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 717
 718
 719 class RallyFull(RallyBase):
 720     """Rally full testcase implementation."""
 721
 722     task_timeout = 7200
 723
 724     def __init__(self, **kwargs):
 725         """Initialize RallyFull object."""
 726         if "case_name" not in kwargs:
 727             kwargs["case_name"] = "rally_full"
 728         super(RallyFull, self).__init__(**kwargs)
 729         self.smoke = False
 730         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 731
 732
 733 class RallyJobs(RallyBase):
 734     """Rally OpenStack CI testcase implementation."""
 735
 736     stests = ["neutron"]
 737     task_timeout = 7200
 738
 739     def __init__(self, **kwargs):
 740         """Initialize RallyJobs object."""
 741         if "case_name" not in kwargs:
 742             kwargs["case_name"] = "rally_jobs"
 743         super(RallyJobs, self).__init__(**kwargs)
 744         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 745         self.task_yaml = None
 746
 747     def prepare_run(self, **kwargs):
 748         """Create resources needed by test scenarios."""
 749         super(RallyJobs, self).prepare_run(**kwargs)
 750         with open(os.path.join(self.rally_dir,
 751                                'rally_jobs.yaml'), 'r') as task_file:
 752             self.task_yaml = yaml.safe_load(task_file)
 753
 754         for task in self.task_yaml:
 755             if task not in self.tests:
 756                 raise Exception("Test '%s' not in '%s'" %
 757                                 (task, self.tests))
 758
 759     def apply_blacklist(self, case_file_name, result_file_name):
 760         # pylint: disable=too-many-branches
 761         """Apply blacklist."""
 762         LOGGER.debug("Applying blacklist...")
 763         black_tests = list(set(self.excl_func() +
 764                                self.excl_scenario()))
 765         if black_tests:
 766             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 767
 768         template = YAML(typ='jinja2')
 769         with open(case_file_name, 'r') as fname:
 770             cases = template.load(fname)
 771         if cases.get("version", 1) == 1:
 772             # scenarios in dictionary
 773             for name in cases.keys():
 774                 if self.in_iterable_re(name, black_tests):
 775                     cases.pop(name)
 776         else:
 777             # workloads in subtasks
 778             for sind, subtask in reversed(list(
 779                     enumerate(cases.get('subtasks', [])))):
 780                 for wind, workload in reversed(list(
 781                         enumerate(subtask.get('workloads', [])))):
 782                     scenario = workload.get('scenario', {})
 783                     for name in scenario.keys():
 784                         if self.in_iterable_re(name, black_tests):
 785                             cases['subtasks'][sind]['workloads'].pop(wind)
 786                             break
 787                 if 'workloads' in cases['subtasks'][sind]:
 788                     if not cases['subtasks'][sind]['workloads']:
 789                         cases['subtasks'].pop(sind)
 790             # scenarios in subtasks
 791             for sind, subtask in reversed(list(
 792                     enumerate(cases.get('subtasks', [])))):
 793                 scenario = subtask.get('scenario', {})
 794                 for name in scenario.keys():
 795                     if self.in_iterable_re(name, black_tests):
 796                         cases['subtasks'].pop(sind)
 797                         break
 798
 799         with open(result_file_name, 'w') as fname:
 800             template.dump(cases, fname)
 801
 802     def build_task_args(self, test_name):
 803         """Build arguments for the Rally task."""
 804         task_args = {}
 805         if self.ext_net:
 806             task_args['floating_network'] = str(self.ext_net.name)
 807         else:
 808             task_args['floating_network'] = ''
 809         task_args['image_name'] = str(self.image.name)
 810         task_args['flavor_name'] = str(self.flavor.name)
 811         return task_args
 812
 813     @staticmethod
 814     def _remove_plugins_extra():
 815         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 816         try:
 817             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 818             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 819         except Exception:  # pylint: disable=broad-except
 820             pass
 821
 822     def prepare_task(self, test_name):
 823         """Prepare resources for test run."""
 824         self._remove_plugins_extra()
 825         jobs_dir = os.path.join(
 826             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 827         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 828         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 829                         os.path.join(inst_dir, 'plugins'))
 830         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 831                         os.path.join(inst_dir, 'extra'))
 832
 833         task_name = self.task_yaml.get(test_name).get("task")
 834         task = os.path.join(jobs_dir, task_name)
 835         if not os.path.exists(task):
 836             raise Exception("The scenario '%s' does not exist." % task)
 837         LOGGER.debug('Scenario fetched from : %s', task)
 838
 839         if not os.path.exists(self.temp_dir):
 840             os.makedirs(self.temp_dir)
 841         task_file_name = os.path.join(self.temp_dir, task_name)
 842         self.apply_blacklist(task, task_file_name)
 843         self.run_cmd = (["rally", "task", "start", "--tag", test_name,
 844                          "--task", task_file_name,
 845                          "--task-args", str(self.build_task_args(test_name))])
 846         return True
 847
 848     def clean(self):
 849         self._remove_plugins_extra()
 850         super(RallyJobs, self).clean()