functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican']
  45
  46     rally_conf_path = "/etc/rally/rally.conf"
  47     rally_aar4_patch_path = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     rally_dir = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     rally_scenario_dir = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     template_dir = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     support_dir = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     users_amount = 2
  58     tenants_amount = 3
  59     iterations_amount = 10
  60     concurrency = 4
  61     volume_version = 3
  62     volume_service_type = "volumev3"
  63     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  64     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     temp_dir = os.path.join(task_dir, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69     allow_no_fip = True
  70     task_timeout = 3600
  71
  72     def __init__(self, **kwargs):
  73         """Initialize RallyBase object."""
  74         super(RallyBase, self).__init__(**kwargs)
  75         assert self.orig_cloud
  76         assert self.project
  77         if self.orig_cloud.get_role("admin"):
  78             role_name = "admin"
  79         elif self.orig_cloud.get_role("Admin"):
  80             role_name = "Admin"
  81         else:
  82             raise Exception("Cannot detect neither admin nor Admin")
  83         self.orig_cloud.grant_role(
  84             role_name, user=self.project.user.id,
  85             project=self.project.project.id,
  86             domain=self.project.domain.id)
  87         self.results_dir = os.path.join(
  88             getattr(config.CONF, 'dir_results'), self.case_name)
  89         self.task_file = ''
  90         self.creators = []
  91         self.summary = []
  92         self.scenario_dir = ''
  93         self.smoke = None
  94         self.start_time = None
  95         self.result = None
  96         self.details = None
  97         self.compute_cnt = 0
  98         self.flavor_alt = None
  99         self.tests = []
 100         self.run_cmd = ''
 101         self.network_extensions = []
 102         self.services = []
 103
 104     def build_task_args(self, test_name):
 105         """Build arguments for the Rally task."""
 106         task_args = {'service_list': [test_name]}
 107         task_args['image_name'] = str(self.image.name)
 108         task_args['flavor_name'] = str(self.flavor.name)
 109         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 110         task_args['glance_image_location'] = str(self.filename)
 111         task_args['glance_image_format'] = str(self.image_format)
 112         task_args['tmpl_dir'] = str(self.template_dir)
 113         task_args['sup_dir'] = str(self.support_dir)
 114         task_args['users_amount'] = self.users_amount
 115         task_args['tenants_amount'] = self.tenants_amount
 116         task_args['use_existing_users'] = False
 117         task_args['iterations'] = self.iterations_amount
 118         task_args['concurrency'] = self.concurrency
 119         task_args['smoke'] = self.smoke
 120         task_args['volume_version'] = self.volume_version
 121         task_args['volume_service_type'] = self.volume_service_type
 122         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 123
 124         if self.ext_net:
 125             task_args['floating_network'] = str(self.ext_net.name)
 126         else:
 127             task_args['floating_network'] = ''
 128
 129         if self.network:
 130             task_args['netid'] = str(self.network.id)
 131         else:
 132             task_args['netid'] = ''
 133
 134         return task_args
 135
 136     def _prepare_test_list(self, test_name):
 137         """Build the list of test cases to be executed."""
 138         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 139         scenario_file_name = os.path.join(self.rally_scenario_dir,
 140                                           test_yaml_file_name)
 141
 142         if not os.path.exists(scenario_file_name):
 143             scenario_file_name = os.path.join(self.scenario_dir,
 144                                               test_yaml_file_name)
 145
 146             if not os.path.exists(scenario_file_name):
 147                 raise Exception("The scenario '%s' does not exist."
 148                                 % scenario_file_name)
 149
 150         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 151         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 152
 153         if not os.path.exists(self.temp_dir):
 154             os.makedirs(self.temp_dir)
 155
 156         self.apply_blacklist(scenario_file_name, test_file_name)
 157         return test_file_name
 158
 159     @staticmethod
 160     def get_verifier_deployment_id():
 161         """
 162         Returns deployment id for active Rally deployment
 163         """
 164         cmd = ("rally deployment list | awk '/" +
 165                getattr(config.CONF, 'rally_deployment_name') +
 166                "/ {print $2}'")
 167         proc = subprocess.Popen(cmd, shell=True,
 168                                 stdout=subprocess.PIPE,
 169                                 stderr=subprocess.STDOUT)
 170         deployment_uuid = proc.stdout.readline().rstrip()
 171         return deployment_uuid.decode("utf-8")
 172
 173     @staticmethod
 174     def create_rally_deployment(environ=None):
 175         # pylint: disable=unexpected-keyword-arg
 176         """Create new rally deployment"""
 177         # set the architecture to default
 178         pod_arch = env.get("POD_ARCH")
 179         arch_filter = ['aarch64']
 180
 181         if pod_arch and pod_arch in arch_filter:
 182             LOGGER.info("Apply aarch64 specific to rally config...")
 183             with open(RallyBase.rally_aar4_patch_path, "r") as pfile:
 184                 rally_patch_conf = pfile.read()
 185
 186             for line in fileinput.input(RallyBase.rally_conf_path):
 187                 print(line, end=' ')
 188                 if "cirros|testvm" in line:
 189                     print(rally_patch_conf)
 190
 191         LOGGER.info("Creating Rally environment...")
 192         try:
 193             cmd = ['rally', 'deployment', 'destroy',
 194                    '--deployment',
 195                    str(getattr(config.CONF, 'rally_deployment_name'))]
 196             output = subprocess.check_output(cmd)
 197             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 198         except subprocess.CalledProcessError:
 199             pass
 200
 201         cmd = ['rally', 'deployment', 'create', '--fromenv',
 202                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 203         output = subprocess.check_output(cmd, env=environ)
 204         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 205
 206         cmd = ['rally', 'deployment', 'check']
 207         output = subprocess.check_output(cmd)
 208         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 209         return RallyBase.get_verifier_deployment_id()
 210
 211     @staticmethod
 212     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 213         """Set keystone_default_role in rally.conf"""
 214         if env.get("NEW_USER_ROLE").lower() != "member":
 215             rconfig = configparser.RawConfigParser()
 216             rconfig.read(rally_conf)
 217             if not rconfig.has_section('openstack'):
 218                 rconfig.add_section('openstack')
 219             rconfig.set(
 220                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 221             with open(rally_conf, 'w') as config_file:
 222                 rconfig.write(config_file)
 223
 224     @staticmethod
 225     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 226         """Clean Rally config"""
 227         if env.get("NEW_USER_ROLE").lower() != "member":
 228             rconfig = configparser.RawConfigParser()
 229             rconfig.read(rally_conf)
 230             if rconfig.has_option('openstack', 'keystone_default_role'):
 231                 rconfig.remove_option('openstack', 'keystone_default_role')
 232             with open(rally_conf, 'w') as config_file:
 233                 rconfig.write(config_file)
 234
 235     @staticmethod
 236     def get_task_id(cmd_raw):
 237         """
 238         Get task id from command rally result.
 239
 240         :param cmd_raw:
 241         :return: task_id as string
 242         """
 243         taskid_re = re.compile('^Task +(.*): started$')
 244         for line in cmd_raw.splitlines(True):
 245             line = line.strip()
 246             match = taskid_re.match(line.decode("utf-8"))
 247             if match:
 248                 return match.group(1)
 249         return None
 250
 251     @staticmethod
 252     def task_succeed(json_raw):
 253         """
 254         Parse JSON from rally JSON results.
 255
 256         :param json_raw:
 257         :return: Bool
 258         """
 259         rally_report = json.loads(json_raw)
 260         tasks = rally_report.get('tasks')
 261         if tasks:
 262             for task in tasks:
 263                 if task.get('status') != 'finished' or \
 264                    task.get('pass_sla') is not True:
 265                     return False
 266         else:
 267             return False
 268         return True
 269
 270     def _migration_supported(self):
 271         """Determine if migration is supported."""
 272         if self.compute_cnt > 1:
 273             return True
 274         return False
 275
 276     def _network_trunk_supported(self):
 277         """Determine if network trunk service is available"""
 278         if 'trunk' in self.network_extensions:
 279             return True
 280         return False
 281
 282     @staticmethod
 283     def excl_scenario():
 284         """Exclude scenario."""
 285         black_tests = []
 286         try:
 287             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 288                 black_list_yaml = yaml.safe_load(black_list_file)
 289
 290             deploy_scenario = env.get('DEPLOY_SCENARIO')
 291             if (bool(deploy_scenario) and
 292                     'scenario' in black_list_yaml.keys()):
 293                 for item in black_list_yaml['scenario']:
 294                     scenarios = item['scenarios']
 295                     in_it = RallyBase.in_iterable_re
 296                     if in_it(deploy_scenario, scenarios):
 297                         tests = item['tests']
 298                         black_tests.extend(tests)
 299         except Exception:  # pylint: disable=broad-except
 300             LOGGER.debug("Scenario exclusion not applied.")
 301
 302         return black_tests
 303
 304     @staticmethod
 305     def in_iterable_re(needle, haystack):
 306         """
 307         Check if given needle is in the iterable haystack, using regex.
 308
 309         :param needle: string to be matched
 310         :param haystack: iterable of strings (optionally regex patterns)
 311         :return: True if needle is eqial to any of the elements in haystack,
 312                  or if a nonempty regex pattern in haystack is found in needle.
 313         """
 314         # match without regex
 315         if needle in haystack:
 316             return True
 317
 318         for pattern in haystack:
 319             # match if regex pattern is set and found in the needle
 320             if pattern and re.search(pattern, needle) is not None:
 321                 return True
 322
 323         return False
 324
 325     def excl_func(self):
 326         """Exclude functionalities."""
 327         black_tests = []
 328         func_list = []
 329
 330         try:
 331             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 332                 black_list_yaml = yaml.safe_load(black_list_file)
 333
 334             if env.get('BLOCK_MIGRATION').lower() == 'true':
 335                 func_list.append("block_migration")
 336             if not self._migration_supported():
 337                 func_list.append("no_migration")
 338             if not self._network_trunk_supported():
 339                 func_list.append("no_net_trunk_service")
 340             if not self.ext_net:
 341                 func_list.append("no_floating_ip")
 342
 343             if 'functionality' in black_list_yaml.keys():
 344                 for item in black_list_yaml['functionality']:
 345                     functions = item['functions']
 346                     for func in func_list:
 347                         if func in functions:
 348                             tests = item['tests']
 349                             black_tests.extend(tests)
 350         except Exception:  # pylint: disable=broad-except
 351             LOGGER.debug("Functionality exclusion not applied.")
 352
 353         return black_tests
 354
 355     def apply_blacklist(self, case_file_name, result_file_name):
 356         """Apply blacklist."""
 357         LOGGER.debug("Applying blacklist...")
 358         cases_file = open(case_file_name, 'r')
 359         result_file = open(result_file_name, 'w')
 360
 361         black_tests = list(set(self.excl_func() +
 362                                self.excl_scenario()))
 363
 364         if black_tests:
 365             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 366
 367         include = True
 368         for cases_line in cases_file:
 369             if include:
 370                 for black_tests_line in black_tests:
 371                     if re.search(black_tests_line,
 372                                  cases_line.strip().rstrip(':')):
 373                         include = False
 374                         break
 375                 else:
 376                     result_file.write(str(cases_line))
 377             else:
 378                 if cases_line.isspace():
 379                     include = True
 380
 381         cases_file.close()
 382         result_file.close()
 383
 384     @staticmethod
 385     def file_is_empty(file_name):
 386         """Determine is a file is empty."""
 387         try:
 388             if os.stat(file_name).st_size > 0:
 389                 return False
 390         except Exception:  # pylint: disable=broad-except
 391             pass
 392
 393         return True
 394
 395     def _save_results(self, test_name, task_id):
 396         """ Generate and save task execution results"""
 397         # check for result directory and create it otherwise
 398         if not os.path.exists(self.results_dir):
 399             LOGGER.debug('%s does not exist, we create it.',
 400                          self.results_dir)
 401             os.makedirs(self.results_dir)
 402
 403         # put detailed result to log
 404         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 405         LOGGER.debug('running command: %s', cmd)
 406         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 407         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 408
 409         # save report as JSON
 410         report_json_name = '{}.json'.format(test_name)
 411         report_json_dir = os.path.join(self.results_dir, report_json_name)
 412         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 413                 "--out", report_json_dir])
 414         LOGGER.debug('running command: %s', cmd)
 415         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 416         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 417
 418         json_results = open(report_json_dir).read()
 419         self._append_summary(json_results, test_name)
 420
 421         # parse JSON operation result
 422         if self.task_succeed(json_results):
 423             LOGGER.info('Test scenario: "%s" OK.', test_name)
 424         else:
 425             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 426
 427     def run_task(self, test_name):
 428         """Run a task."""
 429         LOGGER.info('Starting test scenario "%s" ...', test_name)
 430         LOGGER.debug('running command: %s', self.run_cmd)
 431         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 432                                 stderr=subprocess.STDOUT)
 433         try:
 434             output = proc.communicate(timeout=self.task_timeout)[0]
 435         except subprocess.TimeoutExpired:
 436             proc.kill()
 437             proc.communicate()
 438             LOGGER.error("Failed to complete run task")
 439             raise Exception("Failed to complete run task")
 440         task_id = self.get_task_id(output)
 441         LOGGER.debug('task_id : %s', task_id)
 442         if task_id is None:
 443             LOGGER.error("Failed to retrieve task_id")
 444             LOGGER.error("Result:\n%s", output.decode("utf-8"))
 445             raise Exception("Failed to retrieve task id")
 446         self._save_results(test_name, task_id)
 447
 448     def _append_summary(self, json_raw, test_name):
 449         # pylint: disable=too-many-locals
 450         """Update statistics summary info."""
 451         nb_tests = 0
 452         nb_success = 0
 453         overall_duration = 0.0
 454         success = []
 455         failures = []
 456
 457         rally_report = json.loads(json_raw)
 458         for task in rally_report.get('tasks'):
 459             for subtask in task.get('subtasks'):
 460                 has_errors = False
 461                 for workload in subtask.get('workloads'):
 462                     if workload.get('full_duration'):
 463                         overall_duration += workload.get('full_duration')
 464
 465                     if workload.get('data'):
 466                         nb_tests += len(workload.get('data'))
 467
 468                     for result in workload.get('data'):
 469                         if not result.get('error'):
 470                             nb_success += 1
 471                         else:
 472                             has_errors = True
 473
 474                 if has_errors:
 475                     failures.append(subtask['title'])
 476                 else:
 477                     success.append(subtask['title'])
 478
 479         scenario_summary = {'test_name': test_name,
 480                             'overall_duration': overall_duration,
 481                             'nb_tests': nb_tests,
 482                             'nb_success': nb_success,
 483                             'success': success,
 484                             'failures': failures,
 485                             'task_status': self.task_succeed(json_raw)}
 486         self.summary.append(scenario_summary)
 487
 488     def prepare_run(self, **kwargs):
 489         """Prepare resources needed by test scenarios."""
 490         assert self.cloud
 491         LOGGER.debug('Validating run tests...')
 492         for test in kwargs.get('tests', self.stests):
 493             if test in self.stests:
 494                 self.tests.append(test)
 495             else:
 496                 raise Exception("Test name '%s' is invalid" % test)
 497
 498         if not os.path.exists(self.task_dir):
 499             os.makedirs(self.task_dir)
 500
 501         task = os.path.join(self.rally_dir, 'task.yaml')
 502         if not os.path.exists(task):
 503             LOGGER.error("Task file '%s' does not exist.", task)
 504             raise Exception("Task file '{}' does not exist.".
 505                             format(task))
 506         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 507         shutil.copyfile(task, self.task_file)
 508
 509         task_macro = os.path.join(self.rally_dir, 'macro')
 510         if not os.path.exists(task_macro):
 511             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 512             raise Exception("Task macro dir '{}' does not exist.".
 513                             format(task_macro))
 514         macro_dir = os.path.join(self.task_dir, 'macro')
 515         if os.path.exists(macro_dir):
 516             shutil.rmtree(macro_dir)
 517         shutil.copytree(task_macro, macro_dir)
 518
 519         self.update_keystone_default_role()
 520         self.compute_cnt = len(self.cloud.list_hypervisors())
 521         self.network_extensions = self.cloud.get_network_extensions()
 522         self.flavor_alt = self.create_flavor_alt()
 523         self.services = [service.name for service in
 524                          self.cloud.list_services()]
 525
 526         LOGGER.debug("flavor: %s", self.flavor_alt)
 527
 528     def prepare_task(self, test_name):
 529         """Prepare resources for test run."""
 530         file_name = self._prepare_test_list(test_name)
 531         if self.file_is_empty(file_name):
 532             LOGGER.info('No tests for scenario "%s"', test_name)
 533             return False
 534         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 535                          "--task", self.task_file, "--task-args",
 536                          str(self.build_task_args(test_name))])
 537         return True
 538
 539     def run_tests(self, **kwargs):
 540         """Execute tests."""
 541         optional = kwargs.get('optional', [])
 542         for test in self.tests:
 543             if test in self.services or test not in optional:
 544                 if self.prepare_task(test):
 545                     self.run_task(test)
 546
 547     def _generate_report(self):
 548         """Generate test execution summary report."""
 549         total_duration = 0.0
 550         total_nb_tests = 0
 551         total_nb_success = 0
 552         nb_modules = 0
 553         payload = []
 554
 555         res_table = prettytable.PrettyTable(
 556             padding_width=2,
 557             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 558         res_table.align['Module'] = "l"
 559         res_table.align['Duration'] = "r"
 560         res_table.align['Success'] = "r"
 561
 562         # for each scenario we draw a row for the table
 563         for item in self.summary:
 564             if item['task_status'] is True:
 565                 nb_modules += 1
 566             total_duration += item['overall_duration']
 567             total_nb_tests += item['nb_tests']
 568             total_nb_success += item['nb_success']
 569             try:
 570                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 571             except ZeroDivisionError:
 572                 success_avg = 0
 573             success_str = str("{:0.2f}".format(success_avg)) + '%'
 574             duration_str = time.strftime("%H:%M:%S",
 575                                          time.gmtime(item['overall_duration']))
 576             res_table.add_row([item['test_name'], duration_str,
 577                                item['nb_tests'], success_str])
 578             payload.append({'module': item['test_name'],
 579                             'details': {'duration': item['overall_duration'],
 580                                         'nb tests': item['nb_tests'],
 581                                         'success rate': success_str,
 582                                         'success': item['success'],
 583                                         'failures': item['failures']}})
 584
 585         total_duration_str = time.strftime("%H:%M:%S",
 586                                            time.gmtime(total_duration))
 587         try:
 588             self.result = 100 * total_nb_success / total_nb_tests
 589         except ZeroDivisionError:
 590             self.result = 100
 591         success_rate = "{:0.2f}".format(self.result)
 592         success_rate_str = str(success_rate) + '%'
 593         res_table.add_row(["", "", "", ""])
 594         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 595                            success_rate_str])
 596
 597         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 598         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 599                     self.case_name, success_rate, nb_modules,
 600                     len(self.summary))
 601         payload.append({'summary': {'duration': total_duration,
 602                                     'nb tests': total_nb_tests,
 603                                     'nb success': success_rate}})
 604         self.details = payload
 605
 606     @staticmethod
 607     def export_task(file_name, export_type="html"):
 608         """Export all task results (e.g. html or xunit report)
 609
 610         Raises:
 611             subprocess.CalledProcessError: if Rally doesn't return 0
 612
 613         Returns:
 614             None
 615         """
 616         cmd = ["rally", "task", "export", "--type", export_type,
 617                "--deployment",
 618                str(getattr(config.CONF, 'rally_deployment_name')),
 619                "--to", file_name]
 620         LOGGER.debug('running command: %s', cmd)
 621         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 622         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 623
 624     @staticmethod
 625     def verify_report(file_name, uuid, export_type="html"):
 626         """Generate the verifier report (e.g. html or xunit report)
 627
 628         Raises:
 629             subprocess.CalledProcessError: if Rally doesn't return 0
 630
 631         Returns:
 632             None
 633         """
 634         cmd = ["rally", "verify", "report", "--type", export_type,
 635                "--uuid", uuid, "--to", file_name]
 636         LOGGER.debug('running command: %s', cmd)
 637         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 638         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 639
 640     def clean(self):
 641         """Cleanup of OpenStack resources. Should be called on completion."""
 642         self.clean_rally_conf()
 643         self.clean_rally_logs()
 644         if self.flavor_alt:
 645             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 646         super(RallyBase, self).clean()
 647
 648     def is_successful(self):
 649         """The overall result of the test."""
 650         for item in self.summary:
 651             if item['task_status'] is False:
 652                 return testcase.TestCase.EX_TESTCASE_FAILED
 653
 654         return super(RallyBase, self).is_successful()
 655
 656     @staticmethod
 657     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 658         """Print rally logs in res dir"""
 659         if not os.path.exists(res_dir):
 660             os.makedirs(res_dir)
 661         rconfig = configparser.RawConfigParser()
 662         rconfig.read(rally_conf)
 663         rconfig.set('DEFAULT', 'debug', True)
 664         rconfig.set('DEFAULT', 'use_stderr', False)
 665         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 666         rconfig.set('DEFAULT', 'log_dir', res_dir)
 667         with open(rally_conf, 'w') as config_file:
 668             rconfig.write(config_file)
 669
 670     @staticmethod
 671     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 672         """Clean Rally config"""
 673         rconfig = configparser.RawConfigParser()
 674         rconfig.read(rally_conf)
 675         if rconfig.has_option('DEFAULT', 'use_stderr'):
 676             rconfig.remove_option('DEFAULT', 'use_stderr')
 677         if rconfig.has_option('DEFAULT', 'debug'):
 678             rconfig.remove_option('DEFAULT', 'debug')
 679         if rconfig.has_option('DEFAULT', 'log-file'):
 680             rconfig.remove_option('DEFAULT', 'log-file')
 681         if rconfig.has_option('DEFAULT', 'log_dir'):
 682             rconfig.remove_option('DEFAULT', 'log_dir')
 683         with open(rally_conf, 'w') as config_file:
 684             rconfig.write(config_file)
 685
 686     def run(self, **kwargs):
 687         """Run testcase."""
 688         self.start_time = time.time()
 689         try:
 690             assert super(RallyBase, self).run(
 691                 **kwargs) == testcase.TestCase.EX_OK
 692             self.update_rally_logs(self.res_dir)
 693             self.create_rally_deployment(environ=self.project.get_environ())
 694             self.prepare_run(**kwargs)
 695             self.run_tests(**kwargs)
 696             self._generate_report()
 697             self.export_task(
 698                 "{}/{}.html".format(self.results_dir, self.case_name))
 699             self.export_task(
 700                 "{}/{}.xml".format(self.results_dir, self.case_name),
 701                 export_type="junit-xml")
 702             res = testcase.TestCase.EX_OK
 703         except Exception:   # pylint: disable=broad-except
 704             LOGGER.exception('Error with run:')
 705             self.result = 0
 706             res = testcase.TestCase.EX_RUN_ERROR
 707         self.stop_time = time.time()
 708         return res
 709
 710
 711 class RallySanity(RallyBase):
 712     """Rally sanity testcase implementation."""
 713
 714     def __init__(self, **kwargs):
 715         """Initialize RallySanity object."""
 716         if "case_name" not in kwargs:
 717             kwargs["case_name"] = "rally_sanity"
 718         super(RallySanity, self).__init__(**kwargs)
 719         self.smoke = True
 720         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 721
 722
 723 class RallyFull(RallyBase):
 724     """Rally full testcase implementation."""
 725
 726     task_timeout = 7200
 727
 728     def __init__(self, **kwargs):
 729         """Initialize RallyFull object."""
 730         if "case_name" not in kwargs:
 731             kwargs["case_name"] = "rally_full"
 732         super(RallyFull, self).__init__(**kwargs)
 733         self.smoke = False
 734         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 735
 736
 737 class RallyJobs(RallyBase):
 738     """Rally OpenStack CI testcase implementation."""
 739
 740     stests = ["neutron"]
 741     task_timeout = 7200
 742
 743     def __init__(self, **kwargs):
 744         """Initialize RallyJobs object."""
 745         if "case_name" not in kwargs:
 746             kwargs["case_name"] = "rally_jobs"
 747         super(RallyJobs, self).__init__(**kwargs)
 748         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 749         self.task_yaml = None
 750
 751     def prepare_run(self, **kwargs):
 752         """Create resources needed by test scenarios."""
 753         super(RallyJobs, self).prepare_run(**kwargs)
 754         with open(os.path.join(self.rally_dir,
 755                                'rally_jobs.yaml'), 'r') as task_file:
 756             self.task_yaml = yaml.safe_load(task_file)
 757
 758         for task in self.task_yaml:
 759             if task not in self.tests:
 760                 raise Exception("Test '%s' not in '%s'" %
 761                                 (task, self.tests))
 762
 763     def apply_blacklist(self, case_file_name, result_file_name):
 764         # pylint: disable=too-many-branches
 765         """Apply blacklist."""
 766         LOGGER.debug("Applying blacklist...")
 767         black_tests = list(set(self.excl_func() +
 768                                self.excl_scenario()))
 769         if black_tests:
 770             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 771
 772         template = YAML(typ='jinja2')
 773         with open(case_file_name, 'r') as fname:
 774             cases = template.load(fname)
 775         if cases.get("version", 1) == 1:
 776             # scenarios in dictionary
 777             for name in cases.keys():
 778                 if self.in_iterable_re(name, black_tests):
 779                     cases.pop(name)
 780         else:
 781             # workloads in subtasks
 782             for sind, subtask in reversed(list(
 783                     enumerate(cases.get('subtasks', [])))):
 784                 for wind, workload in reversed(list(
 785                         enumerate(subtask.get('workloads', [])))):
 786                     scenario = workload.get('scenario', {})
 787                     for name in scenario.keys():
 788                         if self.in_iterable_re(name, black_tests):
 789                             cases['subtasks'][sind]['workloads'].pop(wind)
 790                             break
 791                 if 'workloads' in cases['subtasks'][sind]:
 792                     if not cases['subtasks'][sind]['workloads']:
 793                         cases['subtasks'].pop(sind)
 794             # scenarios in subtasks
 795             for sind, subtask in reversed(list(
 796                     enumerate(cases.get('subtasks', [])))):
 797                 scenario = subtask.get('scenario', {})
 798                 for name in scenario.keys():
 799                     if self.in_iterable_re(name, black_tests):
 800                         cases['subtasks'].pop(sind)
 801                         break
 802
 803         with open(result_file_name, 'w') as fname:
 804             template.dump(cases, fname)
 805
 806     def build_task_args(self, test_name):
 807         """Build arguments for the Rally task."""
 808         task_args = {}
 809         if self.ext_net:
 810             task_args['floating_network'] = str(self.ext_net.name)
 811         else:
 812             task_args['floating_network'] = ''
 813         task_args['image_name'] = str(self.image.name)
 814         task_args['flavor_name'] = str(self.flavor.name)
 815         return task_args
 816
 817     @staticmethod
 818     def _remove_plugins_extra():
 819         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 820         try:
 821             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 822             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 823         except Exception:  # pylint: disable=broad-except
 824             pass
 825
 826     def prepare_task(self, test_name):
 827         """Prepare resources for test run."""
 828         self._remove_plugins_extra()
 829         jobs_dir = os.path.join(
 830             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 831         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 832         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 833                         os.path.join(inst_dir, 'plugins'))
 834         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 835                         os.path.join(inst_dir, 'extra'))
 836
 837         task_name = self.task_yaml.get(test_name).get("task")
 838         task = os.path.join(jobs_dir, task_name)
 839         if not os.path.exists(task):
 840             raise Exception("The scenario '%s' does not exist." % task)
 841         LOGGER.debug('Scenario fetched from : %s', task)
 842
 843         if not os.path.exists(self.temp_dir):
 844             os.makedirs(self.temp_dir)
 845         task_file_name = os.path.join(self.temp_dir, task_name)
 846         self.apply_blacklist(task, task_file_name)
 847         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 848                          "--task-args", str(self.build_task_args(test_name))])
 849         return True
 850
 851     def clean(self):
 852         self._remove_plugins_extra()
 853         super(RallyJobs, self).clean()