functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 import prettytable
  25 from ruamel.yaml import YAML
  26 from six.moves import configparser
  27 from xtesting.core import testcase
  28 from xtesting.energy import energy
  29 import yaml
  30
  31 from functest.core import singlevm
  32 from functest.opnfv_tests.openstack.tempest import conf_utils
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_DIR = pkg_resources.resource_filename(
  47         'functest', 'opnfv_tests/openstack/rally')
  48     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  49         'functest', 'opnfv_tests/openstack/rally/scenario')
  50     TEMPLATE_DIR = pkg_resources.resource_filename(
  51         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  52     SUPPORT_DIR = pkg_resources.resource_filename(
  53         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  54     USERS_AMOUNT = 2
  55     TENANTS_AMOUNT = 3
  56     ITERATIONS_AMOUNT = 10
  57     CONCURRENCY = 4
  58     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
  59     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  60     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  61
  62     visibility = 'public'
  63     shared_network = True
  64
  65     def __init__(self, **kwargs):
  66         """Initialize RallyBase object."""
  67         super(RallyBase, self).__init__(**kwargs)
  68         assert self.orig_cloud
  69         assert self.project
  70         if self.orig_cloud.get_role("admin"):
  71             role_name = "admin"
  72         elif self.orig_cloud.get_role("Admin"):
  73             role_name = "Admin"
  74         else:
  75             raise Exception("Cannot detect neither admin nor Admin")
  76         self.orig_cloud.grant_role(
  77             role_name, user=self.project.user.id,
  78             project=self.project.project.id,
  79             domain=self.project.domain.id)
  80         self.results_dir = os.path.join(
  81             getattr(config.CONF, 'dir_results'), self.case_name)
  82         self.task_file = ''
  83         self.creators = []
  84         self.summary = []
  85         self.scenario_dir = ''
  86         self.smoke = None
  87         self.test_name = None
  88         self.start_time = None
  89         self.result = None
  90         self.details = None
  91         self.compute_cnt = 0
  92         self.flavor_alt = None
  93         self.tests = []
  94         self.run_cmd = ''
  95         self.network_extensions = []
  96
  97     def _build_task_args(self, test_file_name):
  98         """Build arguments for the Rally task."""
  99         task_args = {'service_list': [test_file_name]}
 100         task_args['image_name'] = str(self.image.name)
 101         task_args['flavor_name'] = str(self.flavor.name)
 102         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 103         task_args['glance_image_location'] = str(self.filename)
 104         task_args['glance_image_format'] = str(self.image_format)
 105         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 106         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 107         task_args['users_amount'] = self.USERS_AMOUNT
 108         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 109         task_args['use_existing_users'] = False
 110         task_args['iterations'] = self.ITERATIONS_AMOUNT
 111         task_args['concurrency'] = self.CONCURRENCY
 112         task_args['smoke'] = self.smoke
 113
 114         if self.ext_net:
 115             task_args['floating_network'] = str(self.ext_net.name)
 116         else:
 117             task_args['floating_network'] = ''
 118
 119         if self.network:
 120             task_args['netid'] = str(self.network.id)
 121         else:
 122             task_args['netid'] = ''
 123
 124         return task_args
 125
 126     def _prepare_test_list(self, test_name):
 127         """Build the list of test cases to be executed."""
 128         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 129         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 130                                           test_yaml_file_name)
 131
 132         if not os.path.exists(scenario_file_name):
 133             scenario_file_name = os.path.join(self.scenario_dir,
 134                                               test_yaml_file_name)
 135
 136             if not os.path.exists(scenario_file_name):
 137                 raise Exception("The scenario '%s' does not exist."
 138                                 % scenario_file_name)
 139
 140         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 141         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 142
 143         if not os.path.exists(self.TEMP_DIR):
 144             os.makedirs(self.TEMP_DIR)
 145
 146         self.apply_blacklist(scenario_file_name, test_file_name)
 147         return test_file_name
 148
 149     @staticmethod
 150     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 151         """Set keystone_default_role in rally.conf"""
 152         if env.get("NEW_USER_ROLE").lower() != "member":
 153             rconfig = configparser.RawConfigParser()
 154             rconfig.read(rally_conf)
 155             if not rconfig.has_section('openstack'):
 156                 rconfig.add_section('openstack')
 157             rconfig.set(
 158                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 159             with open(rally_conf, 'wb') as config_file:
 160                 rconfig.write(config_file)
 161
 162     @staticmethod
 163     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 164         """Clean Rally config"""
 165         if env.get("NEW_USER_ROLE").lower() != "member":
 166             rconfig = configparser.RawConfigParser()
 167             rconfig.read(rally_conf)
 168             if rconfig.has_option('openstack', 'keystone_default_role'):
 169                 rconfig.remove_option('openstack', 'keystone_default_role')
 170             with open(rally_conf, 'wb') as config_file:
 171                 rconfig.write(config_file)
 172
 173     @staticmethod
 174     def get_task_id(cmd_raw):
 175         """
 176         Get task id from command rally result.
 177
 178         :param cmd_raw:
 179         :return: task_id as string
 180         """
 181         taskid_re = re.compile('^Task +(.*): started$')
 182         for line in cmd_raw.splitlines(True):
 183             line = line.strip()
 184             match = taskid_re.match(line)
 185             if match:
 186                 return match.group(1)
 187         return None
 188
 189     @staticmethod
 190     def task_succeed(json_raw):
 191         """
 192         Parse JSON from rally JSON results.
 193
 194         :param json_raw:
 195         :return: Bool
 196         """
 197         rally_report = json.loads(json_raw)
 198         tasks = rally_report.get('tasks')
 199         if tasks:
 200             for task in tasks:
 201                 if task.get('status') != 'finished' or \
 202                    task.get('pass_sla') is not True:
 203                     return False
 204         else:
 205             return False
 206         return True
 207
 208     def _migration_supported(self):
 209         """Determine if migration is supported."""
 210         if self.compute_cnt > 1:
 211             return True
 212         return False
 213
 214     def _network_trunk_supported(self):
 215         """Determine if network trunk service is available"""
 216         if 'trunk' in self.network_extensions:
 217             return True
 218         return False
 219
 220     @staticmethod
 221     def excl_scenario():
 222         """Exclude scenario."""
 223         black_tests = []
 224         try:
 225             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 226                 black_list_yaml = yaml.safe_load(black_list_file)
 227
 228             deploy_scenario = env.get('DEPLOY_SCENARIO')
 229             if (bool(deploy_scenario) and
 230                     'scenario' in black_list_yaml.keys()):
 231                 for item in black_list_yaml['scenario']:
 232                     scenarios = item['scenarios']
 233                     in_it = RallyBase.in_iterable_re
 234                     if in_it(deploy_scenario, scenarios):
 235                         tests = item['tests']
 236                         black_tests.extend(tests)
 237         except Exception:  # pylint: disable=broad-except
 238             LOGGER.debug("Scenario exclusion not applied.")
 239
 240         return black_tests
 241
 242     @staticmethod
 243     def in_iterable_re(needle, haystack):
 244         """
 245         Check if given needle is in the iterable haystack, using regex.
 246
 247         :param needle: string to be matched
 248         :param haystack: iterable of strings (optionally regex patterns)
 249         :return: True if needle is eqial to any of the elements in haystack,
 250                  or if a nonempty regex pattern in haystack is found in needle.
 251         """
 252         # match without regex
 253         if needle in haystack:
 254             return True
 255
 256         for pattern in haystack:
 257             # match if regex pattern is set and found in the needle
 258             if pattern and re.search(pattern, needle) is not None:
 259                 return True
 260
 261         return False
 262
 263     def excl_func(self):
 264         """Exclude functionalities."""
 265         black_tests = []
 266         func_list = []
 267
 268         try:
 269             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 270                 black_list_yaml = yaml.safe_load(black_list_file)
 271
 272             if not self._migration_supported():
 273                 func_list.append("no_migration")
 274             if not self._network_trunk_supported():
 275                 func_list.append("no_net_trunk_service")
 276
 277             if 'functionality' in black_list_yaml.keys():
 278                 for item in black_list_yaml['functionality']:
 279                     functions = item['functions']
 280                     for func in func_list:
 281                         if func in functions:
 282                             tests = item['tests']
 283                             black_tests.extend(tests)
 284         except Exception:  # pylint: disable=broad-except
 285             LOGGER.debug("Functionality exclusion not applied.")
 286
 287         return black_tests
 288
 289     def apply_blacklist(self, case_file_name, result_file_name):
 290         """Apply blacklist."""
 291         LOGGER.debug("Applying blacklist...")
 292         cases_file = open(case_file_name, 'r')
 293         result_file = open(result_file_name, 'w')
 294
 295         black_tests = list(set(self.excl_func() +
 296                                self.excl_scenario()))
 297
 298         if black_tests:
 299             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 300
 301         include = True
 302         for cases_line in cases_file:
 303             if include:
 304                 for black_tests_line in black_tests:
 305                     if re.search(black_tests_line,
 306                                  cases_line.strip().rstrip(':')):
 307                         include = False
 308                         break
 309                 else:
 310                     result_file.write(str(cases_line))
 311             else:
 312                 if cases_line.isspace():
 313                     include = True
 314
 315         cases_file.close()
 316         result_file.close()
 317
 318     @staticmethod
 319     def file_is_empty(file_name):
 320         """Determine is a file is empty."""
 321         try:
 322             if os.stat(file_name).st_size > 0:
 323                 return False
 324         except Exception:  # pylint: disable=broad-except
 325             pass
 326
 327         return True
 328
 329     def _save_results(self, test_name, task_id):
 330         """ Generate and save task execution results"""
 331         # check for result directory and create it otherwise
 332         if not os.path.exists(self.results_dir):
 333             LOGGER.debug('%s does not exist, we create it.',
 334                          self.results_dir)
 335             os.makedirs(self.results_dir)
 336
 337         # put detailed result to log
 338         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 339         LOGGER.debug('running command: %s', cmd)
 340         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 341         LOGGER.info("%s\n%s", " ".join(cmd), output)
 342
 343         # save report as JSON
 344         report_json_name = '{}.json'.format(test_name)
 345         report_json_dir = os.path.join(self.results_dir, report_json_name)
 346         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 347                 "--out", report_json_dir])
 348         LOGGER.debug('running command: %s', cmd)
 349         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 350         LOGGER.info("%s\n%s", " ".join(cmd), output)
 351
 352         # save report as HTML
 353         report_html_name = '{}.html'.format(test_name)
 354         report_html_dir = os.path.join(self.results_dir, report_html_name)
 355         cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
 356                 "--out", report_html_dir])
 357         LOGGER.debug('running command: %s', cmd)
 358         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 359         LOGGER.info("%s\n%s", " ".join(cmd), output)
 360
 361         json_results = open(report_json_dir).read()
 362         self._append_summary(json_results, test_name)
 363
 364         # parse JSON operation result
 365         if self.task_succeed(json_results):
 366             LOGGER.info('Test scenario: "%s" OK.', test_name)
 367         else:
 368             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 369
 370     def run_task(self, test_name):
 371         """Run a task."""
 372         LOGGER.info('Starting test scenario "%s" ...', test_name)
 373         LOGGER.debug('running command: %s', self.run_cmd)
 374         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 375                                 stderr=subprocess.STDOUT)
 376         output = proc.communicate()[0]
 377
 378         task_id = self.get_task_id(output)
 379         LOGGER.debug('task_id : %s', task_id)
 380         if task_id is None:
 381             LOGGER.error("Failed to retrieve task_id")
 382             LOGGER.error("Result:\n%s", output)
 383             raise Exception("Failed to retrieve task id")
 384         self._save_results(test_name, task_id)
 385
 386     def _append_summary(self, json_raw, test_name):
 387         """Update statistics summary info."""
 388         nb_tests = 0
 389         nb_success = 0
 390         overall_duration = 0.0
 391
 392         rally_report = json.loads(json_raw)
 393         for task in rally_report.get('tasks'):
 394             for subtask in task.get('subtasks'):
 395                 for workload in subtask.get('workloads'):
 396                     if workload.get('full_duration'):
 397                         overall_duration += workload.get('full_duration')
 398
 399                     if workload.get('data'):
 400                         nb_tests += len(workload.get('data'))
 401
 402                     for result in workload.get('data'):
 403                         if not result.get('error'):
 404                             nb_success += 1
 405
 406         scenario_summary = {'test_name': test_name,
 407                             'overall_duration': overall_duration,
 408                             'nb_tests': nb_tests,
 409                             'nb_success': nb_success,
 410                             'task_status': self.task_succeed(json_raw)}
 411         self.summary.append(scenario_summary)
 412
 413     def prepare_run(self):
 414         """Prepare resources needed by test scenarios."""
 415         assert self.cloud
 416         LOGGER.debug('Validating the test name...')
 417         if self.test_name == 'all':
 418             self.tests = self.TESTS
 419         elif self.test_name in self.TESTS:
 420             self.tests = [self.test_name]
 421         else:
 422             raise Exception("Test name '%s' is invalid" % self.test_name)
 423
 424         if not os.path.exists(self.TASK_DIR):
 425             os.makedirs(self.TASK_DIR)
 426
 427         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 428         if not os.path.exists(task):
 429             LOGGER.error("Task file '%s' does not exist.", task)
 430             raise Exception("Task file '{}' does not exist.".
 431                             format(task))
 432         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 433         shutil.copyfile(task, self.task_file)
 434
 435         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 436         if not os.path.exists(task_macro):
 437             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 438             raise Exception("Task macro dir '{}' does not exist.".
 439                             format(task_macro))
 440         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 441         if os.path.exists(macro_dir):
 442             shutil.rmtree(macro_dir)
 443         shutil.copytree(task_macro, macro_dir)
 444
 445         self.update_keystone_default_role()
 446         self.compute_cnt = len(self.cloud.list_hypervisors())
 447         self.network_extensions = self.cloud.get_network_extensions()
 448         self.flavor_alt = self.create_flavor_alt()
 449         LOGGER.debug("flavor: %s", self.flavor_alt)
 450
 451     def prepare_task(self, test_name):
 452         """Prepare resources for test run."""
 453         file_name = self._prepare_test_list(test_name)
 454         if self.file_is_empty(file_name):
 455             LOGGER.info('No tests for scenario "%s"', test_name)
 456             return False
 457         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 458                          "--task", self.task_file, "--task-args",
 459                          str(self._build_task_args(test_name))])
 460         return True
 461
 462     def run_tests(self):
 463         """Execute tests."""
 464         for test in self.tests:
 465             if self.prepare_task(test):
 466                 self.run_task(test)
 467
 468     def _generate_report(self):
 469         """Generate test execution summary report."""
 470         total_duration = 0.0
 471         total_nb_tests = 0
 472         total_nb_success = 0
 473         nb_modules = 0
 474         payload = []
 475
 476         res_table = prettytable.PrettyTable(
 477             padding_width=2,
 478             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 479         res_table.align['Module'] = "l"
 480         res_table.align['Duration'] = "r"
 481         res_table.align['Success'] = "r"
 482
 483         # for each scenario we draw a row for the table
 484         for item in self.summary:
 485             if item['task_status'] is True:
 486                 nb_modules += 1
 487             total_duration += item['overall_duration']
 488             total_nb_tests += item['nb_tests']
 489             total_nb_success += item['nb_success']
 490             try:
 491                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 492             except ZeroDivisionError:
 493                 success_avg = 0
 494             success_str = str("{:0.2f}".format(success_avg)) + '%'
 495             duration_str = time.strftime("%H:%M:%S",
 496                                          time.gmtime(item['overall_duration']))
 497             res_table.add_row([item['test_name'], duration_str,
 498                                item['nb_tests'], success_str])
 499             payload.append({'module': item['test_name'],
 500                             'details': {'duration': item['overall_duration'],
 501                                         'nb tests': item['nb_tests'],
 502                                         'success': success_str}})
 503
 504         total_duration_str = time.strftime("%H:%M:%S",
 505                                            time.gmtime(total_duration))
 506         try:
 507             self.result = 100 * total_nb_success / total_nb_tests
 508         except ZeroDivisionError:
 509             self.result = 100
 510         success_rate = "{:0.2f}".format(self.result)
 511         success_rate_str = str(success_rate) + '%'
 512         res_table.add_row(["", "", "", ""])
 513         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 514                            success_rate_str])
 515
 516         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 517         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 518                     self.case_name, success_rate, nb_modules,
 519                     len(self.summary))
 520         payload.append({'summary': {'duration': total_duration,
 521                                     'nb tests': total_nb_tests,
 522                                     'nb success': success_rate}})
 523         self.details = payload
 524
 525     def clean(self):
 526         """Cleanup of OpenStack resources. Should be called on completion."""
 527         self.clean_rally_conf()
 528         if self.flavor_alt:
 529             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 530         super(RallyBase, self).clean()
 531
 532     def is_successful(self):
 533         """The overall result of the test."""
 534         for item in self.summary:
 535             if item['task_status'] is False:
 536                 return testcase.TestCase.EX_TESTCASE_FAILED
 537
 538         return super(RallyBase, self).is_successful()
 539
 540     @energy.enable_recording
 541     def run(self, **kwargs):
 542         """Run testcase."""
 543         self.start_time = time.time()
 544         try:
 545             assert super(RallyBase, self).run(
 546                 **kwargs) == testcase.TestCase.EX_OK
 547             environ = dict(
 548                 os.environ,
 549                 OS_USERNAME=self.project.user.name,
 550                 OS_PROJECT_NAME=self.project.project.name,
 551                 OS_PROJECT_ID=self.project.project.id,
 552                 OS_PASSWORD=self.project.password)
 553             try:
 554                 del environ['OS_TENANT_NAME']
 555                 del environ['OS_TENANT_ID']
 556             except Exception:  # pylint: disable=broad-except
 557                 pass
 558             conf_utils.create_rally_deployment(environ=environ)
 559             self.prepare_run()
 560             self.run_tests()
 561             self._generate_report()
 562             res = testcase.TestCase.EX_OK
 563         except Exception as exc:   # pylint: disable=broad-except
 564             LOGGER.error('Error with run: %s', exc)
 565             self.result = 0
 566             res = testcase.TestCase.EX_RUN_ERROR
 567         self.stop_time = time.time()
 568         return res
 569
 570
 571 class RallySanity(RallyBase):
 572     """Rally sanity testcase implementation."""
 573
 574     def __init__(self, **kwargs):
 575         """Initialize RallySanity object."""
 576         if "case_name" not in kwargs:
 577             kwargs["case_name"] = "rally_sanity"
 578         super(RallySanity, self).__init__(**kwargs)
 579         self.test_name = 'all'
 580         self.smoke = True
 581         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 582
 583
 584 class RallyFull(RallyBase):
 585     """Rally full testcase implementation."""
 586
 587     def __init__(self, **kwargs):
 588         """Initialize RallyFull object."""
 589         if "case_name" not in kwargs:
 590             kwargs["case_name"] = "rally_full"
 591         super(RallyFull, self).__init__(**kwargs)
 592         self.test_name = 'all'
 593         self.smoke = False
 594         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 595
 596
 597 class RallyJobs(RallyBase):
 598     """Rally OpenStack CI testcase implementation."""
 599
 600     TESTS = ["neutron"]
 601
 602     def __init__(self, **kwargs):
 603         """Initialize RallyJobs object."""
 604         if "case_name" not in kwargs:
 605             kwargs["case_name"] = "rally_jobs"
 606         super(RallyJobs, self).__init__(**kwargs)
 607         self.test_name = 'all'
 608         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 609         self.task_yaml = None
 610
 611     def prepare_run(self):
 612         """Create resources needed by test scenarios."""
 613         super(RallyJobs, self).prepare_run()
 614         with open(os.path.join(self.RALLY_DIR,
 615                                'rally_jobs.yaml'), 'r') as task_file:
 616             self.task_yaml = yaml.safe_load(task_file)
 617
 618         if not all(task in self.task_yaml for task in self.tests):
 619             raise Exception("Test '%s' not in '%s'" %
 620                             (self.test_name, self.tests))
 621
 622     def apply_blacklist(self, case_file_name, result_file_name):
 623         # pylint: disable=too-many-branches
 624         """Apply blacklist."""
 625         LOGGER.debug("Applying blacklist...")
 626         black_tests = list(set(self.excl_func() +
 627                                self.excl_scenario()))
 628         if black_tests:
 629             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 630
 631         template = YAML(typ='jinja2')
 632         with open(case_file_name, 'r') as fname:
 633             cases = template.load(fname)
 634         if cases.get("version", 1) == 1:
 635             # scenarios in dictionary
 636             for name in cases.keys():
 637                 if self.in_iterable_re(name, black_tests):
 638                     cases.pop(name)
 639         else:
 640             # workloads in subtasks
 641             for sind, subtask in enumerate(cases.get('subtasks', [])):
 642                 idx = []
 643                 for wind, workload in enumerate(subtask.get('workloads', [])):
 644                     scenario = workload.get('scenario', {})
 645                     for name in scenario.keys():
 646                         if self.in_iterable_re(name, black_tests):
 647                             idx.append(wind)
 648                             break
 649                 for wind in reversed(idx):
 650                     cases['subtasks'][sind]['workloads'].pop(wind)
 651             # scenarios in subtasks
 652             idx = []
 653             for sind, subtask in enumerate(cases.get('subtasks', [])):
 654                 scenario = subtask.get('scenario', {})
 655                 for name in scenario.keys():
 656                     if self.in_iterable_re(name, black_tests):
 657                         idx.append(sind)
 658                         break
 659             for sind in reversed(idx):
 660                 cases['subtasks'].pop(sind)
 661
 662         with open(result_file_name, 'w') as fname:
 663             template.dump(cases, fname)
 664
 665     @staticmethod
 666     def _remove_plugins_extra():
 667         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 668         try:
 669             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 670             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 671         except Exception:  # pylint: disable=broad-except
 672             pass
 673
 674     def prepare_task(self, test_name):
 675         """Prepare resources for test run."""
 676         self._remove_plugins_extra()
 677         jobs_dir = os.path.join(
 678             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 679         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 680         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 681                         os.path.join(inst_dir, 'plugins'))
 682         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 683                         os.path.join(inst_dir, 'extra'))
 684
 685         task_name = self.task_yaml.get(test_name).get("task")
 686         task = os.path.join(jobs_dir, task_name)
 687         if not os.path.exists(task):
 688             raise Exception("The scenario '%s' does not exist." % task)
 689         LOGGER.debug('Scenario fetched from : %s', task)
 690
 691         if not os.path.exists(self.TEMP_DIR):
 692             os.makedirs(self.TEMP_DIR)
 693         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 694         self.apply_blacklist(task, task_file_name)
 695         self.run_cmd = (["rally", "task", "start", "--task", task_file_name])
 696         return True
 697
 698     def clean(self):
 699         self._remove_plugins_extra()
 700         super(RallyJobs, self).clean()