functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 import prettytable
  25 from ruamel.yaml import YAML
  26 from six.moves import configparser
  27 from xtesting.core import testcase
  28 import yaml
  29
  30 from functest.core import singlevm
  31 from functest.opnfv_tests.openstack.tempest import conf_utils
  32 from functest.utils import config
  33 from functest.utils import env
  34
  35 LOGGER = logging.getLogger(__name__)
  36
  37
  38 class RallyBase(singlevm.VmReady2):
  39     """Base class form Rally testcases implementation."""
  40
  41     # pylint: disable=too-many-instance-attributes
  42     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  43              'keystone', 'neutron', 'nova', 'quotas']
  44
  45     RALLY_DIR = pkg_resources.resource_filename(
  46         'functest', 'opnfv_tests/openstack/rally')
  47     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  48         'functest', 'opnfv_tests/openstack/rally/scenario')
  49     TEMPLATE_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  51     SUPPORT_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  53     USERS_AMOUNT = 2
  54     TENANTS_AMOUNT = 3
  55     ITERATIONS_AMOUNT = 10
  56     CONCURRENCY = 4
  57     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  58     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  59     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  60
  61     visibility = 'public'
  62     shared_network = True
  63
  64     def __init__(self, **kwargs):
  65         """Initialize RallyBase object."""
  66         super(RallyBase, self).__init__(**kwargs)
  67         assert self.orig_cloud
  68         assert self.project
  69         if self.orig_cloud.get_role("admin"):
  70             role_name = "admin"
  71         elif self.orig_cloud.get_role("Admin"):
  72             role_name = "Admin"
  73         else:
  74             raise Exception("Cannot detect neither admin nor Admin")
  75         self.orig_cloud.grant_role(
  76             role_name, user=self.project.user.id,
  77             project=self.project.project.id,
  78             domain=self.project.domain.id)
  79         self.results_dir = os.path.join(
  80             getattr(config.CONF, 'dir_results'), self.case_name)
  81         self.task_file = ''
  82         self.creators = []
  83         self.summary = []
  84         self.scenario_dir = ''
  85         self.smoke = None
  86         self.start_time = None
  87         self.result = None
  88         self.details = None
  89         self.compute_cnt = 0
  90         self.flavor_alt = None
  91         self.tests = []
  92         self.run_cmd = ''
  93         self.network_extensions = []
  94         self.services = []
  95
  96     def build_task_args(self, test_name):
  97         """Build arguments for the Rally task."""
  98         task_args = {'service_list': [test_name]}
  99         task_args['image_name'] = str(self.image.name)
 100         task_args['flavor_name'] = str(self.flavor.name)
 101         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 102         task_args['glance_image_location'] = str(self.filename)
 103         task_args['glance_image_format'] = str(self.image_format)
 104         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 105         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 106         task_args['users_amount'] = self.USERS_AMOUNT
 107         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 108         task_args['use_existing_users'] = False
 109         task_args['iterations'] = self.ITERATIONS_AMOUNT
 110         task_args['concurrency'] = self.CONCURRENCY
 111         task_args['smoke'] = self.smoke
 112
 113         if self.ext_net:
 114             task_args['floating_network'] = str(self.ext_net.name)
 115         else:
 116             task_args['floating_network'] = ''
 117
 118         if self.network:
 119             task_args['netid'] = str(self.network.id)
 120         else:
 121             task_args['netid'] = ''
 122
 123         return task_args
 124
 125     def _prepare_test_list(self, test_name):
 126         """Build the list of test cases to be executed."""
 127         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 128         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 129                                           test_yaml_file_name)
 130
 131         if not os.path.exists(scenario_file_name):
 132             scenario_file_name = os.path.join(self.scenario_dir,
 133                                               test_yaml_file_name)
 134
 135             if not os.path.exists(scenario_file_name):
 136                 raise Exception("The scenario '%s' does not exist."
 137                                 % scenario_file_name)
 138
 139         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 140         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 141
 142         if not os.path.exists(self.TEMP_DIR):
 143             os.makedirs(self.TEMP_DIR)
 144
 145         self.apply_blacklist(scenario_file_name, test_file_name)
 146         return test_file_name
 147
 148     @staticmethod
 149     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 150         """Set keystone_default_role in rally.conf"""
 151         if env.get("NEW_USER_ROLE").lower() != "member":
 152             rconfig = configparser.RawConfigParser()
 153             rconfig.read(rally_conf)
 154             if not rconfig.has_section('openstack'):
 155                 rconfig.add_section('openstack')
 156             rconfig.set(
 157                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 158             with open(rally_conf, 'wb') as config_file:
 159                 rconfig.write(config_file)
 160
 161     @staticmethod
 162     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 163         """Clean Rally config"""
 164         if env.get("NEW_USER_ROLE").lower() != "member":
 165             rconfig = configparser.RawConfigParser()
 166             rconfig.read(rally_conf)
 167             if rconfig.has_option('openstack', 'keystone_default_role'):
 168                 rconfig.remove_option('openstack', 'keystone_default_role')
 169             with open(rally_conf, 'wb') as config_file:
 170                 rconfig.write(config_file)
 171
 172     @staticmethod
 173     def get_task_id(cmd_raw):
 174         """
 175         Get task id from command rally result.
 176
 177         :param cmd_raw:
 178         :return: task_id as string
 179         """
 180         taskid_re = re.compile('^Task +(.*): started$')
 181         for line in cmd_raw.splitlines(True):
 182             line = line.strip()
 183             match = taskid_re.match(line)
 184             if match:
 185                 return match.group(1)
 186         return None
 187
 188     @staticmethod
 189     def task_succeed(json_raw):
 190         """
 191         Parse JSON from rally JSON results.
 192
 193         :param json_raw:
 194         :return: Bool
 195         """
 196         rally_report = json.loads(json_raw)
 197         tasks = rally_report.get('tasks')
 198         if tasks:
 199             for task in tasks:
 200                 if task.get('status') != 'finished' or \
 201                    task.get('pass_sla') is not True:
 202                     return False
 203         else:
 204             return False
 205         return True
 206
 207     def _migration_supported(self):
 208         """Determine if migration is supported."""
 209         if self.compute_cnt > 1:
 210             return True
 211         return False
 212
 213     def _network_trunk_supported(self):
 214         """Determine if network trunk service is available"""
 215         if 'trunk' in self.network_extensions:
 216             return True
 217         return False
 218
 219     @staticmethod
 220     def excl_scenario():
 221         """Exclude scenario."""
 222         black_tests = []
 223         try:
 224             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 225                 black_list_yaml = yaml.safe_load(black_list_file)
 226
 227             deploy_scenario = env.get('DEPLOY_SCENARIO')
 228             if (bool(deploy_scenario) and
 229                     'scenario' in black_list_yaml.keys()):
 230                 for item in black_list_yaml['scenario']:
 231                     scenarios = item['scenarios']
 232                     in_it = RallyBase.in_iterable_re
 233                     if in_it(deploy_scenario, scenarios):
 234                         tests = item['tests']
 235                         black_tests.extend(tests)
 236         except Exception:  # pylint: disable=broad-except
 237             LOGGER.debug("Scenario exclusion not applied.")
 238
 239         return black_tests
 240
 241     @staticmethod
 242     def in_iterable_re(needle, haystack):
 243         """
 244         Check if given needle is in the iterable haystack, using regex.
 245
 246         :param needle: string to be matched
 247         :param haystack: iterable of strings (optionally regex patterns)
 248         :return: True if needle is eqial to any of the elements in haystack,
 249                  or if a nonempty regex pattern in haystack is found in needle.
 250         """
 251         # match without regex
 252         if needle in haystack:
 253             return True
 254
 255         for pattern in haystack:
 256             # match if regex pattern is set and found in the needle
 257             if pattern and re.search(pattern, needle) is not None:
 258                 return True
 259
 260         return False
 261
 262     def excl_func(self):
 263         """Exclude functionalities."""
 264         black_tests = []
 265         func_list = []
 266
 267         try:
 268             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 269                 black_list_yaml = yaml.safe_load(black_list_file)
 270
 271             if not self._migration_supported():
 272                 func_list.append("no_migration")
 273             if not self._network_trunk_supported():
 274                 func_list.append("no_net_trunk_service")
 275
 276             if 'functionality' in black_list_yaml.keys():
 277                 for item in black_list_yaml['functionality']:
 278                     functions = item['functions']
 279                     for func in func_list:
 280                         if func in functions:
 281                             tests = item['tests']
 282                             black_tests.extend(tests)
 283         except Exception:  # pylint: disable=broad-except
 284             LOGGER.debug("Functionality exclusion not applied.")
 285
 286         return black_tests
 287
 288     def apply_blacklist(self, case_file_name, result_file_name):
 289         """Apply blacklist."""
 290         LOGGER.debug("Applying blacklist...")
 291         cases_file = open(case_file_name, 'r')
 292         result_file = open(result_file_name, 'w')
 293
 294         black_tests = list(set(self.excl_func() +
 295                                self.excl_scenario()))
 296
 297         if black_tests:
 298             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 299
 300         include = True
 301         for cases_line in cases_file:
 302             if include:
 303                 for black_tests_line in black_tests:
 304                     if re.search(black_tests_line,
 305                                  cases_line.strip().rstrip(':')):
 306                         include = False
 307                         break
 308                 else:
 309                     result_file.write(str(cases_line))
 310             else:
 311                 if cases_line.isspace():
 312                     include = True
 313
 314         cases_file.close()
 315         result_file.close()
 316
 317     @staticmethod
 318     def file_is_empty(file_name):
 319         """Determine is a file is empty."""
 320         try:
 321             if os.stat(file_name).st_size > 0:
 322                 return False
 323         except Exception:  # pylint: disable=broad-except
 324             pass
 325
 326         return True
 327
 328     def _save_results(self, test_name, task_id):
 329         """ Generate and save task execution results"""
 330         # check for result directory and create it otherwise
 331         if not os.path.exists(self.results_dir):
 332             LOGGER.debug('%s does not exist, we create it.',
 333                          self.results_dir)
 334             os.makedirs(self.results_dir)
 335
 336         # put detailed result to log
 337         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 338         LOGGER.debug('running command: %s', cmd)
 339         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 340         LOGGER.info("%s\n%s", " ".join(cmd), output)
 341
 342         # save report as JSON
 343         report_json_name = '{}.json'.format(test_name)
 344         report_json_dir = os.path.join(self.results_dir, report_json_name)
 345         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 346                 "--out", report_json_dir])
 347         LOGGER.debug('running command: %s', cmd)
 348         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 349         LOGGER.info("%s\n%s", " ".join(cmd), output)
 350
 351         json_results = open(report_json_dir).read()
 352         self._append_summary(json_results, test_name)
 353
 354         # parse JSON operation result
 355         if self.task_succeed(json_results):
 356             LOGGER.info('Test scenario: "%s" OK.', test_name)
 357         else:
 358             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 359
 360     def run_task(self, test_name):
 361         """Run a task."""
 362         LOGGER.info('Starting test scenario "%s" ...', test_name)
 363         LOGGER.debug('running command: %s', self.run_cmd)
 364         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 365                                 stderr=subprocess.STDOUT)
 366         output = proc.communicate()[0]
 367
 368         task_id = self.get_task_id(output)
 369         LOGGER.debug('task_id : %s', task_id)
 370         if task_id is None:
 371             LOGGER.error("Failed to retrieve task_id")
 372             LOGGER.error("Result:\n%s", output)
 373             raise Exception("Failed to retrieve task id")
 374         self._save_results(test_name, task_id)
 375
 376     def _append_summary(self, json_raw, test_name):
 377         """Update statistics summary info."""
 378         nb_tests = 0
 379         nb_success = 0
 380         overall_duration = 0.0
 381
 382         rally_report = json.loads(json_raw)
 383         for task in rally_report.get('tasks'):
 384             for subtask in task.get('subtasks'):
 385                 for workload in subtask.get('workloads'):
 386                     if workload.get('full_duration'):
 387                         overall_duration += workload.get('full_duration')
 388
 389                     if workload.get('data'):
 390                         nb_tests += len(workload.get('data'))
 391
 392                     for result in workload.get('data'):
 393                         if not result.get('error'):
 394                             nb_success += 1
 395
 396         scenario_summary = {'test_name': test_name,
 397                             'overall_duration': overall_duration,
 398                             'nb_tests': nb_tests,
 399                             'nb_success': nb_success,
 400                             'task_status': self.task_succeed(json_raw)}
 401         self.summary.append(scenario_summary)
 402
 403     def prepare_run(self, **kwargs):
 404         """Prepare resources needed by test scenarios."""
 405         assert self.cloud
 406         LOGGER.debug('Validating run tests...')
 407         for test in kwargs.get('tests', self.TESTS):
 408             if test in self.TESTS:
 409                 self.tests.append(test)
 410             else:
 411                 raise Exception("Test name '%s' is invalid" % test)
 412
 413         if not os.path.exists(self.TASK_DIR):
 414             os.makedirs(self.TASK_DIR)
 415
 416         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 417         if not os.path.exists(task):
 418             LOGGER.error("Task file '%s' does not exist.", task)
 419             raise Exception("Task file '{}' does not exist.".
 420                             format(task))
 421         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 422         shutil.copyfile(task, self.task_file)
 423
 424         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 425         if not os.path.exists(task_macro):
 426             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 427             raise Exception("Task macro dir '{}' does not exist.".
 428                             format(task_macro))
 429         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 430         if os.path.exists(macro_dir):
 431             shutil.rmtree(macro_dir)
 432         shutil.copytree(task_macro, macro_dir)
 433
 434         self.update_keystone_default_role()
 435         self.compute_cnt = len(self.cloud.list_hypervisors())
 436         self.network_extensions = self.cloud.get_network_extensions()
 437         self.flavor_alt = self.create_flavor_alt()
 438         self.services = [service.name for service in
 439                          self.cloud.list_services()]
 440
 441         LOGGER.debug("flavor: %s", self.flavor_alt)
 442
 443     def prepare_task(self, test_name):
 444         """Prepare resources for test run."""
 445         file_name = self._prepare_test_list(test_name)
 446         if self.file_is_empty(file_name):
 447             LOGGER.info('No tests for scenario "%s"', test_name)
 448             return False
 449         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 450                          "--task", self.task_file, "--task-args",
 451                          str(self.build_task_args(test_name))])
 452         return True
 453
 454     def run_tests(self, **kwargs):
 455         """Execute tests."""
 456         optional = kwargs.get('optional', [])
 457         for test in self.tests:
 458             if test in self.services or test not in optional:
 459                 if self.prepare_task(test):
 460                     self.run_task(test)
 461
 462     def _generate_report(self):
 463         """Generate test execution summary report."""
 464         total_duration = 0.0
 465         total_nb_tests = 0
 466         total_nb_success = 0
 467         nb_modules = 0
 468         payload = []
 469
 470         res_table = prettytable.PrettyTable(
 471             padding_width=2,
 472             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 473         res_table.align['Module'] = "l"
 474         res_table.align['Duration'] = "r"
 475         res_table.align['Success'] = "r"
 476
 477         # for each scenario we draw a row for the table
 478         for item in self.summary:
 479             if item['task_status'] is True:
 480                 nb_modules += 1
 481             total_duration += item['overall_duration']
 482             total_nb_tests += item['nb_tests']
 483             total_nb_success += item['nb_success']
 484             try:
 485                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 486             except ZeroDivisionError:
 487                 success_avg = 0
 488             success_str = str("{:0.2f}".format(success_avg)) + '%'
 489             duration_str = time.strftime("%H:%M:%S",
 490                                          time.gmtime(item['overall_duration']))
 491             res_table.add_row([item['test_name'], duration_str,
 492                                item['nb_tests'], success_str])
 493             payload.append({'module': item['test_name'],
 494                             'details': {'duration': item['overall_duration'],
 495                                         'nb tests': item['nb_tests'],
 496                                         'success': success_str}})
 497
 498         total_duration_str = time.strftime("%H:%M:%S",
 499                                            time.gmtime(total_duration))
 500         try:
 501             self.result = 100 * total_nb_success / total_nb_tests
 502         except ZeroDivisionError:
 503             self.result = 100
 504         success_rate = "{:0.2f}".format(self.result)
 505         success_rate_str = str(success_rate) + '%'
 506         res_table.add_row(["", "", "", ""])
 507         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 508                            success_rate_str])
 509
 510         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 511         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 512                     self.case_name, success_rate, nb_modules,
 513                     len(self.summary))
 514         payload.append({'summary': {'duration': total_duration,
 515                                     'nb tests': total_nb_tests,
 516                                     'nb success': success_rate}})
 517         self.details = payload
 518
 519     def generate_html_report(self):
 520         """Save all task reports as single HTML
 521
 522         Raises:
 523             subprocess.CalledProcessError: if Rally doesn't return 0
 524
 525         Returns:
 526             None
 527         """
 528         cmd = ["rally", "task", "report", "--deployment",
 529                str(getattr(config.CONF, 'rally_deployment_name')),
 530                "--out", "{}/{}.html".format(self.results_dir, self.case_name)]
 531         LOGGER.debug('running command: %s', cmd)
 532         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 533         LOGGER.info("%s\n%s", " ".join(cmd), output)
 534
 535     def clean(self):
 536         """Cleanup of OpenStack resources. Should be called on completion."""
 537         self.clean_rally_conf()
 538         if self.flavor_alt:
 539             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 540         super(RallyBase, self).clean()
 541
 542     def is_successful(self):
 543         """The overall result of the test."""
 544         for item in self.summary:
 545             if item['task_status'] is False:
 546                 return testcase.TestCase.EX_TESTCASE_FAILED
 547
 548         return super(RallyBase, self).is_successful()
 549
 550     def run(self, **kwargs):
 551         """Run testcase."""
 552         self.start_time = time.time()
 553         try:
 554             assert super(RallyBase, self).run(
 555                 **kwargs) == testcase.TestCase.EX_OK
 556             environ = dict(
 557                 os.environ,
 558                 OS_USERNAME=self.project.user.name,
 559                 OS_PROJECT_NAME=self.project.project.name,
 560                 OS_PROJECT_ID=self.project.project.id,
 561                 OS_PASSWORD=self.project.password)
 562             try:
 563                 del environ['OS_TENANT_NAME']
 564                 del environ['OS_TENANT_ID']
 565             except Exception:  # pylint: disable=broad-except
 566                 pass
 567             conf_utils.create_rally_deployment(environ=environ)
 568             self.prepare_run(**kwargs)
 569             self.run_tests(**kwargs)
 570             self._generate_report()
 571             self.generate_html_report()
 572             res = testcase.TestCase.EX_OK
 573         except Exception as exc:   # pylint: disable=broad-except
 574             LOGGER.error('Error with run: %s', exc)
 575             self.result = 0
 576             res = testcase.TestCase.EX_RUN_ERROR
 577         self.stop_time = time.time()
 578         return res
 579
 580
 581 class RallySanity(RallyBase):
 582     """Rally sanity testcase implementation."""
 583
 584     def __init__(self, **kwargs):
 585         """Initialize RallySanity object."""
 586         if "case_name" not in kwargs:
 587             kwargs["case_name"] = "rally_sanity"
 588         super(RallySanity, self).__init__(**kwargs)
 589         self.smoke = True
 590         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 591
 592
 593 class RallyFull(RallyBase):
 594     """Rally full testcase implementation."""
 595
 596     def __init__(self, **kwargs):
 597         """Initialize RallyFull object."""
 598         if "case_name" not in kwargs:
 599             kwargs["case_name"] = "rally_full"
 600         super(RallyFull, self).__init__(**kwargs)
 601         self.smoke = False
 602         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 603
 604
 605 class RallyJobs(RallyBase):
 606     """Rally OpenStack CI testcase implementation."""
 607
 608     TESTS = ["neutron"]
 609
 610     def __init__(self, **kwargs):
 611         """Initialize RallyJobs object."""
 612         if "case_name" not in kwargs:
 613             kwargs["case_name"] = "rally_jobs"
 614         super(RallyJobs, self).__init__(**kwargs)
 615         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 616         self.task_yaml = None
 617
 618     def prepare_run(self, **kwargs):
 619         """Create resources needed by test scenarios."""
 620         super(RallyJobs, self).prepare_run(**kwargs)
 621         with open(os.path.join(self.RALLY_DIR,
 622                                'rally_jobs.yaml'), 'r') as task_file:
 623             self.task_yaml = yaml.safe_load(task_file)
 624
 625         for task in self.task_yaml:
 626             if task not in self.tests:
 627                 raise Exception("Test '%s' not in '%s'" %
 628                                 (task, self.tests))
 629
 630     def apply_blacklist(self, case_file_name, result_file_name):
 631         # pylint: disable=too-many-branches
 632         """Apply blacklist."""
 633         LOGGER.debug("Applying blacklist...")
 634         black_tests = list(set(self.excl_func() +
 635                                self.excl_scenario()))
 636         if black_tests:
 637             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 638
 639         template = YAML(typ='jinja2')
 640         with open(case_file_name, 'r') as fname:
 641             cases = template.load(fname)
 642         if cases.get("version", 1) == 1:
 643             # scenarios in dictionary
 644             for name in cases.keys():
 645                 if self.in_iterable_re(name, black_tests):
 646                     cases.pop(name)
 647         else:
 648             # workloads in subtasks
 649             for sind, subtask in enumerate(cases.get('subtasks', [])):
 650                 idx = []
 651                 for wind, workload in enumerate(subtask.get('workloads', [])):
 652                     scenario = workload.get('scenario', {})
 653                     for name in scenario.keys():
 654                         if self.in_iterable_re(name, black_tests):
 655                             idx.append(wind)
 656                             break
 657                 for wind in reversed(idx):
 658                     cases['subtasks'][sind]['workloads'].pop(wind)
 659             # scenarios in subtasks
 660             idx = []
 661             for sind, subtask in enumerate(cases.get('subtasks', [])):
 662                 scenario = subtask.get('scenario', {})
 663                 for name in scenario.keys():
 664                     if self.in_iterable_re(name, black_tests):
 665                         idx.append(sind)
 666                         break
 667             for sind in reversed(idx):
 668                 cases['subtasks'].pop(sind)
 669
 670         with open(result_file_name, 'w') as fname:
 671             template.dump(cases, fname)
 672
 673     def build_task_args(self, test_name):
 674         """Build arguments for the Rally task."""
 675         task_args = {}
 676         if self.ext_net:
 677             task_args['floating_network'] = str(self.ext_net.name)
 678         else:
 679             task_args['floating_network'] = ''
 680         return task_args
 681
 682     @staticmethod
 683     def _remove_plugins_extra():
 684         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 685         try:
 686             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 687             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 688         except Exception:  # pylint: disable=broad-except
 689             pass
 690
 691     def prepare_task(self, test_name):
 692         """Prepare resources for test run."""
 693         self._remove_plugins_extra()
 694         jobs_dir = os.path.join(
 695             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 696         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 697         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 698                         os.path.join(inst_dir, 'plugins'))
 699         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 700                         os.path.join(inst_dir, 'extra'))
 701
 702         task_name = self.task_yaml.get(test_name).get("task")
 703         task = os.path.join(jobs_dir, task_name)
 704         if not os.path.exists(task):
 705             raise Exception("The scenario '%s' does not exist." % task)
 706         LOGGER.debug('Scenario fetched from : %s', task)
 707
 708         if not os.path.exists(self.TEMP_DIR):
 709             os.makedirs(self.TEMP_DIR)
 710         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 711         self.apply_blacklist(task, task_file_name)
 712         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 713                          "--task-args", str(self.build_task_args(test_name))])
 714         return True
 715
 716     def clean(self):
 717         self._remove_plugins_extra()
 718         super(RallyJobs, self).clean()