functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 import prettytable
  25 from ruamel.yaml import YAML
  26 from six.moves import configparser
  27 from xtesting.core import testcase
  28 import yaml
  29
  30 from functest.core import singlevm
  31 from functest.opnfv_tests.openstack.tempest import conf_utils
  32 from functest.utils import config
  33 from functest.utils import env
  34
  35 LOGGER = logging.getLogger(__name__)
  36
  37
  38 class RallyBase(singlevm.VmReady2):
  39     """Base class form Rally testcases implementation."""
  40
  41     # pylint: disable=too-many-instance-attributes
  42     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  43              'keystone', 'neutron', 'nova', 'quotas']
  44
  45     RALLY_DIR = pkg_resources.resource_filename(
  46         'functest', 'opnfv_tests/openstack/rally')
  47     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  48         'functest', 'opnfv_tests/openstack/rally/scenario')
  49     TEMPLATE_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  51     SUPPORT_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  53     USERS_AMOUNT = 2
  54     TENANTS_AMOUNT = 3
  55     ITERATIONS_AMOUNT = 10
  56     CONCURRENCY = 4
  57     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  58     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  59     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  60
  61     visibility = 'public'
  62     shared_network = True
  63
  64     def __init__(self, **kwargs):
  65         """Initialize RallyBase object."""
  66         super(RallyBase, self).__init__(**kwargs)
  67         assert self.orig_cloud
  68         assert self.project
  69         if self.orig_cloud.get_role("admin"):
  70             role_name = "admin"
  71         elif self.orig_cloud.get_role("Admin"):
  72             role_name = "Admin"
  73         else:
  74             raise Exception("Cannot detect neither admin nor Admin")
  75         self.orig_cloud.grant_role(
  76             role_name, user=self.project.user.id,
  77             project=self.project.project.id,
  78             domain=self.project.domain.id)
  79         self.results_dir = os.path.join(
  80             getattr(config.CONF, 'dir_results'), self.case_name)
  81         self.task_file = ''
  82         self.creators = []
  83         self.summary = []
  84         self.scenario_dir = ''
  85         self.smoke = None
  86         self.test_name = None
  87         self.start_time = None
  88         self.result = None
  89         self.details = None
  90         self.compute_cnt = 0
  91         self.flavor_alt = None
  92         self.tests = []
  93         self.run_cmd = ''
  94         self.network_extensions = []
  95         self.services = []
  96
  97     def build_task_args(self, test_name):
  98         """Build arguments for the Rally task."""
  99         task_args = {'service_list': [test_name]}
 100         task_args['image_name'] = str(self.image.name)
 101         task_args['flavor_name'] = str(self.flavor.name)
 102         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 103         task_args['glance_image_location'] = str(self.filename)
 104         task_args['glance_image_format'] = str(self.image_format)
 105         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 106         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 107         task_args['users_amount'] = self.USERS_AMOUNT
 108         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 109         task_args['use_existing_users'] = False
 110         task_args['iterations'] = self.ITERATIONS_AMOUNT
 111         task_args['concurrency'] = self.CONCURRENCY
 112         task_args['smoke'] = self.smoke
 113
 114         if self.ext_net:
 115             task_args['floating_network'] = str(self.ext_net.name)
 116         else:
 117             task_args['floating_network'] = ''
 118
 119         if self.network:
 120             task_args['netid'] = str(self.network.id)
 121         else:
 122             task_args['netid'] = ''
 123
 124         return task_args
 125
 126     def _prepare_test_list(self, test_name):
 127         """Build the list of test cases to be executed."""
 128         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 129         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 130                                           test_yaml_file_name)
 131
 132         if not os.path.exists(scenario_file_name):
 133             scenario_file_name = os.path.join(self.scenario_dir,
 134                                               test_yaml_file_name)
 135
 136             if not os.path.exists(scenario_file_name):
 137                 raise Exception("The scenario '%s' does not exist."
 138                                 % scenario_file_name)
 139
 140         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 141         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 142
 143         if not os.path.exists(self.TEMP_DIR):
 144             os.makedirs(self.TEMP_DIR)
 145
 146         self.apply_blacklist(scenario_file_name, test_file_name)
 147         return test_file_name
 148
 149     @staticmethod
 150     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 151         """Set keystone_default_role in rally.conf"""
 152         if env.get("NEW_USER_ROLE").lower() != "member":
 153             rconfig = configparser.RawConfigParser()
 154             rconfig.read(rally_conf)
 155             if not rconfig.has_section('openstack'):
 156                 rconfig.add_section('openstack')
 157             rconfig.set(
 158                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 159             with open(rally_conf, 'wb') as config_file:
 160                 rconfig.write(config_file)
 161
 162     @staticmethod
 163     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 164         """Clean Rally config"""
 165         if env.get("NEW_USER_ROLE").lower() != "member":
 166             rconfig = configparser.RawConfigParser()
 167             rconfig.read(rally_conf)
 168             if rconfig.has_option('openstack', 'keystone_default_role'):
 169                 rconfig.remove_option('openstack', 'keystone_default_role')
 170             with open(rally_conf, 'wb') as config_file:
 171                 rconfig.write(config_file)
 172
 173     @staticmethod
 174     def get_task_id(cmd_raw):
 175         """
 176         Get task id from command rally result.
 177
 178         :param cmd_raw:
 179         :return: task_id as string
 180         """
 181         taskid_re = re.compile('^Task +(.*): started$')
 182         for line in cmd_raw.splitlines(True):
 183             line = line.strip()
 184             match = taskid_re.match(line)
 185             if match:
 186                 return match.group(1)
 187         return None
 188
 189     @staticmethod
 190     def task_succeed(json_raw):
 191         """
 192         Parse JSON from rally JSON results.
 193
 194         :param json_raw:
 195         :return: Bool
 196         """
 197         rally_report = json.loads(json_raw)
 198         tasks = rally_report.get('tasks')
 199         if tasks:
 200             for task in tasks:
 201                 if task.get('status') != 'finished' or \
 202                    task.get('pass_sla') is not True:
 203                     return False
 204         else:
 205             return False
 206         return True
 207
 208     def _migration_supported(self):
 209         """Determine if migration is supported."""
 210         if self.compute_cnt > 1:
 211             return True
 212         return False
 213
 214     def _network_trunk_supported(self):
 215         """Determine if network trunk service is available"""
 216         if 'trunk' in self.network_extensions:
 217             return True
 218         return False
 219
 220     @staticmethod
 221     def excl_scenario():
 222         """Exclude scenario."""
 223         black_tests = []
 224         try:
 225             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 226                 black_list_yaml = yaml.safe_load(black_list_file)
 227
 228             deploy_scenario = env.get('DEPLOY_SCENARIO')
 229             if (bool(deploy_scenario) and
 230                     'scenario' in black_list_yaml.keys()):
 231                 for item in black_list_yaml['scenario']:
 232                     scenarios = item['scenarios']
 233                     in_it = RallyBase.in_iterable_re
 234                     if in_it(deploy_scenario, scenarios):
 235                         tests = item['tests']
 236                         black_tests.extend(tests)
 237         except Exception:  # pylint: disable=broad-except
 238             LOGGER.debug("Scenario exclusion not applied.")
 239
 240         return black_tests
 241
 242     @staticmethod
 243     def in_iterable_re(needle, haystack):
 244         """
 245         Check if given needle is in the iterable haystack, using regex.
 246
 247         :param needle: string to be matched
 248         :param haystack: iterable of strings (optionally regex patterns)
 249         :return: True if needle is eqial to any of the elements in haystack,
 250                  or if a nonempty regex pattern in haystack is found in needle.
 251         """
 252         # match without regex
 253         if needle in haystack:
 254             return True
 255
 256         for pattern in haystack:
 257             # match if regex pattern is set and found in the needle
 258             if pattern and re.search(pattern, needle) is not None:
 259                 return True
 260
 261         return False
 262
 263     def excl_func(self):
 264         """Exclude functionalities."""
 265         black_tests = []
 266         func_list = []
 267
 268         try:
 269             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 270                 black_list_yaml = yaml.safe_load(black_list_file)
 271
 272             if not self._migration_supported():
 273                 func_list.append("no_migration")
 274             if not self._network_trunk_supported():
 275                 func_list.append("no_net_trunk_service")
 276
 277             if 'functionality' in black_list_yaml.keys():
 278                 for item in black_list_yaml['functionality']:
 279                     functions = item['functions']
 280                     for func in func_list:
 281                         if func in functions:
 282                             tests = item['tests']
 283                             black_tests.extend(tests)
 284         except Exception:  # pylint: disable=broad-except
 285             LOGGER.debug("Functionality exclusion not applied.")
 286
 287         return black_tests
 288
 289     def apply_blacklist(self, case_file_name, result_file_name):
 290         """Apply blacklist."""
 291         LOGGER.debug("Applying blacklist...")
 292         cases_file = open(case_file_name, 'r')
 293         result_file = open(result_file_name, 'w')
 294
 295         black_tests = list(set(self.excl_func() +
 296                                self.excl_scenario()))
 297
 298         if black_tests:
 299             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 300
 301         include = True
 302         for cases_line in cases_file:
 303             if include:
 304                 for black_tests_line in black_tests:
 305                     if re.search(black_tests_line,
 306                                  cases_line.strip().rstrip(':')):
 307                         include = False
 308                         break
 309                 else:
 310                     result_file.write(str(cases_line))
 311             else:
 312                 if cases_line.isspace():
 313                     include = True
 314
 315         cases_file.close()
 316         result_file.close()
 317
 318     @staticmethod
 319     def file_is_empty(file_name):
 320         """Determine is a file is empty."""
 321         try:
 322             if os.stat(file_name).st_size > 0:
 323                 return False
 324         except Exception:  # pylint: disable=broad-except
 325             pass
 326
 327         return True
 328
 329     def _save_results(self, test_name, task_id):
 330         """ Generate and save task execution results"""
 331         # check for result directory and create it otherwise
 332         if not os.path.exists(self.results_dir):
 333             LOGGER.debug('%s does not exist, we create it.',
 334                          self.results_dir)
 335             os.makedirs(self.results_dir)
 336
 337         # put detailed result to log
 338         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 339         LOGGER.debug('running command: %s', cmd)
 340         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 341         LOGGER.info("%s\n%s", " ".join(cmd), output)
 342
 343         # save report as JSON
 344         report_json_name = '{}.json'.format(test_name)
 345         report_json_dir = os.path.join(self.results_dir, report_json_name)
 346         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 347                 "--out", report_json_dir])
 348         LOGGER.debug('running command: %s', cmd)
 349         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 350         LOGGER.info("%s\n%s", " ".join(cmd), output)
 351
 352         # save report as HTML
 353         report_html_name = '{}.html'.format(test_name)
 354         report_html_dir = os.path.join(self.results_dir, report_html_name)
 355         cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
 356                 "--out", report_html_dir])
 357         LOGGER.debug('running command: %s', cmd)
 358         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 359         LOGGER.info("%s\n%s", " ".join(cmd), output)
 360
 361         json_results = open(report_json_dir).read()
 362         self._append_summary(json_results, test_name)
 363
 364         # parse JSON operation result
 365         if self.task_succeed(json_results):
 366             LOGGER.info('Test scenario: "%s" OK.', test_name)
 367         else:
 368             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 369
 370     def run_task(self, test_name):
 371         """Run a task."""
 372         LOGGER.info('Starting test scenario "%s" ...', test_name)
 373         LOGGER.debug('running command: %s', self.run_cmd)
 374         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 375                                 stderr=subprocess.STDOUT)
 376         output = proc.communicate()[0]
 377
 378         task_id = self.get_task_id(output)
 379         LOGGER.debug('task_id : %s', task_id)
 380         if task_id is None:
 381             LOGGER.error("Failed to retrieve task_id")
 382             LOGGER.error("Result:\n%s", output)
 383             raise Exception("Failed to retrieve task id")
 384         self._save_results(test_name, task_id)
 385
 386     def _append_summary(self, json_raw, test_name):
 387         """Update statistics summary info."""
 388         nb_tests = 0
 389         nb_success = 0
 390         overall_duration = 0.0
 391
 392         rally_report = json.loads(json_raw)
 393         for task in rally_report.get('tasks'):
 394             for subtask in task.get('subtasks'):
 395                 for workload in subtask.get('workloads'):
 396                     if workload.get('full_duration'):
 397                         overall_duration += workload.get('full_duration')
 398
 399                     if workload.get('data'):
 400                         nb_tests += len(workload.get('data'))
 401
 402                     for result in workload.get('data'):
 403                         if not result.get('error'):
 404                             nb_success += 1
 405
 406         scenario_summary = {'test_name': test_name,
 407                             'overall_duration': overall_duration,
 408                             'nb_tests': nb_tests,
 409                             'nb_success': nb_success,
 410                             'task_status': self.task_succeed(json_raw)}
 411         self.summary.append(scenario_summary)
 412
 413     def prepare_run(self):
 414         """Prepare resources needed by test scenarios."""
 415         assert self.cloud
 416         LOGGER.debug('Validating the test name...')
 417         if self.test_name == 'all':
 418             self.tests = self.TESTS
 419         elif self.test_name in self.TESTS:
 420             self.tests = [self.test_name]
 421         else:
 422             raise Exception("Test name '%s' is invalid" % self.test_name)
 423
 424         if not os.path.exists(self.TASK_DIR):
 425             os.makedirs(self.TASK_DIR)
 426
 427         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 428         if not os.path.exists(task):
 429             LOGGER.error("Task file '%s' does not exist.", task)
 430             raise Exception("Task file '{}' does not exist.".
 431                             format(task))
 432         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 433         shutil.copyfile(task, self.task_file)
 434
 435         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 436         if not os.path.exists(task_macro):
 437             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 438             raise Exception("Task macro dir '{}' does not exist.".
 439                             format(task_macro))
 440         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 441         if os.path.exists(macro_dir):
 442             shutil.rmtree(macro_dir)
 443         shutil.copytree(task_macro, macro_dir)
 444
 445         self.update_keystone_default_role()
 446         self.compute_cnt = len(self.cloud.list_hypervisors())
 447         self.network_extensions = self.cloud.get_network_extensions()
 448         self.flavor_alt = self.create_flavor_alt()
 449         self.services = [service.name for service in
 450                          self.cloud.list_services()]
 451
 452         LOGGER.debug("flavor: %s", self.flavor_alt)
 453
 454     def prepare_task(self, test_name):
 455         """Prepare resources for test run."""
 456         file_name = self._prepare_test_list(test_name)
 457         if self.file_is_empty(file_name):
 458             LOGGER.info('No tests for scenario "%s"', test_name)
 459             return False
 460         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 461                          "--task", self.task_file, "--task-args",
 462                          str(self.build_task_args(test_name))])
 463         return True
 464
 465     def run_tests(self, **kwargs):
 466         """Execute tests."""
 467         optional = kwargs.get('optional', [])
 468         for test in self.tests:
 469             if test in self.services or test not in optional:
 470                 if self.prepare_task(test):
 471                     self.run_task(test)
 472
 473     def _generate_report(self):
 474         """Generate test execution summary report."""
 475         total_duration = 0.0
 476         total_nb_tests = 0
 477         total_nb_success = 0
 478         nb_modules = 0
 479         payload = []
 480
 481         res_table = prettytable.PrettyTable(
 482             padding_width=2,
 483             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 484         res_table.align['Module'] = "l"
 485         res_table.align['Duration'] = "r"
 486         res_table.align['Success'] = "r"
 487
 488         # for each scenario we draw a row for the table
 489         for item in self.summary:
 490             if item['task_status'] is True:
 491                 nb_modules += 1
 492             total_duration += item['overall_duration']
 493             total_nb_tests += item['nb_tests']
 494             total_nb_success += item['nb_success']
 495             try:
 496                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 497             except ZeroDivisionError:
 498                 success_avg = 0
 499             success_str = str("{:0.2f}".format(success_avg)) + '%'
 500             duration_str = time.strftime("%H:%M:%S",
 501                                          time.gmtime(item['overall_duration']))
 502             res_table.add_row([item['test_name'], duration_str,
 503                                item['nb_tests'], success_str])
 504             payload.append({'module': item['test_name'],
 505                             'details': {'duration': item['overall_duration'],
 506                                         'nb tests': item['nb_tests'],
 507                                         'success': success_str}})
 508
 509         total_duration_str = time.strftime("%H:%M:%S",
 510                                            time.gmtime(total_duration))
 511         try:
 512             self.result = 100 * total_nb_success / total_nb_tests
 513         except ZeroDivisionError:
 514             self.result = 100
 515         success_rate = "{:0.2f}".format(self.result)
 516         success_rate_str = str(success_rate) + '%'
 517         res_table.add_row(["", "", "", ""])
 518         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 519                            success_rate_str])
 520
 521         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 522         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 523                     self.case_name, success_rate, nb_modules,
 524                     len(self.summary))
 525         payload.append({'summary': {'duration': total_duration,
 526                                     'nb tests': total_nb_tests,
 527                                     'nb success': success_rate}})
 528         self.details = payload
 529
 530     def clean(self):
 531         """Cleanup of OpenStack resources. Should be called on completion."""
 532         self.clean_rally_conf()
 533         if self.flavor_alt:
 534             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 535         super(RallyBase, self).clean()
 536
 537     def is_successful(self):
 538         """The overall result of the test."""
 539         for item in self.summary:
 540             if item['task_status'] is False:
 541                 return testcase.TestCase.EX_TESTCASE_FAILED
 542
 543         return super(RallyBase, self).is_successful()
 544
 545     def run(self, **kwargs):
 546         """Run testcase."""
 547         self.start_time = time.time()
 548         try:
 549             assert super(RallyBase, self).run(
 550                 **kwargs) == testcase.TestCase.EX_OK
 551             environ = dict(
 552                 os.environ,
 553                 OS_USERNAME=self.project.user.name,
 554                 OS_PROJECT_NAME=self.project.project.name,
 555                 OS_PROJECT_ID=self.project.project.id,
 556                 OS_PASSWORD=self.project.password)
 557             try:
 558                 del environ['OS_TENANT_NAME']
 559                 del environ['OS_TENANT_ID']
 560             except Exception:  # pylint: disable=broad-except
 561                 pass
 562             conf_utils.create_rally_deployment(environ=environ)
 563             self.prepare_run()
 564             self.run_tests(**kwargs)
 565             self._generate_report()
 566             res = testcase.TestCase.EX_OK
 567         except Exception as exc:   # pylint: disable=broad-except
 568             LOGGER.error('Error with run: %s', exc)
 569             self.result = 0
 570             res = testcase.TestCase.EX_RUN_ERROR
 571         self.stop_time = time.time()
 572         return res
 573
 574
 575 class RallySanity(RallyBase):
 576     """Rally sanity testcase implementation."""
 577
 578     def __init__(self, **kwargs):
 579         """Initialize RallySanity object."""
 580         if "case_name" not in kwargs:
 581             kwargs["case_name"] = "rally_sanity"
 582         super(RallySanity, self).__init__(**kwargs)
 583         self.test_name = 'all'
 584         self.smoke = True
 585         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 586
 587
 588 class RallyFull(RallyBase):
 589     """Rally full testcase implementation."""
 590
 591     def __init__(self, **kwargs):
 592         """Initialize RallyFull object."""
 593         if "case_name" not in kwargs:
 594             kwargs["case_name"] = "rally_full"
 595         super(RallyFull, self).__init__(**kwargs)
 596         self.test_name = 'all'
 597         self.smoke = False
 598         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 599
 600
 601 class RallyJobs(RallyBase):
 602     """Rally OpenStack CI testcase implementation."""
 603
 604     TESTS = ["neutron"]
 605
 606     def __init__(self, **kwargs):
 607         """Initialize RallyJobs object."""
 608         if "case_name" not in kwargs:
 609             kwargs["case_name"] = "rally_jobs"
 610         super(RallyJobs, self).__init__(**kwargs)
 611         self.test_name = 'all'
 612         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 613         self.task_yaml = None
 614
 615     def prepare_run(self):
 616         """Create resources needed by test scenarios."""
 617         super(RallyJobs, self).prepare_run()
 618         with open(os.path.join(self.RALLY_DIR,
 619                                'rally_jobs.yaml'), 'r') as task_file:
 620             self.task_yaml = yaml.safe_load(task_file)
 621
 622         if not all(task in self.task_yaml for task in self.tests):
 623             raise Exception("Test '%s' not in '%s'" %
 624                             (self.test_name, self.tests))
 625
 626     def apply_blacklist(self, case_file_name, result_file_name):
 627         # pylint: disable=too-many-branches
 628         """Apply blacklist."""
 629         LOGGER.debug("Applying blacklist...")
 630         black_tests = list(set(self.excl_func() +
 631                                self.excl_scenario()))
 632         if black_tests:
 633             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 634
 635         template = YAML(typ='jinja2')
 636         with open(case_file_name, 'r') as fname:
 637             cases = template.load(fname)
 638         if cases.get("version", 1) == 1:
 639             # scenarios in dictionary
 640             for name in cases.keys():
 641                 if self.in_iterable_re(name, black_tests):
 642                     cases.pop(name)
 643         else:
 644             # workloads in subtasks
 645             for sind, subtask in enumerate(cases.get('subtasks', [])):
 646                 idx = []
 647                 for wind, workload in enumerate(subtask.get('workloads', [])):
 648                     scenario = workload.get('scenario', {})
 649                     for name in scenario.keys():
 650                         if self.in_iterable_re(name, black_tests):
 651                             idx.append(wind)
 652                             break
 653                 for wind in reversed(idx):
 654                     cases['subtasks'][sind]['workloads'].pop(wind)
 655             # scenarios in subtasks
 656             idx = []
 657             for sind, subtask in enumerate(cases.get('subtasks', [])):
 658                 scenario = subtask.get('scenario', {})
 659                 for name in scenario.keys():
 660                     if self.in_iterable_re(name, black_tests):
 661                         idx.append(sind)
 662                         break
 663             for sind in reversed(idx):
 664                 cases['subtasks'].pop(sind)
 665
 666         with open(result_file_name, 'w') as fname:
 667             template.dump(cases, fname)
 668
 669     def build_task_args(self, test_name):
 670         """Build arguments for the Rally task."""
 671         task_args = {}
 672         if self.ext_net:
 673             task_args['floating_network'] = str(self.ext_net.name)
 674         else:
 675             task_args['floating_network'] = ''
 676         return task_args
 677
 678     @staticmethod
 679     def _remove_plugins_extra():
 680         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 681         try:
 682             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 683             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 684         except Exception:  # pylint: disable=broad-except
 685             pass
 686
 687     def prepare_task(self, test_name):
 688         """Prepare resources for test run."""
 689         self._remove_plugins_extra()
 690         jobs_dir = os.path.join(
 691             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 692         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 693         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 694                         os.path.join(inst_dir, 'plugins'))
 695         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 696                         os.path.join(inst_dir, 'extra'))
 697
 698         task_name = self.task_yaml.get(test_name).get("task")
 699         task = os.path.join(jobs_dir, task_name)
 700         if not os.path.exists(task):
 701             raise Exception("The scenario '%s' does not exist." % task)
 702         LOGGER.debug('Scenario fetched from : %s', task)
 703
 704         if not os.path.exists(self.TEMP_DIR):
 705             os.makedirs(self.TEMP_DIR)
 706         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 707         self.apply_blacklist(task, task_file_name)
 708         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 709                          "--task-args", str(self.build_task_args(test_name))])
 710         return True
 711
 712     def clean(self):
 713         self._remove_plugins_extra()
 714         super(RallyJobs, self).clean()