functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import time
  22
  23 import pkg_resources
  24 import prettytable
  25 from ruamel.yaml import YAML
  26 from six.moves import configparser
  27 from xtesting.core import testcase
  28 from xtesting.energy import energy
  29 import yaml
  30
  31 from functest.core import singlevm
  32 from functest.opnfv_tests.openstack.tempest import conf_utils
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_DIR = pkg_resources.resource_filename(
  47         'functest', 'opnfv_tests/openstack/rally')
  48     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  49         'functest', 'opnfv_tests/openstack/rally/scenario')
  50     TEMPLATE_DIR = pkg_resources.resource_filename(
  51         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  52     SUPPORT_DIR = pkg_resources.resource_filename(
  53         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  54     USERS_AMOUNT = 2
  55     TENANTS_AMOUNT = 3
  56     ITERATIONS_AMOUNT = 10
  57     CONCURRENCY = 4
  58     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
  59     TEMP_DIR = os.path.join(RALLY_DIR, "var")
  60
  61     visibility = 'public'
  62     shared_network = True
  63
  64     def __init__(self, **kwargs):
  65         """Initialize RallyBase object."""
  66         super(RallyBase, self).__init__(**kwargs)
  67         assert self.orig_cloud
  68         assert self.project
  69         if self.orig_cloud.get_role("admin"):
  70             role_name = "admin"
  71         elif self.orig_cloud.get_role("Admin"):
  72             role_name = "Admin"
  73         else:
  74             raise Exception("Cannot detect neither admin nor Admin")
  75         self.orig_cloud.grant_role(
  76             role_name, user=self.project.user.id,
  77             project=self.project.project.id,
  78             domain=self.project.domain.id)
  79         self.results_dir = os.path.join(
  80             getattr(config.CONF, 'dir_results'), self.case_name)
  81         self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
  82         self.creators = []
  83         self.summary = []
  84         self.scenario_dir = ''
  85         self.smoke = None
  86         self.test_name = None
  87         self.start_time = None
  88         self.result = None
  89         self.details = None
  90         self.compute_cnt = 0
  91         self.flavor_alt = None
  92         self.tests = []
  93         self.run_cmd = ''
  94         self.network_extensions = []
  95
  96     def _build_task_args(self, test_file_name):
  97         """Build arguments for the Rally task."""
  98         task_args = {'service_list': [test_file_name]}
  99         task_args['image_name'] = str(self.image.name)
 100         task_args['flavor_name'] = str(self.flavor.name)
 101         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 102         task_args['glance_image_location'] = str(self.filename)
 103         task_args['glance_image_format'] = str(self.image_format)
 104         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 105         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 106         task_args['users_amount'] = self.USERS_AMOUNT
 107         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 108         task_args['use_existing_users'] = False
 109         task_args['iterations'] = self.ITERATIONS_AMOUNT
 110         task_args['concurrency'] = self.CONCURRENCY
 111         task_args['smoke'] = self.smoke
 112
 113         if self.ext_net:
 114             task_args['floating_network'] = str(self.ext_net.name)
 115         else:
 116             task_args['floating_network'] = ''
 117
 118         if self.network:
 119             task_args['netid'] = str(self.network.id)
 120         else:
 121             task_args['netid'] = ''
 122
 123         return task_args
 124
 125     def _prepare_test_list(self, test_name):
 126         """Build the list of test cases to be executed."""
 127         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 128         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 129                                           test_yaml_file_name)
 130
 131         if not os.path.exists(scenario_file_name):
 132             scenario_file_name = os.path.join(self.scenario_dir,
 133                                               test_yaml_file_name)
 134
 135             if not os.path.exists(scenario_file_name):
 136                 raise Exception("The scenario '%s' does not exist."
 137                                 % scenario_file_name)
 138
 139         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 140         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 141
 142         if not os.path.exists(self.TEMP_DIR):
 143             os.makedirs(self.TEMP_DIR)
 144
 145         self.apply_blacklist(scenario_file_name, test_file_name)
 146         return test_file_name
 147
 148     @staticmethod
 149     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 150         """Set keystone_default_role in rally.conf"""
 151         if env.get("NEW_USER_ROLE").lower() != "member":
 152             rconfig = configparser.RawConfigParser()
 153             rconfig.read(rally_conf)
 154             if not rconfig.has_section('openstack'):
 155                 rconfig.add_section('openstack')
 156             rconfig.set(
 157                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 158             with open(rally_conf, 'wb') as config_file:
 159                 rconfig.write(config_file)
 160
 161     @staticmethod
 162     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 163         """Clean Rally config"""
 164         if env.get("NEW_USER_ROLE").lower() != "member":
 165             rconfig = configparser.RawConfigParser()
 166             rconfig.read(rally_conf)
 167             if rconfig.has_option('openstack', 'keystone_default_role'):
 168                 rconfig.remove_option('openstack', 'keystone_default_role')
 169             with open(rally_conf, 'wb') as config_file:
 170                 rconfig.write(config_file)
 171
 172     @staticmethod
 173     def get_task_id(cmd_raw):
 174         """
 175         Get task id from command rally result.
 176
 177         :param cmd_raw:
 178         :return: task_id as string
 179         """
 180         taskid_re = re.compile('^Task +(.*): started$')
 181         for line in cmd_raw.splitlines(True):
 182             line = line.strip()
 183             match = taskid_re.match(line)
 184             if match:
 185                 return match.group(1)
 186         return None
 187
 188     @staticmethod
 189     def task_succeed(json_raw):
 190         """
 191         Parse JSON from rally JSON results.
 192
 193         :param json_raw:
 194         :return: Bool
 195         """
 196         rally_report = json.loads(json_raw)
 197         tasks = rally_report.get('tasks')
 198         if tasks:
 199             for task in tasks:
 200                 if task.get('status') != 'finished' or \
 201                    task.get('pass_sla') is not True:
 202                     return False
 203         else:
 204             return False
 205         return True
 206
 207     def _migration_supported(self):
 208         """Determine if migration is supported."""
 209         if self.compute_cnt > 1:
 210             return True
 211         return False
 212
 213     def _network_trunk_supported(self):
 214         """Determine if network trunk service is available"""
 215         if 'trunk' in self.network_extensions:
 216             return True
 217         return False
 218
 219     @staticmethod
 220     def excl_scenario():
 221         """Exclude scenario."""
 222         black_tests = []
 223         try:
 224             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 225                 black_list_yaml = yaml.safe_load(black_list_file)
 226
 227             deploy_scenario = env.get('DEPLOY_SCENARIO')
 228             if (bool(deploy_scenario) and
 229                     'scenario' in black_list_yaml.keys()):
 230                 for item in black_list_yaml['scenario']:
 231                     scenarios = item['scenarios']
 232                     in_it = RallyBase.in_iterable_re
 233                     if in_it(deploy_scenario, scenarios):
 234                         tests = item['tests']
 235                         black_tests.extend(tests)
 236         except Exception:  # pylint: disable=broad-except
 237             LOGGER.debug("Scenario exclusion not applied.")
 238
 239         return black_tests
 240
 241     @staticmethod
 242     def in_iterable_re(needle, haystack):
 243         """
 244         Check if given needle is in the iterable haystack, using regex.
 245
 246         :param needle: string to be matched
 247         :param haystack: iterable of strings (optionally regex patterns)
 248         :return: True if needle is eqial to any of the elements in haystack,
 249                  or if a nonempty regex pattern in haystack is found in needle.
 250         """
 251         # match without regex
 252         if needle in haystack:
 253             return True
 254
 255         for pattern in haystack:
 256             # match if regex pattern is set and found in the needle
 257             if pattern and re.search(pattern, needle) is not None:
 258                 return True
 259
 260         return False
 261
 262     def excl_func(self):
 263         """Exclude functionalities."""
 264         black_tests = []
 265         func_list = []
 266
 267         try:
 268             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 269                 black_list_yaml = yaml.safe_load(black_list_file)
 270
 271             if not self._migration_supported():
 272                 func_list.append("no_migration")
 273             if not self._network_trunk_supported():
 274                 func_list.append("no_net_trunk_service")
 275
 276             if 'functionality' in black_list_yaml.keys():
 277                 for item in black_list_yaml['functionality']:
 278                     functions = item['functions']
 279                     for func in func_list:
 280                         if func in functions:
 281                             tests = item['tests']
 282                             black_tests.extend(tests)
 283         except Exception:  # pylint: disable=broad-except
 284             LOGGER.debug("Functionality exclusion not applied.")
 285
 286         return black_tests
 287
 288     def apply_blacklist(self, case_file_name, result_file_name):
 289         """Apply blacklist."""
 290         LOGGER.debug("Applying blacklist...")
 291         cases_file = open(case_file_name, 'r')
 292         result_file = open(result_file_name, 'w')
 293
 294         black_tests = list(set(self.excl_func() +
 295                                self.excl_scenario()))
 296
 297         if black_tests:
 298             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 299
 300         include = True
 301         for cases_line in cases_file:
 302             if include:
 303                 for black_tests_line in black_tests:
 304                     if re.search(black_tests_line,
 305                                  cases_line.strip().rstrip(':')):
 306                         include = False
 307                         break
 308                 else:
 309                     result_file.write(str(cases_line))
 310             else:
 311                 if cases_line.isspace():
 312                     include = True
 313
 314         cases_file.close()
 315         result_file.close()
 316
 317     @staticmethod
 318     def file_is_empty(file_name):
 319         """Determine is a file is empty."""
 320         try:
 321             if os.stat(file_name).st_size > 0:
 322                 return False
 323         except Exception:  # pylint: disable=broad-except
 324             pass
 325
 326         return True
 327
 328     def _save_results(self, test_name, task_id):
 329         """ Generate and save task execution results"""
 330         # check for result directory and create it otherwise
 331         if not os.path.exists(self.results_dir):
 332             LOGGER.debug('%s does not exist, we create it.',
 333                          self.results_dir)
 334             os.makedirs(self.results_dir)
 335
 336         # put detailed result to log
 337         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 338         LOGGER.debug('running command: %s', cmd)
 339         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 340         LOGGER.info("%s\n%s", " ".join(cmd), output)
 341
 342         # save report as JSON
 343         report_json_name = '{}.json'.format(test_name)
 344         report_json_dir = os.path.join(self.results_dir, report_json_name)
 345         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 346                 "--out", report_json_dir])
 347         LOGGER.debug('running command: %s', cmd)
 348         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 349         LOGGER.info("%s\n%s", " ".join(cmd), output)
 350
 351         # save report as HTML
 352         report_html_name = '{}.html'.format(test_name)
 353         report_html_dir = os.path.join(self.results_dir, report_html_name)
 354         cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
 355                 "--out", report_html_dir])
 356         LOGGER.debug('running command: %s', cmd)
 357         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 358         LOGGER.info("%s\n%s", " ".join(cmd), output)
 359
 360         json_results = open(report_json_dir).read()
 361         self._append_summary(json_results, test_name)
 362
 363         # parse JSON operation result
 364         if self.task_succeed(json_results):
 365             LOGGER.info('Test scenario: "%s" OK.', test_name)
 366         else:
 367             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 368
 369     def run_task(self, test_name):
 370         """Run a task."""
 371         LOGGER.info('Starting test scenario "%s" ...', test_name)
 372         LOGGER.debug('running command: %s', self.run_cmd)
 373         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 374                                 stderr=subprocess.STDOUT)
 375         output = proc.communicate()[0]
 376
 377         task_id = self.get_task_id(output)
 378         LOGGER.debug('task_id : %s', task_id)
 379         if task_id is None:
 380             LOGGER.error("Failed to retrieve task_id")
 381             LOGGER.error("Result:\n%s", output)
 382             raise Exception("Failed to retrieve task id")
 383         self._save_results(test_name, task_id)
 384
 385     def _append_summary(self, json_raw, test_name):
 386         """Update statistics summary info."""
 387         nb_tests = 0
 388         nb_success = 0
 389         overall_duration = 0.0
 390
 391         rally_report = json.loads(json_raw)
 392         for task in rally_report.get('tasks'):
 393             for subtask in task.get('subtasks'):
 394                 for workload in subtask.get('workloads'):
 395                     if workload.get('full_duration'):
 396                         overall_duration += workload.get('full_duration')
 397
 398                     if workload.get('data'):
 399                         nb_tests += len(workload.get('data'))
 400
 401                     for result in workload.get('data'):
 402                         if not result.get('error'):
 403                             nb_success += 1
 404
 405         scenario_summary = {'test_name': test_name,
 406                             'overall_duration': overall_duration,
 407                             'nb_tests': nb_tests,
 408                             'nb_success': nb_success,
 409                             'task_status': self.task_succeed(json_raw)}
 410         self.summary.append(scenario_summary)
 411
 412     def prepare_run(self):
 413         """Prepare resources needed by test scenarios."""
 414         assert self.cloud
 415         LOGGER.debug('Validating the test name...')
 416         if self.test_name == 'all':
 417             self.tests = self.TESTS
 418         elif self.test_name in self.TESTS:
 419             self.tests = [self.test_name]
 420         else:
 421             raise Exception("Test name '%s' is invalid" % self.test_name)
 422
 423         if not os.path.exists(self.task_file):
 424             LOGGER.error("Task file '%s' does not exist.", self.task_file)
 425             raise Exception("Task file '{}' does not exist.".
 426                             format(self.task_file))
 427
 428         self.update_keystone_default_role()
 429         self.compute_cnt = len(self.cloud.list_hypervisors())
 430         self.network_extensions = self.cloud.get_network_extensions()
 431         self.flavor_alt = self.create_flavor_alt()
 432         LOGGER.debug("flavor: %s", self.flavor_alt)
 433
 434     def prepare_task(self, test_name):
 435         """Prepare resources for test run."""
 436         file_name = self._prepare_test_list(test_name)
 437         if self.file_is_empty(file_name):
 438             LOGGER.info('No tests for scenario "%s"', test_name)
 439             return False
 440         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 441                          "--task", self.task_file, "--task-args",
 442                          str(self._build_task_args(test_name))])
 443         return True
 444
 445     def run_tests(self):
 446         """Execute tests."""
 447         for test in self.tests:
 448             if self.prepare_task(test):
 449                 self.run_task(test)
 450
 451     def _generate_report(self):
 452         """Generate test execution summary report."""
 453         total_duration = 0.0
 454         total_nb_tests = 0
 455         total_nb_success = 0
 456         nb_modules = 0
 457         payload = []
 458
 459         res_table = prettytable.PrettyTable(
 460             padding_width=2,
 461             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 462         res_table.align['Module'] = "l"
 463         res_table.align['Duration'] = "r"
 464         res_table.align['Success'] = "r"
 465
 466         # for each scenario we draw a row for the table
 467         for item in self.summary:
 468             if item['task_status'] is True:
 469                 nb_modules += 1
 470             total_duration += item['overall_duration']
 471             total_nb_tests += item['nb_tests']
 472             total_nb_success += item['nb_success']
 473             try:
 474                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 475             except ZeroDivisionError:
 476                 success_avg = 0
 477             success_str = str("{:0.2f}".format(success_avg)) + '%'
 478             duration_str = time.strftime("%H:%M:%S",
 479                                          time.gmtime(item['overall_duration']))
 480             res_table.add_row([item['test_name'], duration_str,
 481                                item['nb_tests'], success_str])
 482             payload.append({'module': item['test_name'],
 483                             'details': {'duration': item['overall_duration'],
 484                                         'nb tests': item['nb_tests'],
 485                                         'success': success_str}})
 486
 487         total_duration_str = time.strftime("%H:%M:%S",
 488                                            time.gmtime(total_duration))
 489         try:
 490             self.result = 100 * total_nb_success / total_nb_tests
 491         except ZeroDivisionError:
 492             self.result = 100
 493         success_rate = "{:0.2f}".format(self.result)
 494         success_rate_str = str(success_rate) + '%'
 495         res_table.add_row(["", "", "", ""])
 496         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 497                            success_rate_str])
 498
 499         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 500         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 501                     self.case_name, success_rate, nb_modules,
 502                     len(self.summary))
 503         payload.append({'summary': {'duration': total_duration,
 504                                     'nb tests': total_nb_tests,
 505                                     'nb success': success_rate}})
 506         self.details = payload
 507
 508     def clean(self):
 509         """Cleanup of OpenStack resources. Should be called on completion."""
 510         self.clean_rally_conf()
 511         if self.flavor_alt:
 512             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 513         super(RallyBase, self).clean()
 514
 515     def is_successful(self):
 516         """The overall result of the test."""
 517         for item in self.summary:
 518             if item['task_status'] is False:
 519                 return testcase.TestCase.EX_TESTCASE_FAILED
 520
 521         return super(RallyBase, self).is_successful()
 522
 523     @energy.enable_recording
 524     def run(self, **kwargs):
 525         """Run testcase."""
 526         self.start_time = time.time()
 527         try:
 528             assert super(RallyBase, self).run(
 529                 **kwargs) == testcase.TestCase.EX_OK
 530             environ = dict(
 531                 os.environ,
 532                 OS_USERNAME=self.project.user.name,
 533                 OS_PROJECT_NAME=self.project.project.name,
 534                 OS_PROJECT_ID=self.project.project.id,
 535                 OS_PASSWORD=self.project.password)
 536             conf_utils.create_rally_deployment(environ=environ)
 537             self.prepare_run()
 538             self.run_tests()
 539             self._generate_report()
 540             res = testcase.TestCase.EX_OK
 541         except Exception as exc:   # pylint: disable=broad-except
 542             LOGGER.error('Error with run: %s', exc)
 543             self.result = 0
 544             res = testcase.TestCase.EX_RUN_ERROR
 545         self.stop_time = time.time()
 546         return res
 547
 548
 549 class RallySanity(RallyBase):
 550     """Rally sanity testcase implementation."""
 551
 552     def __init__(self, **kwargs):
 553         """Initialize RallySanity object."""
 554         if "case_name" not in kwargs:
 555             kwargs["case_name"] = "rally_sanity"
 556         super(RallySanity, self).__init__(**kwargs)
 557         self.test_name = 'all'
 558         self.smoke = True
 559         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 560
 561
 562 class RallyFull(RallyBase):
 563     """Rally full testcase implementation."""
 564
 565     def __init__(self, **kwargs):
 566         """Initialize RallyFull object."""
 567         if "case_name" not in kwargs:
 568             kwargs["case_name"] = "rally_full"
 569         super(RallyFull, self).__init__(**kwargs)
 570         self.test_name = 'all'
 571         self.smoke = False
 572         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 573
 574
 575 class RallyJobs(RallyBase):
 576     """Rally OpenStack CI testcase implementation."""
 577
 578     TESTS = ["neutron"]
 579
 580     def __init__(self, **kwargs):
 581         """Initialize RallyJobs object."""
 582         if "case_name" not in kwargs:
 583             kwargs["case_name"] = "rally_jobs"
 584         super(RallyJobs, self).__init__(**kwargs)
 585         self.test_name = 'all'
 586         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 587         self.task_yaml = None
 588
 589     def prepare_run(self):
 590         """Create resources needed by test scenarios."""
 591         super(RallyJobs, self).prepare_run()
 592         with open(os.path.join(self.RALLY_DIR,
 593                                'rally_jobs.yaml'), 'r') as task_file:
 594             self.task_yaml = yaml.safe_load(task_file)
 595
 596         if not all(task in self.task_yaml for task in self.tests):
 597             raise Exception("Test '%s' not in '%s'" %
 598                             (self.test_name, self.tests))
 599
 600     def apply_blacklist(self, case_file_name, result_file_name):
 601         # pylint: disable=too-many-branches
 602         """Apply blacklist."""
 603         LOGGER.debug("Applying blacklist...")
 604         black_tests = list(set(self.excl_func() +
 605                                self.excl_scenario()))
 606         if black_tests:
 607             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 608
 609         template = YAML(typ='jinja2')
 610         with open(case_file_name, 'r') as fname:
 611             cases = template.load(fname)
 612         if cases.get("version", 1) == 1:
 613             # scenarios in dictionary
 614             for name in cases.keys():
 615                 if self.in_iterable_re(name, black_tests):
 616                     cases.pop(name)
 617         else:
 618             # workloads in subtasks
 619             for sind, subtask in enumerate(cases.get('subtasks', [])):
 620                 idx = []
 621                 for wind, workload in enumerate(subtask.get('workloads', [])):
 622                     scenario = workload.get('scenario', {})
 623                     for name in scenario.keys():
 624                         if self.in_iterable_re(name, black_tests):
 625                             idx.append(wind)
 626                             break
 627                 for wind in reversed(idx):
 628                     cases['subtasks'][sind]['workloads'].pop(wind)
 629             # scenarios in subtasks
 630             idx = []
 631             for sind, subtask in enumerate(cases.get('subtasks', [])):
 632                 scenario = subtask.get('scenario', {})
 633                 for name in scenario.keys():
 634                     if self.in_iterable_re(name, black_tests):
 635                         idx.append(sind)
 636                         break
 637             for sind in reversed(idx):
 638                 cases['subtasks'].pop(sind)
 639
 640         with open(result_file_name, 'w') as fname:
 641             template.dump(cases, fname)
 642
 643     @staticmethod
 644     def _remove_plugins_extra():
 645         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 646         try:
 647             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 648             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 649         except Exception:  # pylint: disable=broad-except
 650             pass
 651
 652     def prepare_task(self, test_name):
 653         """Prepare resources for test run."""
 654         self._remove_plugins_extra()
 655         jobs_dir = os.path.join(
 656             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 657         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 658         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 659                         os.path.join(inst_dir, 'plugins'))
 660         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 661                         os.path.join(inst_dir, 'extra'))
 662
 663         task_name = self.task_yaml.get(test_name).get("task")
 664         task = os.path.join(jobs_dir, task_name)
 665         if not os.path.exists(task):
 666             raise Exception("The scenario '%s' does not exist." % task)
 667         LOGGER.debug('Scenario fetched from : %s', task)
 668
 669         if not os.path.exists(self.TEMP_DIR):
 670             os.makedirs(self.TEMP_DIR)
 671         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 672         self.apply_blacklist(task, task_file_name)
 673         self.run_cmd = (["rally", "task", "start", "--task", task_file_name])
 674         return True
 675
 676     def clean(self):
 677         self._remove_plugins_extra()
 678         super(RallyJobs, self).clean()