3 # Copyright (c) 2015 All rights reserved
4 # This program and the accompanying materials
5 # are made available under the terms of the Apache License, Version 2.0
6 # which accompanies this distribution, and is available at
8 # http://www.apache.org/licenses/LICENSE-2.0
11 """Rally testcases implementation."""
13 from __future__ import division
14 from __future__ import print_function
27 from ruamel.yaml import YAML
28 from six.moves import configparser
29 from xtesting.core import testcase
32 from functest.core import singlevm
33 from functest.utils import config
34 from functest.utils import env
36 LOGGER = logging.getLogger(__name__)
39 class RallyBase(singlevm.VmReady2):
40 """Base class form Rally testcases implementation."""
42 # pylint: disable=too-many-instance-attributes, too-many-public-methods
43 TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
44 'keystone', 'neutron', 'nova', 'quotas']
46 RALLY_CONF_PATH = "/etc/rally/rally.conf"
47 RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
48 'functest', 'ci/rally_aarch64_patch.conf')
49 RALLY_DIR = pkg_resources.resource_filename(
50 'functest', 'opnfv_tests/openstack/rally')
51 RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
52 'functest', 'opnfv_tests/openstack/rally/scenario')
53 TEMPLATE_DIR = pkg_resources.resource_filename(
54 'functest', 'opnfv_tests/openstack/rally/scenario/templates')
55 SUPPORT_DIR = pkg_resources.resource_filename(
56 'functest', 'opnfv_tests/openstack/rally/scenario/support')
59 ITERATIONS_AMOUNT = 10
61 BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
62 TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
63 TEMP_DIR = os.path.join(TASK_DIR, 'var')
68 def __init__(self, **kwargs):
69 """Initialize RallyBase object."""
70 super(RallyBase, self).__init__(**kwargs)
71 assert self.orig_cloud
73 if self.orig_cloud.get_role("admin"):
75 elif self.orig_cloud.get_role("Admin"):
78 raise Exception("Cannot detect neither admin nor Admin")
79 self.orig_cloud.grant_role(
80 role_name, user=self.project.user.id,
81 project=self.project.project.id,
82 domain=self.project.domain.id)
83 self.results_dir = os.path.join(
84 getattr(config.CONF, 'dir_results'), self.case_name)
88 self.scenario_dir = ''
90 self.start_time = None
94 self.flavor_alt = None
97 self.network_extensions = []
100 def build_task_args(self, test_name):
101 """Build arguments for the Rally task."""
102 task_args = {'service_list': [test_name]}
103 task_args['image_name'] = str(self.image.name)
104 task_args['flavor_name'] = str(self.flavor.name)
105 task_args['flavor_alt_name'] = str(self.flavor_alt.name)
106 task_args['glance_image_location'] = str(self.filename)
107 task_args['glance_image_format'] = str(self.image_format)
108 task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
109 task_args['sup_dir'] = str(self.SUPPORT_DIR)
110 task_args['users_amount'] = self.USERS_AMOUNT
111 task_args['tenants_amount'] = self.TENANTS_AMOUNT
112 task_args['use_existing_users'] = False
113 task_args['iterations'] = self.ITERATIONS_AMOUNT
114 task_args['concurrency'] = self.CONCURRENCY
115 task_args['smoke'] = self.smoke
118 task_args['floating_network'] = str(self.ext_net.name)
120 task_args['floating_network'] = ''
123 task_args['netid'] = str(self.network.id)
125 task_args['netid'] = ''
129 def _prepare_test_list(self, test_name):
130 """Build the list of test cases to be executed."""
131 test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
132 scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
135 if not os.path.exists(scenario_file_name):
136 scenario_file_name = os.path.join(self.scenario_dir,
139 if not os.path.exists(scenario_file_name):
140 raise Exception("The scenario '%s' does not exist."
141 % scenario_file_name)
143 LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
144 test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
146 if not os.path.exists(self.TEMP_DIR):
147 os.makedirs(self.TEMP_DIR)
149 self.apply_blacklist(scenario_file_name, test_file_name)
150 return test_file_name
153 def get_verifier_deployment_id():
155 Returns deployment id for active Rally deployment
157 cmd = ("rally deployment list | awk '/" +
158 getattr(config.CONF, 'rally_deployment_name') +
160 proc = subprocess.Popen(cmd, shell=True,
161 stdout=subprocess.PIPE,
162 stderr=subprocess.STDOUT)
163 deployment_uuid = proc.stdout.readline().rstrip()
164 return deployment_uuid
167 def create_rally_deployment(environ=None):
168 """Create new rally deployment"""
169 # set the architecture to default
170 pod_arch = env.get("POD_ARCH")
171 arch_filter = ['aarch64']
173 if pod_arch and pod_arch in arch_filter:
174 LOGGER.info("Apply aarch64 specific to rally config...")
175 with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
176 rally_patch_conf = pfile.read()
178 for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
180 if "cirros|testvm" in line:
181 print(rally_patch_conf)
183 LOGGER.info("Creating Rally environment...")
185 cmd = ['rally', 'deployment', 'destroy',
187 str(getattr(config.CONF, 'rally_deployment_name'))]
188 output = subprocess.check_output(cmd)
189 LOGGER.info("%s\n%s", " ".join(cmd), output)
190 except subprocess.CalledProcessError:
193 cmd = ['rally', 'deployment', 'create', '--fromenv',
194 '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
195 output = subprocess.check_output(cmd, env=environ)
196 LOGGER.info("%s\n%s", " ".join(cmd), output)
198 cmd = ['rally', 'deployment', 'check']
199 output = subprocess.check_output(cmd)
200 LOGGER.info("%s\n%s", " ".join(cmd), output)
201 return RallyBase.get_verifier_deployment_id()
204 def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
205 """Set keystone_default_role in rally.conf"""
206 if env.get("NEW_USER_ROLE").lower() != "member":
207 rconfig = configparser.RawConfigParser()
208 rconfig.read(rally_conf)
209 if not rconfig.has_section('openstack'):
210 rconfig.add_section('openstack')
212 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
213 with open(rally_conf, 'wb') as config_file:
214 rconfig.write(config_file)
217 def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
218 """Clean Rally config"""
219 if env.get("NEW_USER_ROLE").lower() != "member":
220 rconfig = configparser.RawConfigParser()
221 rconfig.read(rally_conf)
222 if rconfig.has_option('openstack', 'keystone_default_role'):
223 rconfig.remove_option('openstack', 'keystone_default_role')
224 with open(rally_conf, 'wb') as config_file:
225 rconfig.write(config_file)
228 def get_task_id(cmd_raw):
230 Get task id from command rally result.
233 :return: task_id as string
235 taskid_re = re.compile('^Task +(.*): started$')
236 for line in cmd_raw.splitlines(True):
238 match = taskid_re.match(line)
240 return match.group(1)
244 def task_succeed(json_raw):
246 Parse JSON from rally JSON results.
251 rally_report = json.loads(json_raw)
252 tasks = rally_report.get('tasks')
255 if task.get('status') != 'finished' or \
256 task.get('pass_sla') is not True:
262 def _migration_supported(self):
263 """Determine if migration is supported."""
264 if self.compute_cnt > 1:
268 def _network_trunk_supported(self):
269 """Determine if network trunk service is available"""
270 if 'trunk' in self.network_extensions:
276 """Exclude scenario."""
279 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
280 black_list_yaml = yaml.safe_load(black_list_file)
282 deploy_scenario = env.get('DEPLOY_SCENARIO')
283 if (bool(deploy_scenario) and
284 'scenario' in black_list_yaml.keys()):
285 for item in black_list_yaml['scenario']:
286 scenarios = item['scenarios']
287 in_it = RallyBase.in_iterable_re
288 if in_it(deploy_scenario, scenarios):
289 tests = item['tests']
290 black_tests.extend(tests)
291 except Exception: # pylint: disable=broad-except
292 LOGGER.debug("Scenario exclusion not applied.")
297 def in_iterable_re(needle, haystack):
299 Check if given needle is in the iterable haystack, using regex.
301 :param needle: string to be matched
302 :param haystack: iterable of strings (optionally regex patterns)
303 :return: True if needle is eqial to any of the elements in haystack,
304 or if a nonempty regex pattern in haystack is found in needle.
306 # match without regex
307 if needle in haystack:
310 for pattern in haystack:
311 # match if regex pattern is set and found in the needle
312 if pattern and re.search(pattern, needle) is not None:
318 """Exclude functionalities."""
323 with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
324 black_list_yaml = yaml.safe_load(black_list_file)
326 if not self._migration_supported():
327 func_list.append("no_migration")
328 if not self._network_trunk_supported():
329 func_list.append("no_net_trunk_service")
331 if 'functionality' in black_list_yaml.keys():
332 for item in black_list_yaml['functionality']:
333 functions = item['functions']
334 for func in func_list:
335 if func in functions:
336 tests = item['tests']
337 black_tests.extend(tests)
338 except Exception: # pylint: disable=broad-except
339 LOGGER.debug("Functionality exclusion not applied.")
343 def apply_blacklist(self, case_file_name, result_file_name):
344 """Apply blacklist."""
345 LOGGER.debug("Applying blacklist...")
346 cases_file = open(case_file_name, 'r')
347 result_file = open(result_file_name, 'w')
349 black_tests = list(set(self.excl_func() +
350 self.excl_scenario()))
353 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
356 for cases_line in cases_file:
358 for black_tests_line in black_tests:
359 if re.search(black_tests_line,
360 cases_line.strip().rstrip(':')):
364 result_file.write(str(cases_line))
366 if cases_line.isspace():
373 def file_is_empty(file_name):
374 """Determine is a file is empty."""
376 if os.stat(file_name).st_size > 0:
378 except Exception: # pylint: disable=broad-except
383 def _save_results(self, test_name, task_id):
384 """ Generate and save task execution results"""
385 # check for result directory and create it otherwise
386 if not os.path.exists(self.results_dir):
387 LOGGER.debug('%s does not exist, we create it.',
389 os.makedirs(self.results_dir)
391 # put detailed result to log
392 cmd = (["rally", "task", "detailed", "--uuid", task_id])
393 LOGGER.debug('running command: %s', cmd)
394 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
395 LOGGER.info("%s\n%s", " ".join(cmd), output)
397 # save report as JSON
398 report_json_name = '{}.json'.format(test_name)
399 report_json_dir = os.path.join(self.results_dir, report_json_name)
400 cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
401 "--out", report_json_dir])
402 LOGGER.debug('running command: %s', cmd)
403 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
404 LOGGER.info("%s\n%s", " ".join(cmd), output)
406 json_results = open(report_json_dir).read()
407 self._append_summary(json_results, test_name)
409 # parse JSON operation result
410 if self.task_succeed(json_results):
411 LOGGER.info('Test scenario: "%s" OK.', test_name)
413 LOGGER.info('Test scenario: "%s" Failed.', test_name)
415 def run_task(self, test_name):
417 LOGGER.info('Starting test scenario "%s" ...', test_name)
418 LOGGER.debug('running command: %s', self.run_cmd)
419 proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
420 stderr=subprocess.STDOUT)
421 output = proc.communicate()[0]
423 task_id = self.get_task_id(output)
424 LOGGER.debug('task_id : %s', task_id)
426 LOGGER.error("Failed to retrieve task_id")
427 LOGGER.error("Result:\n%s", output)
428 raise Exception("Failed to retrieve task id")
429 self._save_results(test_name, task_id)
431 def _append_summary(self, json_raw, test_name):
432 """Update statistics summary info."""
435 overall_duration = 0.0
439 rally_report = json.loads(json_raw)
440 for task in rally_report.get('tasks'):
441 for subtask in task.get('subtasks'):
443 for workload in subtask.get('workloads'):
444 if workload.get('full_duration'):
445 overall_duration += workload.get('full_duration')
447 if workload.get('data'):
448 nb_tests += len(workload.get('data'))
450 for result in workload.get('data'):
451 if not result.get('error'):
457 failures.append(subtask['title'])
459 success.append(subtask['title'])
461 scenario_summary = {'test_name': test_name,
462 'overall_duration': overall_duration,
463 'nb_tests': nb_tests,
464 'nb_success': nb_success,
466 'failures': failures,
467 'task_status': self.task_succeed(json_raw)}
468 self.summary.append(scenario_summary)
470 def prepare_run(self, **kwargs):
471 """Prepare resources needed by test scenarios."""
473 LOGGER.debug('Validating run tests...')
474 for test in kwargs.get('tests', self.TESTS):
475 if test in self.TESTS:
476 self.tests.append(test)
478 raise Exception("Test name '%s' is invalid" % test)
480 if not os.path.exists(self.TASK_DIR):
481 os.makedirs(self.TASK_DIR)
483 task = os.path.join(self.RALLY_DIR, 'task.yaml')
484 if not os.path.exists(task):
485 LOGGER.error("Task file '%s' does not exist.", task)
486 raise Exception("Task file '{}' does not exist.".
488 self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
489 shutil.copyfile(task, self.task_file)
491 task_macro = os.path.join(self.RALLY_DIR, 'macro')
492 if not os.path.exists(task_macro):
493 LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
494 raise Exception("Task macro dir '{}' does not exist.".
496 macro_dir = os.path.join(self.TASK_DIR, 'macro')
497 if os.path.exists(macro_dir):
498 shutil.rmtree(macro_dir)
499 shutil.copytree(task_macro, macro_dir)
501 self.update_keystone_default_role()
502 self.compute_cnt = len(self.cloud.list_hypervisors())
503 self.network_extensions = self.cloud.get_network_extensions()
504 self.flavor_alt = self.create_flavor_alt()
505 self.services = [service.name for service in
506 self.cloud.list_services()]
508 LOGGER.debug("flavor: %s", self.flavor_alt)
510 def prepare_task(self, test_name):
511 """Prepare resources for test run."""
512 file_name = self._prepare_test_list(test_name)
513 if self.file_is_empty(file_name):
514 LOGGER.info('No tests for scenario "%s"', test_name)
516 self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
517 "--task", self.task_file, "--task-args",
518 str(self.build_task_args(test_name))])
521 def run_tests(self, **kwargs):
523 optional = kwargs.get('optional', [])
524 for test in self.tests:
525 if test in self.services or test not in optional:
526 if self.prepare_task(test):
529 def _generate_report(self):
530 """Generate test execution summary report."""
537 res_table = prettytable.PrettyTable(
539 field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
540 res_table.align['Module'] = "l"
541 res_table.align['Duration'] = "r"
542 res_table.align['Success'] = "r"
544 # for each scenario we draw a row for the table
545 for item in self.summary:
546 if item['task_status'] is True:
548 total_duration += item['overall_duration']
549 total_nb_tests += item['nb_tests']
550 total_nb_success += item['nb_success']
552 success_avg = 100 * item['nb_success'] / item['nb_tests']
553 except ZeroDivisionError:
555 success_str = str("{:0.2f}".format(success_avg)) + '%'
556 duration_str = time.strftime("%H:%M:%S",
557 time.gmtime(item['overall_duration']))
558 res_table.add_row([item['test_name'], duration_str,
559 item['nb_tests'], success_str])
560 payload.append({'module': item['test_name'],
561 'details': {'duration': item['overall_duration'],
562 'nb tests': item['nb_tests'],
563 'success rate': success_str,
564 'success': item['success'],
565 'failures': item['failures']}})
567 total_duration_str = time.strftime("%H:%M:%S",
568 time.gmtime(total_duration))
570 self.result = 100 * total_nb_success / total_nb_tests
571 except ZeroDivisionError:
573 success_rate = "{:0.2f}".format(self.result)
574 success_rate_str = str(success_rate) + '%'
575 res_table.add_row(["", "", "", ""])
576 res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
579 LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
580 LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
581 self.case_name, success_rate, nb_modules,
583 payload.append({'summary': {'duration': total_duration,
584 'nb tests': total_nb_tests,
585 'nb success': success_rate}})
586 self.details = payload
589 def export_task(file_name, export_type="html"):
590 """Export all task results (e.g. html or xunit report)
593 subprocess.CalledProcessError: if Rally doesn't return 0
598 cmd = ["rally", "task", "export", "--type", export_type,
600 str(getattr(config.CONF, 'rally_deployment_name')),
602 LOGGER.debug('running command: %s', cmd)
603 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
604 LOGGER.info("%s\n%s", " ".join(cmd), output)
607 def verify_report(file_name, uuid, export_type="html"):
608 """Generate the verifier report (e.g. html or xunit report)
611 subprocess.CalledProcessError: if Rally doesn't return 0
616 cmd = ["rally", "verify", "report", "--type", export_type,
617 "--uuid", uuid, "--to", file_name]
618 LOGGER.debug('running command: %s', cmd)
619 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
620 LOGGER.info("%s\n%s", " ".join(cmd), output)
623 """Cleanup of OpenStack resources. Should be called on completion."""
624 self.clean_rally_conf()
626 self.orig_cloud.delete_flavor(self.flavor_alt.id)
627 super(RallyBase, self).clean()
629 def is_successful(self):
630 """The overall result of the test."""
631 for item in self.summary:
632 if item['task_status'] is False:
633 return testcase.TestCase.EX_TESTCASE_FAILED
635 return super(RallyBase, self).is_successful()
637 def run(self, **kwargs):
639 self.start_time = time.time()
641 assert super(RallyBase, self).run(
642 **kwargs) == testcase.TestCase.EX_OK
645 OS_USERNAME=self.project.user.name,
646 OS_PROJECT_NAME=self.project.project.name,
647 OS_PROJECT_ID=self.project.project.id,
648 OS_PASSWORD=self.project.password)
650 del environ['OS_TENANT_NAME']
651 del environ['OS_TENANT_ID']
652 except Exception: # pylint: disable=broad-except
654 self.create_rally_deployment(environ=environ)
655 self.prepare_run(**kwargs)
656 self.run_tests(**kwargs)
657 self._generate_report()
659 "{}/{}.html".format(self.results_dir, self.case_name))
661 "{}/{}.xml".format(self.results_dir, self.case_name),
662 export_type="junit-xml")
663 res = testcase.TestCase.EX_OK
664 except Exception as exc: # pylint: disable=broad-except
665 LOGGER.error('Error with run: %s', exc)
667 res = testcase.TestCase.EX_RUN_ERROR
668 self.stop_time = time.time()
672 class RallySanity(RallyBase):
673 """Rally sanity testcase implementation."""
675 def __init__(self, **kwargs):
676 """Initialize RallySanity object."""
677 if "case_name" not in kwargs:
678 kwargs["case_name"] = "rally_sanity"
679 super(RallySanity, self).__init__(**kwargs)
681 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
684 class RallyFull(RallyBase):
685 """Rally full testcase implementation."""
687 def __init__(self, **kwargs):
688 """Initialize RallyFull object."""
689 if "case_name" not in kwargs:
690 kwargs["case_name"] = "rally_full"
691 super(RallyFull, self).__init__(**kwargs)
693 self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
696 class RallyJobs(RallyBase):
697 """Rally OpenStack CI testcase implementation."""
701 def __init__(self, **kwargs):
702 """Initialize RallyJobs object."""
703 if "case_name" not in kwargs:
704 kwargs["case_name"] = "rally_jobs"
705 super(RallyJobs, self).__init__(**kwargs)
706 self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
707 self.task_yaml = None
709 def prepare_run(self, **kwargs):
710 """Create resources needed by test scenarios."""
711 super(RallyJobs, self).prepare_run(**kwargs)
712 with open(os.path.join(self.RALLY_DIR,
713 'rally_jobs.yaml'), 'r') as task_file:
714 self.task_yaml = yaml.safe_load(task_file)
716 for task in self.task_yaml:
717 if task not in self.tests:
718 raise Exception("Test '%s' not in '%s'" %
721 def apply_blacklist(self, case_file_name, result_file_name):
722 # pylint: disable=too-many-branches
723 """Apply blacklist."""
724 LOGGER.debug("Applying blacklist...")
725 black_tests = list(set(self.excl_func() +
726 self.excl_scenario()))
728 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
730 template = YAML(typ='jinja2')
731 with open(case_file_name, 'r') as fname:
732 cases = template.load(fname)
733 if cases.get("version", 1) == 1:
734 # scenarios in dictionary
735 for name in cases.keys():
736 if self.in_iterable_re(name, black_tests):
739 # workloads in subtasks
740 for sind, subtask in enumerate(cases.get('subtasks', [])):
742 for wind, workload in enumerate(subtask.get('workloads', [])):
743 scenario = workload.get('scenario', {})
744 for name in scenario.keys():
745 if self.in_iterable_re(name, black_tests):
748 for wind in reversed(idx):
749 cases['subtasks'][sind]['workloads'].pop(wind)
750 # scenarios in subtasks
752 for sind, subtask in enumerate(cases.get('subtasks', [])):
753 scenario = subtask.get('scenario', {})
754 for name in scenario.keys():
755 if self.in_iterable_re(name, black_tests):
758 for sind in reversed(idx):
759 cases['subtasks'].pop(sind)
761 with open(result_file_name, 'w') as fname:
762 template.dump(cases, fname)
764 def build_task_args(self, test_name):
765 """Build arguments for the Rally task."""
768 task_args['floating_network'] = str(self.ext_net.name)
770 task_args['floating_network'] = ''
774 def _remove_plugins_extra():
775 inst_dir = getattr(config.CONF, 'dir_rally_inst')
777 shutil.rmtree(os.path.join(inst_dir, 'plugins'))
778 shutil.rmtree(os.path.join(inst_dir, 'extra'))
779 except Exception: # pylint: disable=broad-except
782 def prepare_task(self, test_name):
783 """Prepare resources for test run."""
784 self._remove_plugins_extra()
785 jobs_dir = os.path.join(
786 getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
787 inst_dir = getattr(config.CONF, 'dir_rally_inst')
788 shutil.copytree(os.path.join(jobs_dir, 'plugins'),
789 os.path.join(inst_dir, 'plugins'))
790 shutil.copytree(os.path.join(jobs_dir, 'extra'),
791 os.path.join(inst_dir, 'extra'))
793 task_name = self.task_yaml.get(test_name).get("task")
794 task = os.path.join(jobs_dir, task_name)
795 if not os.path.exists(task):
796 raise Exception("The scenario '%s' does not exist." % task)
797 LOGGER.debug('Scenario fetched from : %s', task)
799 if not os.path.exists(self.TEMP_DIR):
800 os.makedirs(self.TEMP_DIR)
801 task_file_name = os.path.join(self.TEMP_DIR, task_name)
802 self.apply_blacklist(task, task_file_name)
803 self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
804 "--task-args", str(self.build_task_args(test_name))])
808 self._remove_plugins_extra()
809 super(RallyJobs, self).clean()