functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_CONF_PATH = "/etc/rally/rally.conf"
  47     RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     RALLY_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     TEMPLATE_DIR = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     SUPPORT_DIR = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     USERS_AMOUNT = 2
  58     TENANTS_AMOUNT = 3
  59     ITERATIONS_AMOUNT = 10
  60     CONCURRENCY = 4
  61     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  62     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  63     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  64
  65     visibility = 'public'
  66     shared_network = True
  67
  68     def __init__(self, **kwargs):
  69         """Initialize RallyBase object."""
  70         super(RallyBase, self).__init__(**kwargs)
  71         assert self.orig_cloud
  72         assert self.project
  73         if self.orig_cloud.get_role("admin"):
  74             role_name = "admin"
  75         elif self.orig_cloud.get_role("Admin"):
  76             role_name = "Admin"
  77         else:
  78             raise Exception("Cannot detect neither admin nor Admin")
  79         self.orig_cloud.grant_role(
  80             role_name, user=self.project.user.id,
  81             project=self.project.project.id,
  82             domain=self.project.domain.id)
  83         self.results_dir = os.path.join(
  84             getattr(config.CONF, 'dir_results'), self.case_name)
  85         self.task_file = ''
  86         self.creators = []
  87         self.summary = []
  88         self.scenario_dir = ''
  89         self.smoke = None
  90         self.start_time = None
  91         self.result = None
  92         self.details = None
  93         self.compute_cnt = 0
  94         self.flavor_alt = None
  95         self.tests = []
  96         self.run_cmd = ''
  97         self.network_extensions = []
  98         self.services = []
  99
 100     def build_task_args(self, test_name):
 101         """Build arguments for the Rally task."""
 102         task_args = {'service_list': [test_name]}
 103         task_args['image_name'] = str(self.image.name)
 104         task_args['flavor_name'] = str(self.flavor.name)
 105         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 106         task_args['glance_image_location'] = str(self.filename)
 107         task_args['glance_image_format'] = str(self.image_format)
 108         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 109         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 110         task_args['users_amount'] = self.USERS_AMOUNT
 111         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 112         task_args['use_existing_users'] = False
 113         task_args['iterations'] = self.ITERATIONS_AMOUNT
 114         task_args['concurrency'] = self.CONCURRENCY
 115         task_args['smoke'] = self.smoke
 116
 117         if self.ext_net:
 118             task_args['floating_network'] = str(self.ext_net.name)
 119         else:
 120             task_args['floating_network'] = ''
 121
 122         if self.network:
 123             task_args['netid'] = str(self.network.id)
 124         else:
 125             task_args['netid'] = ''
 126
 127         return task_args
 128
 129     def _prepare_test_list(self, test_name):
 130         """Build the list of test cases to be executed."""
 131         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 132         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 133                                           test_yaml_file_name)
 134
 135         if not os.path.exists(scenario_file_name):
 136             scenario_file_name = os.path.join(self.scenario_dir,
 137                                               test_yaml_file_name)
 138
 139             if not os.path.exists(scenario_file_name):
 140                 raise Exception("The scenario '%s' does not exist."
 141                                 % scenario_file_name)
 142
 143         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 144         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 145
 146         if not os.path.exists(self.TEMP_DIR):
 147             os.makedirs(self.TEMP_DIR)
 148
 149         self.apply_blacklist(scenario_file_name, test_file_name)
 150         return test_file_name
 151
 152     @staticmethod
 153     def get_verifier_deployment_id():
 154         """
 155         Returns deployment id for active Rally deployment
 156         """
 157         cmd = ("rally deployment list | awk '/" +
 158                getattr(config.CONF, 'rally_deployment_name') +
 159                "/ {print $2}'")
 160         proc = subprocess.Popen(cmd, shell=True,
 161                                 stdout=subprocess.PIPE,
 162                                 stderr=subprocess.STDOUT)
 163         deployment_uuid = proc.stdout.readline().rstrip()
 164         return deployment_uuid
 165
 166     @staticmethod
 167     def create_rally_deployment(environ=None):
 168         """Create new rally deployment"""
 169         # set the architecture to default
 170         pod_arch = env.get("POD_ARCH")
 171         arch_filter = ['aarch64']
 172
 173         if pod_arch and pod_arch in arch_filter:
 174             LOGGER.info("Apply aarch64 specific to rally config...")
 175             with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
 176                 rally_patch_conf = pfile.read()
 177
 178             for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
 179                 print(line, end=' ')
 180                 if "cirros|testvm" in line:
 181                     print(rally_patch_conf)
 182
 183         LOGGER.info("Creating Rally environment...")
 184         try:
 185             cmd = ['rally', 'deployment', 'destroy',
 186                    '--deployment',
 187                    str(getattr(config.CONF, 'rally_deployment_name'))]
 188             output = subprocess.check_output(cmd)
 189             LOGGER.info("%s\n%s", " ".join(cmd), output)
 190         except subprocess.CalledProcessError:
 191             pass
 192
 193         cmd = ['rally', 'deployment', 'create', '--fromenv',
 194                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 195         output = subprocess.check_output(cmd, env=environ)
 196         LOGGER.info("%s\n%s", " ".join(cmd), output)
 197
 198         cmd = ['rally', 'deployment', 'check']
 199         output = subprocess.check_output(cmd)
 200         LOGGER.info("%s\n%s", " ".join(cmd), output)
 201         return RallyBase.get_verifier_deployment_id()
 202
 203     @staticmethod
 204     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 205         """Set keystone_default_role in rally.conf"""
 206         if env.get("NEW_USER_ROLE").lower() != "member":
 207             rconfig = configparser.RawConfigParser()
 208             rconfig.read(rally_conf)
 209             if not rconfig.has_section('openstack'):
 210                 rconfig.add_section('openstack')
 211             rconfig.set(
 212                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 213             with open(rally_conf, 'wb') as config_file:
 214                 rconfig.write(config_file)
 215
 216     @staticmethod
 217     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 218         """Clean Rally config"""
 219         if env.get("NEW_USER_ROLE").lower() != "member":
 220             rconfig = configparser.RawConfigParser()
 221             rconfig.read(rally_conf)
 222             if rconfig.has_option('openstack', 'keystone_default_role'):
 223                 rconfig.remove_option('openstack', 'keystone_default_role')
 224             with open(rally_conf, 'wb') as config_file:
 225                 rconfig.write(config_file)
 226
 227     @staticmethod
 228     def get_task_id(cmd_raw):
 229         """
 230         Get task id from command rally result.
 231
 232         :param cmd_raw:
 233         :return: task_id as string
 234         """
 235         taskid_re = re.compile('^Task +(.*): started$')
 236         for line in cmd_raw.splitlines(True):
 237             line = line.strip()
 238             match = taskid_re.match(line)
 239             if match:
 240                 return match.group(1)
 241         return None
 242
 243     @staticmethod
 244     def task_succeed(json_raw):
 245         """
 246         Parse JSON from rally JSON results.
 247
 248         :param json_raw:
 249         :return: Bool
 250         """
 251         rally_report = json.loads(json_raw)
 252         tasks = rally_report.get('tasks')
 253         if tasks:
 254             for task in tasks:
 255                 if task.get('status') != 'finished' or \
 256                    task.get('pass_sla') is not True:
 257                     return False
 258         else:
 259             return False
 260         return True
 261
 262     def _migration_supported(self):
 263         """Determine if migration is supported."""
 264         if self.compute_cnt > 1:
 265             return True
 266         return False
 267
 268     def _network_trunk_supported(self):
 269         """Determine if network trunk service is available"""
 270         if 'trunk' in self.network_extensions:
 271             return True
 272         return False
 273
 274     @staticmethod
 275     def excl_scenario():
 276         """Exclude scenario."""
 277         black_tests = []
 278         try:
 279             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 280                 black_list_yaml = yaml.safe_load(black_list_file)
 281
 282             deploy_scenario = env.get('DEPLOY_SCENARIO')
 283             if (bool(deploy_scenario) and
 284                     'scenario' in black_list_yaml.keys()):
 285                 for item in black_list_yaml['scenario']:
 286                     scenarios = item['scenarios']
 287                     in_it = RallyBase.in_iterable_re
 288                     if in_it(deploy_scenario, scenarios):
 289                         tests = item['tests']
 290                         black_tests.extend(tests)
 291         except Exception:  # pylint: disable=broad-except
 292             LOGGER.debug("Scenario exclusion not applied.")
 293
 294         return black_tests
 295
 296     @staticmethod
 297     def in_iterable_re(needle, haystack):
 298         """
 299         Check if given needle is in the iterable haystack, using regex.
 300
 301         :param needle: string to be matched
 302         :param haystack: iterable of strings (optionally regex patterns)
 303         :return: True if needle is eqial to any of the elements in haystack,
 304                  or if a nonempty regex pattern in haystack is found in needle.
 305         """
 306         # match without regex
 307         if needle in haystack:
 308             return True
 309
 310         for pattern in haystack:
 311             # match if regex pattern is set and found in the needle
 312             if pattern and re.search(pattern, needle) is not None:
 313                 return True
 314
 315         return False
 316
 317     def excl_func(self):
 318         """Exclude functionalities."""
 319         black_tests = []
 320         func_list = []
 321
 322         try:
 323             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 324                 black_list_yaml = yaml.safe_load(black_list_file)
 325
 326             if not self._migration_supported():
 327                 func_list.append("no_migration")
 328             if not self._network_trunk_supported():
 329                 func_list.append("no_net_trunk_service")
 330
 331             if 'functionality' in black_list_yaml.keys():
 332                 for item in black_list_yaml['functionality']:
 333                     functions = item['functions']
 334                     for func in func_list:
 335                         if func in functions:
 336                             tests = item['tests']
 337                             black_tests.extend(tests)
 338         except Exception:  # pylint: disable=broad-except
 339             LOGGER.debug("Functionality exclusion not applied.")
 340
 341         return black_tests
 342
 343     def apply_blacklist(self, case_file_name, result_file_name):
 344         """Apply blacklist."""
 345         LOGGER.debug("Applying blacklist...")
 346         cases_file = open(case_file_name, 'r')
 347         result_file = open(result_file_name, 'w')
 348
 349         black_tests = list(set(self.excl_func() +
 350                                self.excl_scenario()))
 351
 352         if black_tests:
 353             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 354
 355         include = True
 356         for cases_line in cases_file:
 357             if include:
 358                 for black_tests_line in black_tests:
 359                     if re.search(black_tests_line,
 360                                  cases_line.strip().rstrip(':')):
 361                         include = False
 362                         break
 363                 else:
 364                     result_file.write(str(cases_line))
 365             else:
 366                 if cases_line.isspace():
 367                     include = True
 368
 369         cases_file.close()
 370         result_file.close()
 371
 372     @staticmethod
 373     def file_is_empty(file_name):
 374         """Determine is a file is empty."""
 375         try:
 376             if os.stat(file_name).st_size > 0:
 377                 return False
 378         except Exception:  # pylint: disable=broad-except
 379             pass
 380
 381         return True
 382
 383     def _save_results(self, test_name, task_id):
 384         """ Generate and save task execution results"""
 385         # check for result directory and create it otherwise
 386         if not os.path.exists(self.results_dir):
 387             LOGGER.debug('%s does not exist, we create it.',
 388                          self.results_dir)
 389             os.makedirs(self.results_dir)
 390
 391         # put detailed result to log
 392         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 393         LOGGER.debug('running command: %s', cmd)
 394         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 395         LOGGER.info("%s\n%s", " ".join(cmd), output)
 396
 397         # save report as JSON
 398         report_json_name = '{}.json'.format(test_name)
 399         report_json_dir = os.path.join(self.results_dir, report_json_name)
 400         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 401                 "--out", report_json_dir])
 402         LOGGER.debug('running command: %s', cmd)
 403         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 404         LOGGER.info("%s\n%s", " ".join(cmd), output)
 405
 406         json_results = open(report_json_dir).read()
 407         self._append_summary(json_results, test_name)
 408
 409         # parse JSON operation result
 410         if self.task_succeed(json_results):
 411             LOGGER.info('Test scenario: "%s" OK.', test_name)
 412         else:
 413             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 414
 415     def run_task(self, test_name):
 416         """Run a task."""
 417         LOGGER.info('Starting test scenario "%s" ...', test_name)
 418         LOGGER.debug('running command: %s', self.run_cmd)
 419         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 420                                 stderr=subprocess.STDOUT)
 421         output = proc.communicate()[0]
 422
 423         task_id = self.get_task_id(output)
 424         LOGGER.debug('task_id : %s', task_id)
 425         if task_id is None:
 426             LOGGER.error("Failed to retrieve task_id")
 427             LOGGER.error("Result:\n%s", output)
 428             raise Exception("Failed to retrieve task id")
 429         self._save_results(test_name, task_id)
 430
 431     def _append_summary(self, json_raw, test_name):
 432         """Update statistics summary info."""
 433         nb_tests = 0
 434         nb_success = 0
 435         overall_duration = 0.0
 436
 437         rally_report = json.loads(json_raw)
 438         for task in rally_report.get('tasks'):
 439             for subtask in task.get('subtasks'):
 440                 for workload in subtask.get('workloads'):
 441                     if workload.get('full_duration'):
 442                         overall_duration += workload.get('full_duration')
 443
 444                     if workload.get('data'):
 445                         nb_tests += len(workload.get('data'))
 446
 447                     for result in workload.get('data'):
 448                         if not result.get('error'):
 449                             nb_success += 1
 450
 451         scenario_summary = {'test_name': test_name,
 452                             'overall_duration': overall_duration,
 453                             'nb_tests': nb_tests,
 454                             'nb_success': nb_success,
 455                             'task_status': self.task_succeed(json_raw)}
 456         self.summary.append(scenario_summary)
 457
 458     def prepare_run(self, **kwargs):
 459         """Prepare resources needed by test scenarios."""
 460         assert self.cloud
 461         LOGGER.debug('Validating run tests...')
 462         for test in kwargs.get('tests', self.TESTS):
 463             if test in self.TESTS:
 464                 self.tests.append(test)
 465             else:
 466                 raise Exception("Test name '%s' is invalid" % test)
 467
 468         if not os.path.exists(self.TASK_DIR):
 469             os.makedirs(self.TASK_DIR)
 470
 471         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 472         if not os.path.exists(task):
 473             LOGGER.error("Task file '%s' does not exist.", task)
 474             raise Exception("Task file '{}' does not exist.".
 475                             format(task))
 476         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 477         shutil.copyfile(task, self.task_file)
 478
 479         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 480         if not os.path.exists(task_macro):
 481             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 482             raise Exception("Task macro dir '{}' does not exist.".
 483                             format(task_macro))
 484         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 485         if os.path.exists(macro_dir):
 486             shutil.rmtree(macro_dir)
 487         shutil.copytree(task_macro, macro_dir)
 488
 489         self.update_keystone_default_role()
 490         self.compute_cnt = len(self.cloud.list_hypervisors())
 491         self.network_extensions = self.cloud.get_network_extensions()
 492         self.flavor_alt = self.create_flavor_alt()
 493         self.services = [service.name for service in
 494                          self.cloud.list_services()]
 495
 496         LOGGER.debug("flavor: %s", self.flavor_alt)
 497
 498     def prepare_task(self, test_name):
 499         """Prepare resources for test run."""
 500         file_name = self._prepare_test_list(test_name)
 501         if self.file_is_empty(file_name):
 502             LOGGER.info('No tests for scenario "%s"', test_name)
 503             return False
 504         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 505                          "--task", self.task_file, "--task-args",
 506                          str(self.build_task_args(test_name))])
 507         return True
 508
 509     def run_tests(self, **kwargs):
 510         """Execute tests."""
 511         optional = kwargs.get('optional', [])
 512         for test in self.tests:
 513             if test in self.services or test not in optional:
 514                 if self.prepare_task(test):
 515                     self.run_task(test)
 516
 517     def _generate_report(self):
 518         """Generate test execution summary report."""
 519         total_duration = 0.0
 520         total_nb_tests = 0
 521         total_nb_success = 0
 522         nb_modules = 0
 523         payload = []
 524
 525         res_table = prettytable.PrettyTable(
 526             padding_width=2,
 527             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 528         res_table.align['Module'] = "l"
 529         res_table.align['Duration'] = "r"
 530         res_table.align['Success'] = "r"
 531
 532         # for each scenario we draw a row for the table
 533         for item in self.summary:
 534             if item['task_status'] is True:
 535                 nb_modules += 1
 536             total_duration += item['overall_duration']
 537             total_nb_tests += item['nb_tests']
 538             total_nb_success += item['nb_success']
 539             try:
 540                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 541             except ZeroDivisionError:
 542                 success_avg = 0
 543             success_str = str("{:0.2f}".format(success_avg)) + '%'
 544             duration_str = time.strftime("%H:%M:%S",
 545                                          time.gmtime(item['overall_duration']))
 546             res_table.add_row([item['test_name'], duration_str,
 547                                item['nb_tests'], success_str])
 548             payload.append({'module': item['test_name'],
 549                             'details': {'duration': item['overall_duration'],
 550                                         'nb tests': item['nb_tests'],
 551                                         'success': success_str}})
 552
 553         total_duration_str = time.strftime("%H:%M:%S",
 554                                            time.gmtime(total_duration))
 555         try:
 556             self.result = 100 * total_nb_success / total_nb_tests
 557         except ZeroDivisionError:
 558             self.result = 100
 559         success_rate = "{:0.2f}".format(self.result)
 560         success_rate_str = str(success_rate) + '%'
 561         res_table.add_row(["", "", "", ""])
 562         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 563                            success_rate_str])
 564
 565         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 566         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 567                     self.case_name, success_rate, nb_modules,
 568                     len(self.summary))
 569         payload.append({'summary': {'duration': total_duration,
 570                                     'nb tests': total_nb_tests,
 571                                     'nb success': success_rate}})
 572         self.details = payload
 573
 574     @staticmethod
 575     def export_task(file_name, export_type="html"):
 576         """Export all task results (e.g. html or xunit report)
 577
 578         Raises:
 579             subprocess.CalledProcessError: if Rally doesn't return 0
 580
 581         Returns:
 582             None
 583         """
 584         cmd = ["rally", "task", "export", "--type", export_type,
 585                "--deployment",
 586                str(getattr(config.CONF, 'rally_deployment_name')),
 587                "--to", file_name]
 588         LOGGER.debug('running command: %s', cmd)
 589         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 590         LOGGER.info("%s\n%s", " ".join(cmd), output)
 591
 592     @staticmethod
 593     def verify_report(file_name, uuid, export_type="html"):
 594         """Generate the verifier report (e.g. html or xunit report)
 595
 596         Raises:
 597             subprocess.CalledProcessError: if Rally doesn't return 0
 598
 599         Returns:
 600             None
 601         """
 602         cmd = ["rally", "verify", "report", "--type", export_type,
 603                "--uuid", uuid, "--to", file_name]
 604         LOGGER.debug('running command: %s', cmd)
 605         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 606         LOGGER.info("%s\n%s", " ".join(cmd), output)
 607
 608     def clean(self):
 609         """Cleanup of OpenStack resources. Should be called on completion."""
 610         self.clean_rally_conf()
 611         if self.flavor_alt:
 612             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 613         super(RallyBase, self).clean()
 614
 615     def is_successful(self):
 616         """The overall result of the test."""
 617         for item in self.summary:
 618             if item['task_status'] is False:
 619                 return testcase.TestCase.EX_TESTCASE_FAILED
 620
 621         return super(RallyBase, self).is_successful()
 622
 623     def run(self, **kwargs):
 624         """Run testcase."""
 625         self.start_time = time.time()
 626         try:
 627             assert super(RallyBase, self).run(
 628                 **kwargs) == testcase.TestCase.EX_OK
 629             environ = dict(
 630                 os.environ,
 631                 OS_USERNAME=self.project.user.name,
 632                 OS_PROJECT_NAME=self.project.project.name,
 633                 OS_PROJECT_ID=self.project.project.id,
 634                 OS_PASSWORD=self.project.password)
 635             try:
 636                 del environ['OS_TENANT_NAME']
 637                 del environ['OS_TENANT_ID']
 638             except Exception:  # pylint: disable=broad-except
 639                 pass
 640             self.create_rally_deployment(environ=environ)
 641             self.prepare_run(**kwargs)
 642             self.run_tests(**kwargs)
 643             self._generate_report()
 644             self.export_task(
 645                 "{}/{}.html".format(self.results_dir, self.case_name))
 646             self.export_task(
 647                 "{}/{}.xml".format(self.results_dir, self.case_name),
 648                 export_type="junit-xml")
 649             res = testcase.TestCase.EX_OK
 650         except Exception as exc:   # pylint: disable=broad-except
 651             LOGGER.error('Error with run: %s', exc)
 652             self.result = 0
 653             res = testcase.TestCase.EX_RUN_ERROR
 654         self.stop_time = time.time()
 655         return res
 656
 657
 658 class RallySanity(RallyBase):
 659     """Rally sanity testcase implementation."""
 660
 661     def __init__(self, **kwargs):
 662         """Initialize RallySanity object."""
 663         if "case_name" not in kwargs:
 664             kwargs["case_name"] = "rally_sanity"
 665         super(RallySanity, self).__init__(**kwargs)
 666         self.smoke = True
 667         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 668
 669
 670 class RallyFull(RallyBase):
 671     """Rally full testcase implementation."""
 672
 673     def __init__(self, **kwargs):
 674         """Initialize RallyFull object."""
 675         if "case_name" not in kwargs:
 676             kwargs["case_name"] = "rally_full"
 677         super(RallyFull, self).__init__(**kwargs)
 678         self.smoke = False
 679         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 680
 681
 682 class RallyJobs(RallyBase):
 683     """Rally OpenStack CI testcase implementation."""
 684
 685     TESTS = ["neutron"]
 686
 687     def __init__(self, **kwargs):
 688         """Initialize RallyJobs object."""
 689         if "case_name" not in kwargs:
 690             kwargs["case_name"] = "rally_jobs"
 691         super(RallyJobs, self).__init__(**kwargs)
 692         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 693         self.task_yaml = None
 694
 695     def prepare_run(self, **kwargs):
 696         """Create resources needed by test scenarios."""
 697         super(RallyJobs, self).prepare_run(**kwargs)
 698         with open(os.path.join(self.RALLY_DIR,
 699                                'rally_jobs.yaml'), 'r') as task_file:
 700             self.task_yaml = yaml.safe_load(task_file)
 701
 702         for task in self.task_yaml:
 703             if task not in self.tests:
 704                 raise Exception("Test '%s' not in '%s'" %
 705                                 (task, self.tests))
 706
 707     def apply_blacklist(self, case_file_name, result_file_name):
 708         # pylint: disable=too-many-branches
 709         """Apply blacklist."""
 710         LOGGER.debug("Applying blacklist...")
 711         black_tests = list(set(self.excl_func() +
 712                                self.excl_scenario()))
 713         if black_tests:
 714             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 715
 716         template = YAML(typ='jinja2')
 717         with open(case_file_name, 'r') as fname:
 718             cases = template.load(fname)
 719         if cases.get("version", 1) == 1:
 720             # scenarios in dictionary
 721             for name in cases.keys():
 722                 if self.in_iterable_re(name, black_tests):
 723                     cases.pop(name)
 724         else:
 725             # workloads in subtasks
 726             for sind, subtask in enumerate(cases.get('subtasks', [])):
 727                 idx = []
 728                 for wind, workload in enumerate(subtask.get('workloads', [])):
 729                     scenario = workload.get('scenario', {})
 730                     for name in scenario.keys():
 731                         if self.in_iterable_re(name, black_tests):
 732                             idx.append(wind)
 733                             break
 734                 for wind in reversed(idx):
 735                     cases['subtasks'][sind]['workloads'].pop(wind)
 736             # scenarios in subtasks
 737             idx = []
 738             for sind, subtask in enumerate(cases.get('subtasks', [])):
 739                 scenario = subtask.get('scenario', {})
 740                 for name in scenario.keys():
 741                     if self.in_iterable_re(name, black_tests):
 742                         idx.append(sind)
 743                         break
 744             for sind in reversed(idx):
 745                 cases['subtasks'].pop(sind)
 746
 747         with open(result_file_name, 'w') as fname:
 748             template.dump(cases, fname)
 749
 750     def build_task_args(self, test_name):
 751         """Build arguments for the Rally task."""
 752         task_args = {}
 753         if self.ext_net:
 754             task_args['floating_network'] = str(self.ext_net.name)
 755         else:
 756             task_args['floating_network'] = ''
 757         return task_args
 758
 759     @staticmethod
 760     def _remove_plugins_extra():
 761         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 762         try:
 763             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 764             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 765         except Exception:  # pylint: disable=broad-except
 766             pass
 767
 768     def prepare_task(self, test_name):
 769         """Prepare resources for test run."""
 770         self._remove_plugins_extra()
 771         jobs_dir = os.path.join(
 772             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 773         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 774         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 775                         os.path.join(inst_dir, 'plugins'))
 776         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 777                         os.path.join(inst_dir, 'extra'))
 778
 779         task_name = self.task_yaml.get(test_name).get("task")
 780         task = os.path.join(jobs_dir, task_name)
 781         if not os.path.exists(task):
 782             raise Exception("The scenario '%s' does not exist." % task)
 783         LOGGER.debug('Scenario fetched from : %s', task)
 784
 785         if not os.path.exists(self.TEMP_DIR):
 786             os.makedirs(self.TEMP_DIR)
 787         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 788         self.apply_blacklist(task, task_file_name)
 789         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 790                          "--task-args", str(self.build_task_args(test_name))])
 791         return True
 792
 793     def clean(self):
 794         self._remove_plugins_extra()
 795         super(RallyJobs, self).clean()