functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_CONF_PATH = "/etc/rally/rally.conf"
  47     RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     RALLY_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     TEMPLATE_DIR = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     SUPPORT_DIR = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     USERS_AMOUNT = 2
  58     TENANTS_AMOUNT = 3
  59     ITERATIONS_AMOUNT = 10
  60     CONCURRENCY = 4
  61     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  62     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  63     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  64
  65     visibility = 'public'
  66     shared_network = True
  67
  68     def __init__(self, **kwargs):
  69         """Initialize RallyBase object."""
  70         super(RallyBase, self).__init__(**kwargs)
  71         assert self.orig_cloud
  72         assert self.project
  73         if self.orig_cloud.get_role("admin"):
  74             role_name = "admin"
  75         elif self.orig_cloud.get_role("Admin"):
  76             role_name = "Admin"
  77         else:
  78             raise Exception("Cannot detect neither admin nor Admin")
  79         self.orig_cloud.grant_role(
  80             role_name, user=self.project.user.id,
  81             project=self.project.project.id,
  82             domain=self.project.domain.id)
  83         self.results_dir = os.path.join(
  84             getattr(config.CONF, 'dir_results'), self.case_name)
  85         self.task_file = ''
  86         self.creators = []
  87         self.summary = []
  88         self.scenario_dir = ''
  89         self.smoke = None
  90         self.start_time = None
  91         self.result = None
  92         self.details = None
  93         self.compute_cnt = 0
  94         self.flavor_alt = None
  95         self.tests = []
  96         self.run_cmd = ''
  97         self.network_extensions = []
  98         self.services = []
  99
 100     def build_task_args(self, test_name):
 101         """Build arguments for the Rally task."""
 102         task_args = {'service_list': [test_name]}
 103         task_args['image_name'] = str(self.image.name)
 104         task_args['flavor_name'] = str(self.flavor.name)
 105         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 106         task_args['glance_image_location'] = str(self.filename)
 107         task_args['glance_image_format'] = str(self.image_format)
 108         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 109         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 110         task_args['users_amount'] = self.USERS_AMOUNT
 111         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 112         task_args['use_existing_users'] = False
 113         task_args['iterations'] = self.ITERATIONS_AMOUNT
 114         task_args['concurrency'] = self.CONCURRENCY
 115         task_args['smoke'] = self.smoke
 116
 117         if self.ext_net:
 118             task_args['floating_network'] = str(self.ext_net.name)
 119         else:
 120             task_args['floating_network'] = ''
 121
 122         if self.network:
 123             task_args['netid'] = str(self.network.id)
 124         else:
 125             task_args['netid'] = ''
 126
 127         return task_args
 128
 129     def _prepare_test_list(self, test_name):
 130         """Build the list of test cases to be executed."""
 131         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 132         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 133                                           test_yaml_file_name)
 134
 135         if not os.path.exists(scenario_file_name):
 136             scenario_file_name = os.path.join(self.scenario_dir,
 137                                               test_yaml_file_name)
 138
 139             if not os.path.exists(scenario_file_name):
 140                 raise Exception("The scenario '%s' does not exist."
 141                                 % scenario_file_name)
 142
 143         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 144         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 145
 146         if not os.path.exists(self.TEMP_DIR):
 147             os.makedirs(self.TEMP_DIR)
 148
 149         self.apply_blacklist(scenario_file_name, test_file_name)
 150         return test_file_name
 151
 152     @staticmethod
 153     def get_verifier_deployment_id():
 154         """
 155         Returns deployment id for active Rally deployment
 156         """
 157         cmd = ("rally deployment list | awk '/" +
 158                getattr(config.CONF, 'rally_deployment_name') +
 159                "/ {print $2}'")
 160         proc = subprocess.Popen(cmd, shell=True,
 161                                 stdout=subprocess.PIPE,
 162                                 stderr=subprocess.STDOUT)
 163         deployment_uuid = proc.stdout.readline().rstrip()
 164         return deployment_uuid
 165
 166     @staticmethod
 167     def create_rally_deployment(environ=None):
 168         """Create new rally deployment"""
 169         # set the architecture to default
 170         pod_arch = env.get("POD_ARCH")
 171         arch_filter = ['aarch64']
 172
 173         if pod_arch and pod_arch in arch_filter:
 174             LOGGER.info("Apply aarch64 specific to rally config...")
 175             with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
 176                 rally_patch_conf = pfile.read()
 177
 178             for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
 179                 print(line, end=' ')
 180                 if "cirros|testvm" in line:
 181                     print(rally_patch_conf)
 182
 183         LOGGER.info("Creating Rally environment...")
 184         try:
 185             cmd = ['rally', 'deployment', 'destroy',
 186                    '--deployment',
 187                    str(getattr(config.CONF, 'rally_deployment_name'))]
 188             output = subprocess.check_output(cmd)
 189             LOGGER.info("%s\n%s", " ".join(cmd), output)
 190         except subprocess.CalledProcessError:
 191             pass
 192
 193         cmd = ['rally', 'deployment', 'create', '--fromenv',
 194                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 195         output = subprocess.check_output(cmd, env=environ)
 196         LOGGER.info("%s\n%s", " ".join(cmd), output)
 197
 198         cmd = ['rally', 'deployment', 'check']
 199         output = subprocess.check_output(cmd)
 200         LOGGER.info("%s\n%s", " ".join(cmd), output)
 201         return RallyBase.get_verifier_deployment_id()
 202
 203     @staticmethod
 204     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 205         """Set keystone_default_role in rally.conf"""
 206         if env.get("NEW_USER_ROLE").lower() != "member":
 207             rconfig = configparser.RawConfigParser()
 208             rconfig.read(rally_conf)
 209             if not rconfig.has_section('openstack'):
 210                 rconfig.add_section('openstack')
 211             rconfig.set(
 212                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 213             with open(rally_conf, 'wb') as config_file:
 214                 rconfig.write(config_file)
 215
 216     @staticmethod
 217     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 218         """Clean Rally config"""
 219         if env.get("NEW_USER_ROLE").lower() != "member":
 220             rconfig = configparser.RawConfigParser()
 221             rconfig.read(rally_conf)
 222             if rconfig.has_option('openstack', 'keystone_default_role'):
 223                 rconfig.remove_option('openstack', 'keystone_default_role')
 224             with open(rally_conf, 'wb') as config_file:
 225                 rconfig.write(config_file)
 226
 227     @staticmethod
 228     def get_task_id(cmd_raw):
 229         """
 230         Get task id from command rally result.
 231
 232         :param cmd_raw:
 233         :return: task_id as string
 234         """
 235         taskid_re = re.compile('^Task +(.*): started$')
 236         for line in cmd_raw.splitlines(True):
 237             line = line.strip()
 238             match = taskid_re.match(line)
 239             if match:
 240                 return match.group(1)
 241         return None
 242
 243     @staticmethod
 244     def task_succeed(json_raw):
 245         """
 246         Parse JSON from rally JSON results.
 247
 248         :param json_raw:
 249         :return: Bool
 250         """
 251         rally_report = json.loads(json_raw)
 252         tasks = rally_report.get('tasks')
 253         if tasks:
 254             for task in tasks:
 255                 if task.get('status') != 'finished' or \
 256                    task.get('pass_sla') is not True:
 257                     return False
 258         else:
 259             return False
 260         return True
 261
 262     def _migration_supported(self):
 263         """Determine if migration is supported."""
 264         if self.compute_cnt > 1:
 265             return True
 266         return False
 267
 268     def _network_trunk_supported(self):
 269         """Determine if network trunk service is available"""
 270         if 'trunk' in self.network_extensions:
 271             return True
 272         return False
 273
 274     @staticmethod
 275     def excl_scenario():
 276         """Exclude scenario."""
 277         black_tests = []
 278         try:
 279             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 280                 black_list_yaml = yaml.safe_load(black_list_file)
 281
 282             deploy_scenario = env.get('DEPLOY_SCENARIO')
 283             if (bool(deploy_scenario) and
 284                     'scenario' in black_list_yaml.keys()):
 285                 for item in black_list_yaml['scenario']:
 286                     scenarios = item['scenarios']
 287                     in_it = RallyBase.in_iterable_re
 288                     if in_it(deploy_scenario, scenarios):
 289                         tests = item['tests']
 290                         black_tests.extend(tests)
 291         except Exception:  # pylint: disable=broad-except
 292             LOGGER.debug("Scenario exclusion not applied.")
 293
 294         return black_tests
 295
 296     @staticmethod
 297     def in_iterable_re(needle, haystack):
 298         """
 299         Check if given needle is in the iterable haystack, using regex.
 300
 301         :param needle: string to be matched
 302         :param haystack: iterable of strings (optionally regex patterns)
 303         :return: True if needle is eqial to any of the elements in haystack,
 304                  or if a nonempty regex pattern in haystack is found in needle.
 305         """
 306         # match without regex
 307         if needle in haystack:
 308             return True
 309
 310         for pattern in haystack:
 311             # match if regex pattern is set and found in the needle
 312             if pattern and re.search(pattern, needle) is not None:
 313                 return True
 314
 315         return False
 316
 317     def excl_func(self):
 318         """Exclude functionalities."""
 319         black_tests = []
 320         func_list = []
 321
 322         try:
 323             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 324                 black_list_yaml = yaml.safe_load(black_list_file)
 325
 326             if not self._migration_supported():
 327                 func_list.append("no_migration")
 328             if not self._network_trunk_supported():
 329                 func_list.append("no_net_trunk_service")
 330
 331             if 'functionality' in black_list_yaml.keys():
 332                 for item in black_list_yaml['functionality']:
 333                     functions = item['functions']
 334                     for func in func_list:
 335                         if func in functions:
 336                             tests = item['tests']
 337                             black_tests.extend(tests)
 338         except Exception:  # pylint: disable=broad-except
 339             LOGGER.debug("Functionality exclusion not applied.")
 340
 341         return black_tests
 342
 343     def apply_blacklist(self, case_file_name, result_file_name):
 344         """Apply blacklist."""
 345         LOGGER.debug("Applying blacklist...")
 346         cases_file = open(case_file_name, 'r')
 347         result_file = open(result_file_name, 'w')
 348
 349         black_tests = list(set(self.excl_func() +
 350                                self.excl_scenario()))
 351
 352         if black_tests:
 353             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 354
 355         include = True
 356         for cases_line in cases_file:
 357             if include:
 358                 for black_tests_line in black_tests:
 359                     if re.search(black_tests_line,
 360                                  cases_line.strip().rstrip(':')):
 361                         include = False
 362                         break
 363                 else:
 364                     result_file.write(str(cases_line))
 365             else:
 366                 if cases_line.isspace():
 367                     include = True
 368
 369         cases_file.close()
 370         result_file.close()
 371
 372     @staticmethod
 373     def file_is_empty(file_name):
 374         """Determine is a file is empty."""
 375         try:
 376             if os.stat(file_name).st_size > 0:
 377                 return False
 378         except Exception:  # pylint: disable=broad-except
 379             pass
 380
 381         return True
 382
 383     def _save_results(self, test_name, task_id):
 384         """ Generate and save task execution results"""
 385         # check for result directory and create it otherwise
 386         if not os.path.exists(self.results_dir):
 387             LOGGER.debug('%s does not exist, we create it.',
 388                          self.results_dir)
 389             os.makedirs(self.results_dir)
 390
 391         # put detailed result to log
 392         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 393         LOGGER.debug('running command: %s', cmd)
 394         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 395         LOGGER.info("%s\n%s", " ".join(cmd), output)
 396
 397         # save report as JSON
 398         report_json_name = '{}.json'.format(test_name)
 399         report_json_dir = os.path.join(self.results_dir, report_json_name)
 400         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 401                 "--out", report_json_dir])
 402         LOGGER.debug('running command: %s', cmd)
 403         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 404         LOGGER.info("%s\n%s", " ".join(cmd), output)
 405
 406         json_results = open(report_json_dir).read()
 407         self._append_summary(json_results, test_name)
 408
 409         # parse JSON operation result
 410         if self.task_succeed(json_results):
 411             LOGGER.info('Test scenario: "%s" OK.', test_name)
 412         else:
 413             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 414
 415     def run_task(self, test_name):
 416         """Run a task."""
 417         LOGGER.info('Starting test scenario "%s" ...', test_name)
 418         LOGGER.debug('running command: %s', self.run_cmd)
 419         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 420                                 stderr=subprocess.STDOUT)
 421         output = proc.communicate()[0]
 422
 423         task_id = self.get_task_id(output)
 424         LOGGER.debug('task_id : %s', task_id)
 425         if task_id is None:
 426             LOGGER.error("Failed to retrieve task_id")
 427             LOGGER.error("Result:\n%s", output)
 428             raise Exception("Failed to retrieve task id")
 429         self._save_results(test_name, task_id)
 430
 431     def _append_summary(self, json_raw, test_name):
 432         """Update statistics summary info."""
 433         nb_tests = 0
 434         nb_success = 0
 435         overall_duration = 0.0
 436         success = []
 437         failures = []
 438
 439         rally_report = json.loads(json_raw)
 440         for task in rally_report.get('tasks'):
 441             for subtask in task.get('subtasks'):
 442                 has_errors = False
 443                 for workload in subtask.get('workloads'):
 444                     if workload.get('full_duration'):
 445                         overall_duration += workload.get('full_duration')
 446
 447                     if workload.get('data'):
 448                         nb_tests += len(workload.get('data'))
 449
 450                     for result in workload.get('data'):
 451                         if not result.get('error'):
 452                             nb_success += 1
 453                         else:
 454                             has_errors = True
 455
 456                 if has_errors:
 457                     failures.append(subtask['title'])
 458                 else:
 459                     success.append(subtask['title'])
 460
 461         scenario_summary = {'test_name': test_name,
 462                             'overall_duration': overall_duration,
 463                             'nb_tests': nb_tests,
 464                             'nb_success': nb_success,
 465                             'success': success,
 466                             'failures': failures,
 467                             'task_status': self.task_succeed(json_raw)}
 468         self.summary.append(scenario_summary)
 469
 470     def prepare_run(self, **kwargs):
 471         """Prepare resources needed by test scenarios."""
 472         assert self.cloud
 473         LOGGER.debug('Validating run tests...')
 474         for test in kwargs.get('tests', self.TESTS):
 475             if test in self.TESTS:
 476                 self.tests.append(test)
 477             else:
 478                 raise Exception("Test name '%s' is invalid" % test)
 479
 480         if not os.path.exists(self.TASK_DIR):
 481             os.makedirs(self.TASK_DIR)
 482
 483         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 484         if not os.path.exists(task):
 485             LOGGER.error("Task file '%s' does not exist.", task)
 486             raise Exception("Task file '{}' does not exist.".
 487                             format(task))
 488         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 489         shutil.copyfile(task, self.task_file)
 490
 491         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 492         if not os.path.exists(task_macro):
 493             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 494             raise Exception("Task macro dir '{}' does not exist.".
 495                             format(task_macro))
 496         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 497         if os.path.exists(macro_dir):
 498             shutil.rmtree(macro_dir)
 499         shutil.copytree(task_macro, macro_dir)
 500
 501         self.update_keystone_default_role()
 502         self.compute_cnt = len(self.cloud.list_hypervisors())
 503         self.network_extensions = self.cloud.get_network_extensions()
 504         self.flavor_alt = self.create_flavor_alt()
 505         self.services = [service.name for service in
 506                          self.cloud.list_services()]
 507
 508         LOGGER.debug("flavor: %s", self.flavor_alt)
 509
 510     def prepare_task(self, test_name):
 511         """Prepare resources for test run."""
 512         file_name = self._prepare_test_list(test_name)
 513         if self.file_is_empty(file_name):
 514             LOGGER.info('No tests for scenario "%s"', test_name)
 515             return False
 516         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 517                          "--task", self.task_file, "--task-args",
 518                          str(self.build_task_args(test_name))])
 519         return True
 520
 521     def run_tests(self, **kwargs):
 522         """Execute tests."""
 523         optional = kwargs.get('optional', [])
 524         for test in self.tests:
 525             if test in self.services or test not in optional:
 526                 if self.prepare_task(test):
 527                     self.run_task(test)
 528
 529     def _generate_report(self):
 530         """Generate test execution summary report."""
 531         total_duration = 0.0
 532         total_nb_tests = 0
 533         total_nb_success = 0
 534         nb_modules = 0
 535         payload = []
 536
 537         res_table = prettytable.PrettyTable(
 538             padding_width=2,
 539             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 540         res_table.align['Module'] = "l"
 541         res_table.align['Duration'] = "r"
 542         res_table.align['Success'] = "r"
 543
 544         # for each scenario we draw a row for the table
 545         for item in self.summary:
 546             if item['task_status'] is True:
 547                 nb_modules += 1
 548             total_duration += item['overall_duration']
 549             total_nb_tests += item['nb_tests']
 550             total_nb_success += item['nb_success']
 551             try:
 552                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 553             except ZeroDivisionError:
 554                 success_avg = 0
 555             success_str = str("{:0.2f}".format(success_avg)) + '%'
 556             duration_str = time.strftime("%H:%M:%S",
 557                                          time.gmtime(item['overall_duration']))
 558             res_table.add_row([item['test_name'], duration_str,
 559                                item['nb_tests'], success_str])
 560             payload.append({'module': item['test_name'],
 561                             'details': {'duration': item['overall_duration'],
 562                                         'nb tests': item['nb_tests'],
 563                                         'success rate': success_str,
 564                                         'success': item['success'],
 565                                         'failures': item['failures']}})
 566
 567         total_duration_str = time.strftime("%H:%M:%S",
 568                                            time.gmtime(total_duration))
 569         try:
 570             self.result = 100 * total_nb_success / total_nb_tests
 571         except ZeroDivisionError:
 572             self.result = 100
 573         success_rate = "{:0.2f}".format(self.result)
 574         success_rate_str = str(success_rate) + '%'
 575         res_table.add_row(["", "", "", ""])
 576         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 577                            success_rate_str])
 578
 579         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 580         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 581                     self.case_name, success_rate, nb_modules,
 582                     len(self.summary))
 583         payload.append({'summary': {'duration': total_duration,
 584                                     'nb tests': total_nb_tests,
 585                                     'nb success': success_rate}})
 586         self.details = payload
 587
 588     @staticmethod
 589     def export_task(file_name, export_type="html"):
 590         """Export all task results (e.g. html or xunit report)
 591
 592         Raises:
 593             subprocess.CalledProcessError: if Rally doesn't return 0
 594
 595         Returns:
 596             None
 597         """
 598         cmd = ["rally", "task", "export", "--type", export_type,
 599                "--deployment",
 600                str(getattr(config.CONF, 'rally_deployment_name')),
 601                "--to", file_name]
 602         LOGGER.debug('running command: %s', cmd)
 603         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 604         LOGGER.info("%s\n%s", " ".join(cmd), output)
 605
 606     @staticmethod
 607     def verify_report(file_name, uuid, export_type="html"):
 608         """Generate the verifier report (e.g. html or xunit report)
 609
 610         Raises:
 611             subprocess.CalledProcessError: if Rally doesn't return 0
 612
 613         Returns:
 614             None
 615         """
 616         cmd = ["rally", "verify", "report", "--type", export_type,
 617                "--uuid", uuid, "--to", file_name]
 618         LOGGER.debug('running command: %s', cmd)
 619         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 620         LOGGER.info("%s\n%s", " ".join(cmd), output)
 621
 622     def clean(self):
 623         """Cleanup of OpenStack resources. Should be called on completion."""
 624         self.clean_rally_conf()
 625         if self.flavor_alt:
 626             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 627         super(RallyBase, self).clean()
 628
 629     def is_successful(self):
 630         """The overall result of the test."""
 631         for item in self.summary:
 632             if item['task_status'] is False:
 633                 return testcase.TestCase.EX_TESTCASE_FAILED
 634
 635         return super(RallyBase, self).is_successful()
 636
 637     def run(self, **kwargs):
 638         """Run testcase."""
 639         self.start_time = time.time()
 640         try:
 641             assert super(RallyBase, self).run(
 642                 **kwargs) == testcase.TestCase.EX_OK
 643             environ = dict(
 644                 os.environ,
 645                 OS_USERNAME=self.project.user.name,
 646                 OS_PROJECT_NAME=self.project.project.name,
 647                 OS_PROJECT_ID=self.project.project.id,
 648                 OS_PASSWORD=self.project.password)
 649             try:
 650                 del environ['OS_TENANT_NAME']
 651                 del environ['OS_TENANT_ID']
 652             except Exception:  # pylint: disable=broad-except
 653                 pass
 654             self.create_rally_deployment(environ=environ)
 655             self.prepare_run(**kwargs)
 656             self.run_tests(**kwargs)
 657             self._generate_report()
 658             self.export_task(
 659                 "{}/{}.html".format(self.results_dir, self.case_name))
 660             self.export_task(
 661                 "{}/{}.xml".format(self.results_dir, self.case_name),
 662                 export_type="junit-xml")
 663             res = testcase.TestCase.EX_OK
 664         except Exception as exc:   # pylint: disable=broad-except
 665             LOGGER.error('Error with run: %s', exc)
 666             self.result = 0
 667             res = testcase.TestCase.EX_RUN_ERROR
 668         self.stop_time = time.time()
 669         return res
 670
 671
 672 class RallySanity(RallyBase):
 673     """Rally sanity testcase implementation."""
 674
 675     def __init__(self, **kwargs):
 676         """Initialize RallySanity object."""
 677         if "case_name" not in kwargs:
 678             kwargs["case_name"] = "rally_sanity"
 679         super(RallySanity, self).__init__(**kwargs)
 680         self.smoke = True
 681         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 682
 683
 684 class RallyFull(RallyBase):
 685     """Rally full testcase implementation."""
 686
 687     def __init__(self, **kwargs):
 688         """Initialize RallyFull object."""
 689         if "case_name" not in kwargs:
 690             kwargs["case_name"] = "rally_full"
 691         super(RallyFull, self).__init__(**kwargs)
 692         self.smoke = False
 693         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 694
 695
 696 class RallyJobs(RallyBase):
 697     """Rally OpenStack CI testcase implementation."""
 698
 699     TESTS = ["neutron"]
 700
 701     def __init__(self, **kwargs):
 702         """Initialize RallyJobs object."""
 703         if "case_name" not in kwargs:
 704             kwargs["case_name"] = "rally_jobs"
 705         super(RallyJobs, self).__init__(**kwargs)
 706         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 707         self.task_yaml = None
 708
 709     def prepare_run(self, **kwargs):
 710         """Create resources needed by test scenarios."""
 711         super(RallyJobs, self).prepare_run(**kwargs)
 712         with open(os.path.join(self.RALLY_DIR,
 713                                'rally_jobs.yaml'), 'r') as task_file:
 714             self.task_yaml = yaml.safe_load(task_file)
 715
 716         for task in self.task_yaml:
 717             if task not in self.tests:
 718                 raise Exception("Test '%s' not in '%s'" %
 719                                 (task, self.tests))
 720
 721     def apply_blacklist(self, case_file_name, result_file_name):
 722         # pylint: disable=too-many-branches
 723         """Apply blacklist."""
 724         LOGGER.debug("Applying blacklist...")
 725         black_tests = list(set(self.excl_func() +
 726                                self.excl_scenario()))
 727         if black_tests:
 728             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 729
 730         template = YAML(typ='jinja2')
 731         with open(case_file_name, 'r') as fname:
 732             cases = template.load(fname)
 733         if cases.get("version", 1) == 1:
 734             # scenarios in dictionary
 735             for name in cases.keys():
 736                 if self.in_iterable_re(name, black_tests):
 737                     cases.pop(name)
 738         else:
 739             # workloads in subtasks
 740             for sind, subtask in enumerate(cases.get('subtasks', [])):
 741                 idx = []
 742                 for wind, workload in enumerate(subtask.get('workloads', [])):
 743                     scenario = workload.get('scenario', {})
 744                     for name in scenario.keys():
 745                         if self.in_iterable_re(name, black_tests):
 746                             idx.append(wind)
 747                             break
 748                 for wind in reversed(idx):
 749                     cases['subtasks'][sind]['workloads'].pop(wind)
 750             # scenarios in subtasks
 751             idx = []
 752             for sind, subtask in enumerate(cases.get('subtasks', [])):
 753                 scenario = subtask.get('scenario', {})
 754                 for name in scenario.keys():
 755                     if self.in_iterable_re(name, black_tests):
 756                         idx.append(sind)
 757                         break
 758             for sind in reversed(idx):
 759                 cases['subtasks'].pop(sind)
 760
 761         with open(result_file_name, 'w') as fname:
 762             template.dump(cases, fname)
 763
 764     def build_task_args(self, test_name):
 765         """Build arguments for the Rally task."""
 766         task_args = {}
 767         if self.ext_net:
 768             task_args['floating_network'] = str(self.ext_net.name)
 769         else:
 770             task_args['floating_network'] = ''
 771         return task_args
 772
 773     @staticmethod
 774     def _remove_plugins_extra():
 775         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 776         try:
 777             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 778             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 779         except Exception:  # pylint: disable=broad-except
 780             pass
 781
 782     def prepare_task(self, test_name):
 783         """Prepare resources for test run."""
 784         self._remove_plugins_extra()
 785         jobs_dir = os.path.join(
 786             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 787         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 788         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 789                         os.path.join(inst_dir, 'plugins'))
 790         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 791                         os.path.join(inst_dir, 'extra'))
 792
 793         task_name = self.task_yaml.get(test_name).get("task")
 794         task = os.path.join(jobs_dir, task_name)
 795         if not os.path.exists(task):
 796             raise Exception("The scenario '%s' does not exist." % task)
 797         LOGGER.debug('Scenario fetched from : %s', task)
 798
 799         if not os.path.exists(self.TEMP_DIR):
 800             os.makedirs(self.TEMP_DIR)
 801         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 802         self.apply_blacklist(task, task_file_name)
 803         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 804                          "--task-args", str(self.build_task_args(test_name))])
 805         return True
 806
 807     def clean(self):
 808         self._remove_plugins_extra()
 809         super(RallyJobs, self).clean()