functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_CONF_PATH = "/etc/rally/rally.conf"
  47     RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     RALLY_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     TEMPLATE_DIR = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     SUPPORT_DIR = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     USERS_AMOUNT = 2
  58     TENANTS_AMOUNT = 3
  59     ITERATIONS_AMOUNT = 10
  60     CONCURRENCY = 4
  61     VOLUME_VERSION = 3
  62     VOLUME_SERVICE_TYPE = "volumev3"
  63     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  64     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69
  70     def __init__(self, **kwargs):
  71         """Initialize RallyBase object."""
  72         super(RallyBase, self).__init__(**kwargs)
  73         assert self.orig_cloud
  74         assert self.project
  75         if self.orig_cloud.get_role("admin"):
  76             role_name = "admin"
  77         elif self.orig_cloud.get_role("Admin"):
  78             role_name = "Admin"
  79         else:
  80             raise Exception("Cannot detect neither admin nor Admin")
  81         self.orig_cloud.grant_role(
  82             role_name, user=self.project.user.id,
  83             project=self.project.project.id,
  84             domain=self.project.domain.id)
  85         self.results_dir = os.path.join(
  86             getattr(config.CONF, 'dir_results'), self.case_name)
  87         self.task_file = ''
  88         self.creators = []
  89         self.summary = []
  90         self.scenario_dir = ''
  91         self.smoke = None
  92         self.start_time = None
  93         self.result = None
  94         self.details = None
  95         self.compute_cnt = 0
  96         self.flavor_alt = None
  97         self.tests = []
  98         self.run_cmd = ''
  99         self.network_extensions = []
 100         self.services = []
 101
 102     def build_task_args(self, test_name):
 103         """Build arguments for the Rally task."""
 104         task_args = {'service_list': [test_name]}
 105         task_args['image_name'] = str(self.image.name)
 106         task_args['flavor_name'] = str(self.flavor.name)
 107         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 108         task_args['glance_image_location'] = str(self.filename)
 109         task_args['glance_image_format'] = str(self.image_format)
 110         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 111         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 112         task_args['users_amount'] = self.USERS_AMOUNT
 113         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 114         task_args['use_existing_users'] = False
 115         task_args['iterations'] = self.ITERATIONS_AMOUNT
 116         task_args['concurrency'] = self.CONCURRENCY
 117         task_args['smoke'] = self.smoke
 118         task_args['volume_version'] = self.VOLUME_VERSION
 119         task_args['volume_service_type'] = self.VOLUME_SERVICE_TYPE
 120         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 121
 122         if self.ext_net:
 123             task_args['floating_network'] = str(self.ext_net.name)
 124         else:
 125             task_args['floating_network'] = ''
 126
 127         if self.network:
 128             task_args['netid'] = str(self.network.id)
 129         else:
 130             task_args['netid'] = ''
 131
 132         return task_args
 133
 134     def _prepare_test_list(self, test_name):
 135         """Build the list of test cases to be executed."""
 136         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 137         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 138                                           test_yaml_file_name)
 139
 140         if not os.path.exists(scenario_file_name):
 141             scenario_file_name = os.path.join(self.scenario_dir,
 142                                               test_yaml_file_name)
 143
 144             if not os.path.exists(scenario_file_name):
 145                 raise Exception("The scenario '%s' does not exist."
 146                                 % scenario_file_name)
 147
 148         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 149         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 150
 151         if not os.path.exists(self.TEMP_DIR):
 152             os.makedirs(self.TEMP_DIR)
 153
 154         self.apply_blacklist(scenario_file_name, test_file_name)
 155         return test_file_name
 156
 157     @staticmethod
 158     def get_verifier_deployment_id():
 159         """
 160         Returns deployment id for active Rally deployment
 161         """
 162         cmd = ("rally deployment list | awk '/" +
 163                getattr(config.CONF, 'rally_deployment_name') +
 164                "/ {print $2}'")
 165         proc = subprocess.Popen(cmd, shell=True,
 166                                 stdout=subprocess.PIPE,
 167                                 stderr=subprocess.STDOUT)
 168         deployment_uuid = proc.stdout.readline().rstrip()
 169         return deployment_uuid.decode("utf-8")
 170
 171     @staticmethod
 172     def create_rally_deployment(environ=None):
 173         """Create new rally deployment"""
 174         # set the architecture to default
 175         pod_arch = env.get("POD_ARCH")
 176         arch_filter = ['aarch64']
 177
 178         if pod_arch and pod_arch in arch_filter:
 179             LOGGER.info("Apply aarch64 specific to rally config...")
 180             with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
 181                 rally_patch_conf = pfile.read()
 182
 183             for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
 184                 print(line, end=' ')
 185                 if "cirros|testvm" in line:
 186                     print(rally_patch_conf)
 187
 188         LOGGER.info("Creating Rally environment...")
 189         try:
 190             cmd = ['rally', 'deployment', 'destroy',
 191                    '--deployment',
 192                    str(getattr(config.CONF, 'rally_deployment_name'))]
 193             output = subprocess.check_output(cmd)
 194             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 195         except subprocess.CalledProcessError:
 196             pass
 197
 198         cmd = ['rally', 'deployment', 'create', '--fromenv',
 199                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 200         output = subprocess.check_output(cmd, env=environ)
 201         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 202
 203         cmd = ['rally', 'deployment', 'check']
 204         output = subprocess.check_output(cmd)
 205         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 206         return RallyBase.get_verifier_deployment_id()
 207
 208     @staticmethod
 209     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 210         """Set keystone_default_role in rally.conf"""
 211         if env.get("NEW_USER_ROLE").lower() != "member":
 212             rconfig = configparser.RawConfigParser()
 213             rconfig.read(rally_conf)
 214             if not rconfig.has_section('openstack'):
 215                 rconfig.add_section('openstack')
 216             rconfig.set(
 217                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 218             with open(rally_conf, 'w') as config_file:
 219                 rconfig.write(config_file)
 220
 221     @staticmethod
 222     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 223         """Clean Rally config"""
 224         if env.get("NEW_USER_ROLE").lower() != "member":
 225             rconfig = configparser.RawConfigParser()
 226             rconfig.read(rally_conf)
 227             if rconfig.has_option('openstack', 'keystone_default_role'):
 228                 rconfig.remove_option('openstack', 'keystone_default_role')
 229             with open(rally_conf, 'w') as config_file:
 230                 rconfig.write(config_file)
 231
 232     @staticmethod
 233     def get_task_id(cmd_raw):
 234         """
 235         Get task id from command rally result.
 236
 237         :param cmd_raw:
 238         :return: task_id as string
 239         """
 240         taskid_re = re.compile('^Task +(.*): started$')
 241         for line in cmd_raw.splitlines(True):
 242             line = line.strip()
 243             match = taskid_re.match(line.decode("utf-8"))
 244             if match:
 245                 return match.group(1)
 246         return None
 247
 248     @staticmethod
 249     def task_succeed(json_raw):
 250         """
 251         Parse JSON from rally JSON results.
 252
 253         :param json_raw:
 254         :return: Bool
 255         """
 256         rally_report = json.loads(json_raw)
 257         tasks = rally_report.get('tasks')
 258         if tasks:
 259             for task in tasks:
 260                 if task.get('status') != 'finished' or \
 261                    task.get('pass_sla') is not True:
 262                     return False
 263         else:
 264             return False
 265         return True
 266
 267     def _migration_supported(self):
 268         """Determine if migration is supported."""
 269         if self.compute_cnt > 1:
 270             return True
 271         return False
 272
 273     def _network_trunk_supported(self):
 274         """Determine if network trunk service is available"""
 275         if 'trunk' in self.network_extensions:
 276             return True
 277         return False
 278
 279     @staticmethod
 280     def excl_scenario():
 281         """Exclude scenario."""
 282         black_tests = []
 283         try:
 284             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 285                 black_list_yaml = yaml.safe_load(black_list_file)
 286
 287             deploy_scenario = env.get('DEPLOY_SCENARIO')
 288             if (bool(deploy_scenario) and
 289                     'scenario' in black_list_yaml.keys()):
 290                 for item in black_list_yaml['scenario']:
 291                     scenarios = item['scenarios']
 292                     in_it = RallyBase.in_iterable_re
 293                     if in_it(deploy_scenario, scenarios):
 294                         tests = item['tests']
 295                         black_tests.extend(tests)
 296         except Exception:  # pylint: disable=broad-except
 297             LOGGER.debug("Scenario exclusion not applied.")
 298
 299         return black_tests
 300
 301     @staticmethod
 302     def in_iterable_re(needle, haystack):
 303         """
 304         Check if given needle is in the iterable haystack, using regex.
 305
 306         :param needle: string to be matched
 307         :param haystack: iterable of strings (optionally regex patterns)
 308         :return: True if needle is eqial to any of the elements in haystack,
 309                  or if a nonempty regex pattern in haystack is found in needle.
 310         """
 311         # match without regex
 312         if needle in haystack:
 313             return True
 314
 315         for pattern in haystack:
 316             # match if regex pattern is set and found in the needle
 317             if pattern and re.search(pattern, needle) is not None:
 318                 return True
 319
 320         return False
 321
 322     def excl_func(self):
 323         """Exclude functionalities."""
 324         black_tests = []
 325         func_list = []
 326
 327         try:
 328             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 329                 black_list_yaml = yaml.safe_load(black_list_file)
 330
 331             if env.get('BLOCK_MIGRATION').lower() == 'true':
 332                 func_list.append("block_migration")
 333             if not self._migration_supported():
 334                 func_list.append("no_migration")
 335             if not self._network_trunk_supported():
 336                 func_list.append("no_net_trunk_service")
 337
 338             if 'functionality' in black_list_yaml.keys():
 339                 for item in black_list_yaml['functionality']:
 340                     functions = item['functions']
 341                     for func in func_list:
 342                         if func in functions:
 343                             tests = item['tests']
 344                             black_tests.extend(tests)
 345         except Exception:  # pylint: disable=broad-except
 346             LOGGER.debug("Functionality exclusion not applied.")
 347
 348         return black_tests
 349
 350     def apply_blacklist(self, case_file_name, result_file_name):
 351         """Apply blacklist."""
 352         LOGGER.debug("Applying blacklist...")
 353         cases_file = open(case_file_name, 'r')
 354         result_file = open(result_file_name, 'w')
 355
 356         black_tests = list(set(self.excl_func() +
 357                                self.excl_scenario()))
 358
 359         if black_tests:
 360             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 361
 362         include = True
 363         for cases_line in cases_file:
 364             if include:
 365                 for black_tests_line in black_tests:
 366                     if re.search(black_tests_line,
 367                                  cases_line.strip().rstrip(':')):
 368                         include = False
 369                         break
 370                 else:
 371                     result_file.write(str(cases_line))
 372             else:
 373                 if cases_line.isspace():
 374                     include = True
 375
 376         cases_file.close()
 377         result_file.close()
 378
 379     @staticmethod
 380     def file_is_empty(file_name):
 381         """Determine is a file is empty."""
 382         try:
 383             if os.stat(file_name).st_size > 0:
 384                 return False
 385         except Exception:  # pylint: disable=broad-except
 386             pass
 387
 388         return True
 389
 390     def _save_results(self, test_name, task_id):
 391         """ Generate and save task execution results"""
 392         # check for result directory and create it otherwise
 393         if not os.path.exists(self.results_dir):
 394             LOGGER.debug('%s does not exist, we create it.',
 395                          self.results_dir)
 396             os.makedirs(self.results_dir)
 397
 398         # put detailed result to log
 399         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 400         LOGGER.debug('running command: %s', cmd)
 401         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 402         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 403
 404         # save report as JSON
 405         report_json_name = '{}.json'.format(test_name)
 406         report_json_dir = os.path.join(self.results_dir, report_json_name)
 407         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 408                 "--out", report_json_dir])
 409         LOGGER.debug('running command: %s', cmd)
 410         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 411         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 412
 413         json_results = open(report_json_dir).read()
 414         self._append_summary(json_results, test_name)
 415
 416         # parse JSON operation result
 417         if self.task_succeed(json_results):
 418             LOGGER.info('Test scenario: "%s" OK.', test_name)
 419         else:
 420             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 421
 422     def run_task(self, test_name):
 423         """Run a task."""
 424         LOGGER.info('Starting test scenario "%s" ...', test_name)
 425         LOGGER.debug('running command: %s', self.run_cmd)
 426         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 427                                 stderr=subprocess.STDOUT)
 428         output = proc.communicate()[0]
 429
 430         task_id = self.get_task_id(output)
 431         LOGGER.debug('task_id : %s', task_id)
 432         if task_id is None:
 433             LOGGER.error("Failed to retrieve task_id")
 434             LOGGER.error("Result:\n%s", output)
 435             raise Exception("Failed to retrieve task id")
 436         self._save_results(test_name, task_id)
 437
 438     def _append_summary(self, json_raw, test_name):
 439         """Update statistics summary info."""
 440         nb_tests = 0
 441         nb_success = 0
 442         overall_duration = 0.0
 443         success = []
 444         failures = []
 445
 446         rally_report = json.loads(json_raw)
 447         for task in rally_report.get('tasks'):
 448             for subtask in task.get('subtasks'):
 449                 has_errors = False
 450                 for workload in subtask.get('workloads'):
 451                     if workload.get('full_duration'):
 452                         overall_duration += workload.get('full_duration')
 453
 454                     if workload.get('data'):
 455                         nb_tests += len(workload.get('data'))
 456
 457                     for result in workload.get('data'):
 458                         if not result.get('error'):
 459                             nb_success += 1
 460                         else:
 461                             has_errors = True
 462
 463                 if has_errors:
 464                     failures.append(subtask['title'])
 465                 else:
 466                     success.append(subtask['title'])
 467
 468         scenario_summary = {'test_name': test_name,
 469                             'overall_duration': overall_duration,
 470                             'nb_tests': nb_tests,
 471                             'nb_success': nb_success,
 472                             'success': success,
 473                             'failures': failures,
 474                             'task_status': self.task_succeed(json_raw)}
 475         self.summary.append(scenario_summary)
 476
 477     def prepare_run(self, **kwargs):
 478         """Prepare resources needed by test scenarios."""
 479         assert self.cloud
 480         LOGGER.debug('Validating run tests...')
 481         for test in kwargs.get('tests', self.TESTS):
 482             if test in self.TESTS:
 483                 self.tests.append(test)
 484             else:
 485                 raise Exception("Test name '%s' is invalid" % test)
 486
 487         if not os.path.exists(self.TASK_DIR):
 488             os.makedirs(self.TASK_DIR)
 489
 490         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 491         if not os.path.exists(task):
 492             LOGGER.error("Task file '%s' does not exist.", task)
 493             raise Exception("Task file '{}' does not exist.".
 494                             format(task))
 495         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 496         shutil.copyfile(task, self.task_file)
 497
 498         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 499         if not os.path.exists(task_macro):
 500             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 501             raise Exception("Task macro dir '{}' does not exist.".
 502                             format(task_macro))
 503         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 504         if os.path.exists(macro_dir):
 505             shutil.rmtree(macro_dir)
 506         shutil.copytree(task_macro, macro_dir)
 507
 508         self.update_keystone_default_role()
 509         self.compute_cnt = len(self.cloud.list_hypervisors())
 510         self.network_extensions = self.cloud.get_network_extensions()
 511         self.flavor_alt = self.create_flavor_alt()
 512         self.services = [service.name for service in
 513                          self.cloud.list_services()]
 514
 515         LOGGER.debug("flavor: %s", self.flavor_alt)
 516
 517     def prepare_task(self, test_name):
 518         """Prepare resources for test run."""
 519         file_name = self._prepare_test_list(test_name)
 520         if self.file_is_empty(file_name):
 521             LOGGER.info('No tests for scenario "%s"', test_name)
 522             return False
 523         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 524                          "--task", self.task_file, "--task-args",
 525                          str(self.build_task_args(test_name))])
 526         return True
 527
 528     def run_tests(self, **kwargs):
 529         """Execute tests."""
 530         optional = kwargs.get('optional', [])
 531         for test in self.tests:
 532             if test in self.services or test not in optional:
 533                 if self.prepare_task(test):
 534                     self.run_task(test)
 535
 536     def _generate_report(self):
 537         """Generate test execution summary report."""
 538         total_duration = 0.0
 539         total_nb_tests = 0
 540         total_nb_success = 0
 541         nb_modules = 0
 542         payload = []
 543
 544         res_table = prettytable.PrettyTable(
 545             padding_width=2,
 546             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 547         res_table.align['Module'] = "l"
 548         res_table.align['Duration'] = "r"
 549         res_table.align['Success'] = "r"
 550
 551         # for each scenario we draw a row for the table
 552         for item in self.summary:
 553             if item['task_status'] is True:
 554                 nb_modules += 1
 555             total_duration += item['overall_duration']
 556             total_nb_tests += item['nb_tests']
 557             total_nb_success += item['nb_success']
 558             try:
 559                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 560             except ZeroDivisionError:
 561                 success_avg = 0
 562             success_str = str("{:0.2f}".format(success_avg)) + '%'
 563             duration_str = time.strftime("%H:%M:%S",
 564                                          time.gmtime(item['overall_duration']))
 565             res_table.add_row([item['test_name'], duration_str,
 566                                item['nb_tests'], success_str])
 567             payload.append({'module': item['test_name'],
 568                             'details': {'duration': item['overall_duration'],
 569                                         'nb tests': item['nb_tests'],
 570                                         'success rate': success_str,
 571                                         'success': item['success'],
 572                                         'failures': item['failures']}})
 573
 574         total_duration_str = time.strftime("%H:%M:%S",
 575                                            time.gmtime(total_duration))
 576         try:
 577             self.result = 100 * total_nb_success / total_nb_tests
 578         except ZeroDivisionError:
 579             self.result = 100
 580         success_rate = "{:0.2f}".format(self.result)
 581         success_rate_str = str(success_rate) + '%'
 582         res_table.add_row(["", "", "", ""])
 583         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 584                            success_rate_str])
 585
 586         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 587         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 588                     self.case_name, success_rate, nb_modules,
 589                     len(self.summary))
 590         payload.append({'summary': {'duration': total_duration,
 591                                     'nb tests': total_nb_tests,
 592                                     'nb success': success_rate}})
 593         self.details = payload
 594
 595     @staticmethod
 596     def export_task(file_name, export_type="html"):
 597         """Export all task results (e.g. html or xunit report)
 598
 599         Raises:
 600             subprocess.CalledProcessError: if Rally doesn't return 0
 601
 602         Returns:
 603             None
 604         """
 605         cmd = ["rally", "task", "export", "--type", export_type,
 606                "--deployment",
 607                str(getattr(config.CONF, 'rally_deployment_name')),
 608                "--to", file_name]
 609         LOGGER.debug('running command: %s', cmd)
 610         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 611         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 612
 613     @staticmethod
 614     def verify_report(file_name, uuid, export_type="html"):
 615         """Generate the verifier report (e.g. html or xunit report)
 616
 617         Raises:
 618             subprocess.CalledProcessError: if Rally doesn't return 0
 619
 620         Returns:
 621             None
 622         """
 623         cmd = ["rally", "verify", "report", "--type", export_type,
 624                "--uuid", uuid, "--to", file_name]
 625         LOGGER.debug('running command: %s', cmd)
 626         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 627         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 628
 629     def clean(self):
 630         """Cleanup of OpenStack resources. Should be called on completion."""
 631         self.clean_rally_conf()
 632         if self.flavor_alt:
 633             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 634         super(RallyBase, self).clean()
 635
 636     def is_successful(self):
 637         """The overall result of the test."""
 638         for item in self.summary:
 639             if item['task_status'] is False:
 640                 return testcase.TestCase.EX_TESTCASE_FAILED
 641
 642         return super(RallyBase, self).is_successful()
 643
 644     def run(self, **kwargs):
 645         """Run testcase."""
 646         self.start_time = time.time()
 647         try:
 648             assert super(RallyBase, self).run(
 649                 **kwargs) == testcase.TestCase.EX_OK
 650             self.create_rally_deployment(environ=self.project.get_environ())
 651             self.prepare_run(**kwargs)
 652             self.run_tests(**kwargs)
 653             self._generate_report()
 654             self.export_task(
 655                 "{}/{}.html".format(self.results_dir, self.case_name))
 656             self.export_task(
 657                 "{}/{}.xml".format(self.results_dir, self.case_name),
 658                 export_type="junit-xml")
 659             res = testcase.TestCase.EX_OK
 660         except Exception:   # pylint: disable=broad-except
 661             LOGGER.exception('Error with run:')
 662             self.result = 0
 663             res = testcase.TestCase.EX_RUN_ERROR
 664         self.stop_time = time.time()
 665         return res
 666
 667
 668 class RallySanity(RallyBase):
 669     """Rally sanity testcase implementation."""
 670
 671     def __init__(self, **kwargs):
 672         """Initialize RallySanity object."""
 673         if "case_name" not in kwargs:
 674             kwargs["case_name"] = "rally_sanity"
 675         super(RallySanity, self).__init__(**kwargs)
 676         self.smoke = True
 677         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 678
 679
 680 class RallyFull(RallyBase):
 681     """Rally full testcase implementation."""
 682
 683     def __init__(self, **kwargs):
 684         """Initialize RallyFull object."""
 685         if "case_name" not in kwargs:
 686             kwargs["case_name"] = "rally_full"
 687         super(RallyFull, self).__init__(**kwargs)
 688         self.smoke = False
 689         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 690
 691
 692 class RallyJobs(RallyBase):
 693     """Rally OpenStack CI testcase implementation."""
 694
 695     TESTS = ["neutron"]
 696
 697     def __init__(self, **kwargs):
 698         """Initialize RallyJobs object."""
 699         if "case_name" not in kwargs:
 700             kwargs["case_name"] = "rally_jobs"
 701         super(RallyJobs, self).__init__(**kwargs)
 702         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 703         self.task_yaml = None
 704
 705     def prepare_run(self, **kwargs):
 706         """Create resources needed by test scenarios."""
 707         super(RallyJobs, self).prepare_run(**kwargs)
 708         with open(os.path.join(self.RALLY_DIR,
 709                                'rally_jobs.yaml'), 'r') as task_file:
 710             self.task_yaml = yaml.safe_load(task_file)
 711
 712         for task in self.task_yaml:
 713             if task not in self.tests:
 714                 raise Exception("Test '%s' not in '%s'" %
 715                                 (task, self.tests))
 716
 717     def apply_blacklist(self, case_file_name, result_file_name):
 718         # pylint: disable=too-many-branches
 719         """Apply blacklist."""
 720         LOGGER.debug("Applying blacklist...")
 721         black_tests = list(set(self.excl_func() +
 722                                self.excl_scenario()))
 723         if black_tests:
 724             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 725
 726         template = YAML(typ='jinja2')
 727         with open(case_file_name, 'r') as fname:
 728             cases = template.load(fname)
 729         if cases.get("version", 1) == 1:
 730             # scenarios in dictionary
 731             for name in cases.keys():
 732                 if self.in_iterable_re(name, black_tests):
 733                     cases.pop(name)
 734         else:
 735             # workloads in subtasks
 736             for sind, subtask in enumerate(cases.get('subtasks', [])):
 737                 idx = []
 738                 for wind, workload in enumerate(subtask.get('workloads', [])):
 739                     scenario = workload.get('scenario', {})
 740                     for name in scenario.keys():
 741                         if self.in_iterable_re(name, black_tests):
 742                             idx.append(wind)
 743                             break
 744                 for wind in reversed(idx):
 745                     cases['subtasks'][sind]['workloads'].pop(wind)
 746             # scenarios in subtasks
 747             idx = []
 748             for sind, subtask in enumerate(cases.get('subtasks', [])):
 749                 scenario = subtask.get('scenario', {})
 750                 for name in scenario.keys():
 751                     if self.in_iterable_re(name, black_tests):
 752                         idx.append(sind)
 753                         break
 754             for sind in reversed(idx):
 755                 cases['subtasks'].pop(sind)
 756
 757         with open(result_file_name, 'w') as fname:
 758             template.dump(cases, fname)
 759
 760     def build_task_args(self, test_name):
 761         """Build arguments for the Rally task."""
 762         task_args = {}
 763         if self.ext_net:
 764             task_args['floating_network'] = str(self.ext_net.name)
 765         else:
 766             task_args['floating_network'] = ''
 767         return task_args
 768
 769     @staticmethod
 770     def _remove_plugins_extra():
 771         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 772         try:
 773             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 774             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 775         except Exception:  # pylint: disable=broad-except
 776             pass
 777
 778     def prepare_task(self, test_name):
 779         """Prepare resources for test run."""
 780         self._remove_plugins_extra()
 781         jobs_dir = os.path.join(
 782             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 783         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 784         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 785                         os.path.join(inst_dir, 'plugins'))
 786         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 787                         os.path.join(inst_dir, 'extra'))
 788
 789         task_name = self.task_yaml.get(test_name).get("task")
 790         task = os.path.join(jobs_dir, task_name)
 791         if not os.path.exists(task):
 792             raise Exception("The scenario '%s' does not exist." % task)
 793         LOGGER.debug('Scenario fetched from : %s', task)
 794
 795         if not os.path.exists(self.TEMP_DIR):
 796             os.makedirs(self.TEMP_DIR)
 797         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 798         self.apply_blacklist(task, task_file_name)
 799         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 800                          "--task-args", str(self.build_task_args(test_name))])
 801         return True
 802
 803     def clean(self):
 804         self._remove_plugins_extra()
 805         super(RallyJobs, self).clean()