functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_CONF_PATH = "/etc/rally/rally.conf"
  47     RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     RALLY_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     TEMPLATE_DIR = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     SUPPORT_DIR = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     USERS_AMOUNT = 2
  58     TENANTS_AMOUNT = 3
  59     ITERATIONS_AMOUNT = 10
  60     CONCURRENCY = 4
  61     VOLUME_VERSION = 3
  62     VOLUME_SERVICE_TYPE = "volumev3"
  63     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  64     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69
  70     def __init__(self, **kwargs):
  71         """Initialize RallyBase object."""
  72         super(RallyBase, self).__init__(**kwargs)
  73         assert self.orig_cloud
  74         assert self.project
  75         if self.orig_cloud.get_role("admin"):
  76             role_name = "admin"
  77         elif self.orig_cloud.get_role("Admin"):
  78             role_name = "Admin"
  79         else:
  80             raise Exception("Cannot detect neither admin nor Admin")
  81         self.orig_cloud.grant_role(
  82             role_name, user=self.project.user.id,
  83             project=self.project.project.id,
  84             domain=self.project.domain.id)
  85         self.results_dir = os.path.join(
  86             getattr(config.CONF, 'dir_results'), self.case_name)
  87         self.task_file = ''
  88         self.creators = []
  89         self.summary = []
  90         self.scenario_dir = ''
  91         self.smoke = None
  92         self.start_time = None
  93         self.result = None
  94         self.details = None
  95         self.compute_cnt = 0
  96         self.flavor_alt = None
  97         self.tests = []
  98         self.run_cmd = ''
  99         self.network_extensions = []
 100         self.services = []
 101
 102     def build_task_args(self, test_name):
 103         """Build arguments for the Rally task."""
 104         task_args = {'service_list': [test_name]}
 105         task_args['image_name'] = str(self.image.name)
 106         task_args['flavor_name'] = str(self.flavor.name)
 107         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 108         task_args['glance_image_location'] = str(self.filename)
 109         task_args['glance_image_format'] = str(self.image_format)
 110         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 111         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 112         task_args['users_amount'] = self.USERS_AMOUNT
 113         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 114         task_args['use_existing_users'] = False
 115         task_args['iterations'] = self.ITERATIONS_AMOUNT
 116         task_args['concurrency'] = self.CONCURRENCY
 117         task_args['smoke'] = self.smoke
 118         task_args['volume_version'] = self.VOLUME_VERSION
 119         task_args['volume_service_type'] = self.VOLUME_SERVICE_TYPE
 120         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 121
 122         if self.ext_net:
 123             task_args['floating_network'] = str(self.ext_net.name)
 124         else:
 125             task_args['floating_network'] = ''
 126
 127         if self.network:
 128             task_args['netid'] = str(self.network.id)
 129         else:
 130             task_args['netid'] = ''
 131
 132         return task_args
 133
 134     def _prepare_test_list(self, test_name):
 135         """Build the list of test cases to be executed."""
 136         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 137         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 138                                           test_yaml_file_name)
 139
 140         if not os.path.exists(scenario_file_name):
 141             scenario_file_name = os.path.join(self.scenario_dir,
 142                                               test_yaml_file_name)
 143
 144             if not os.path.exists(scenario_file_name):
 145                 raise Exception("The scenario '%s' does not exist."
 146                                 % scenario_file_name)
 147
 148         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 149         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 150
 151         if not os.path.exists(self.TEMP_DIR):
 152             os.makedirs(self.TEMP_DIR)
 153
 154         self.apply_blacklist(scenario_file_name, test_file_name)
 155         return test_file_name
 156
 157     @staticmethod
 158     def get_verifier_deployment_id():
 159         """
 160         Returns deployment id for active Rally deployment
 161         """
 162         cmd = ("rally deployment list | awk '/" +
 163                getattr(config.CONF, 'rally_deployment_name') +
 164                "/ {print $2}'")
 165         proc = subprocess.Popen(cmd, shell=True,
 166                                 stdout=subprocess.PIPE,
 167                                 stderr=subprocess.STDOUT)
 168         deployment_uuid = proc.stdout.readline().rstrip()
 169         return deployment_uuid.decode("utf-8")
 170
 171     @staticmethod
 172     def create_rally_deployment(environ=None):
 173         """Create new rally deployment"""
 174         # set the architecture to default
 175         pod_arch = env.get("POD_ARCH")
 176         arch_filter = ['aarch64']
 177
 178         if pod_arch and pod_arch in arch_filter:
 179             LOGGER.info("Apply aarch64 specific to rally config...")
 180             with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
 181                 rally_patch_conf = pfile.read()
 182
 183             for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
 184                 print(line, end=' ')
 185                 if "cirros|testvm" in line:
 186                     print(rally_patch_conf)
 187
 188         LOGGER.info("Creating Rally environment...")
 189         try:
 190             cmd = ['rally', 'deployment', 'destroy',
 191                    '--deployment',
 192                    str(getattr(config.CONF, 'rally_deployment_name'))]
 193             output = subprocess.check_output(cmd)
 194             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 195         except subprocess.CalledProcessError:
 196             pass
 197
 198         cmd = ['rally', 'deployment', 'create', '--fromenv',
 199                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 200         output = subprocess.check_output(cmd, env=environ)
 201         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 202
 203         cmd = ['rally', 'deployment', 'check']
 204         output = subprocess.check_output(cmd)
 205         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 206         return RallyBase.get_verifier_deployment_id()
 207
 208     @staticmethod
 209     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 210         """Set keystone_default_role in rally.conf"""
 211         if env.get("NEW_USER_ROLE").lower() != "member":
 212             rconfig = configparser.RawConfigParser()
 213             rconfig.read(rally_conf)
 214             if not rconfig.has_section('openstack'):
 215                 rconfig.add_section('openstack')
 216             rconfig.set(
 217                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 218             with open(rally_conf, 'w') as config_file:
 219                 rconfig.write(config_file)
 220
 221     @staticmethod
 222     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 223         """Clean Rally config"""
 224         if env.get("NEW_USER_ROLE").lower() != "member":
 225             rconfig = configparser.RawConfigParser()
 226             rconfig.read(rally_conf)
 227             if rconfig.has_option('openstack', 'keystone_default_role'):
 228                 rconfig.remove_option('openstack', 'keystone_default_role')
 229             with open(rally_conf, 'w') as config_file:
 230                 rconfig.write(config_file)
 231
 232     @staticmethod
 233     def get_task_id(cmd_raw):
 234         """
 235         Get task id from command rally result.
 236
 237         :param cmd_raw:
 238         :return: task_id as string
 239         """
 240         taskid_re = re.compile('^Task +(.*): started$')
 241         for line in cmd_raw.splitlines(True):
 242             line = line.strip()
 243             match = taskid_re.match(line.decode("utf-8"))
 244             if match:
 245                 return match.group(1)
 246         return None
 247
 248     @staticmethod
 249     def task_succeed(json_raw):
 250         """
 251         Parse JSON from rally JSON results.
 252
 253         :param json_raw:
 254         :return: Bool
 255         """
 256         rally_report = json.loads(json_raw)
 257         tasks = rally_report.get('tasks')
 258         if tasks:
 259             for task in tasks:
 260                 if task.get('status') != 'finished' or \
 261                    task.get('pass_sla') is not True:
 262                     return False
 263         else:
 264             return False
 265         return True
 266
 267     def _migration_supported(self):
 268         """Determine if migration is supported."""
 269         if self.compute_cnt > 1:
 270             return True
 271         return False
 272
 273     def _network_trunk_supported(self):
 274         """Determine if network trunk service is available"""
 275         if 'trunk' in self.network_extensions:
 276             return True
 277         return False
 278
 279     @staticmethod
 280     def excl_scenario():
 281         """Exclude scenario."""
 282         black_tests = []
 283         try:
 284             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 285                 black_list_yaml = yaml.safe_load(black_list_file)
 286
 287             deploy_scenario = env.get('DEPLOY_SCENARIO')
 288             if (bool(deploy_scenario) and
 289                     'scenario' in black_list_yaml.keys()):
 290                 for item in black_list_yaml['scenario']:
 291                     scenarios = item['scenarios']
 292                     in_it = RallyBase.in_iterable_re
 293                     if in_it(deploy_scenario, scenarios):
 294                         tests = item['tests']
 295                         black_tests.extend(tests)
 296         except Exception:  # pylint: disable=broad-except
 297             LOGGER.debug("Scenario exclusion not applied.")
 298
 299         return black_tests
 300
 301     @staticmethod
 302     def in_iterable_re(needle, haystack):
 303         """
 304         Check if given needle is in the iterable haystack, using regex.
 305
 306         :param needle: string to be matched
 307         :param haystack: iterable of strings (optionally regex patterns)
 308         :return: True if needle is eqial to any of the elements in haystack,
 309                  or if a nonempty regex pattern in haystack is found in needle.
 310         """
 311         # match without regex
 312         if needle in haystack:
 313             return True
 314
 315         for pattern in haystack:
 316             # match if regex pattern is set and found in the needle
 317             if pattern and re.search(pattern, needle) is not None:
 318                 return True
 319
 320         return False
 321
 322     def excl_func(self):
 323         """Exclude functionalities."""
 324         black_tests = []
 325         func_list = []
 326
 327         try:
 328             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 329                 black_list_yaml = yaml.safe_load(black_list_file)
 330
 331             if env.get('BLOCK_MIGRATION').lower() == 'true':
 332                 func_list.append("block_migration")
 333             if not self._migration_supported():
 334                 func_list.append("no_migration")
 335             if not self._network_trunk_supported():
 336                 func_list.append("no_net_trunk_service")
 337
 338             if 'functionality' in black_list_yaml.keys():
 339                 for item in black_list_yaml['functionality']:
 340                     functions = item['functions']
 341                     for func in func_list:
 342                         if func in functions:
 343                             tests = item['tests']
 344                             black_tests.extend(tests)
 345         except Exception:  # pylint: disable=broad-except
 346             LOGGER.debug("Functionality exclusion not applied.")
 347
 348         return black_tests
 349
 350     def apply_blacklist(self, case_file_name, result_file_name):
 351         """Apply blacklist."""
 352         LOGGER.debug("Applying blacklist...")
 353         cases_file = open(case_file_name, 'r')
 354         result_file = open(result_file_name, 'w')
 355
 356         black_tests = list(set(self.excl_func() +
 357                                self.excl_scenario()))
 358
 359         if black_tests:
 360             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 361
 362         include = True
 363         for cases_line in cases_file:
 364             if include:
 365                 for black_tests_line in black_tests:
 366                     if re.search(black_tests_line,
 367                                  cases_line.strip().rstrip(':')):
 368                         include = False
 369                         break
 370                 else:
 371                     result_file.write(str(cases_line))
 372             else:
 373                 if cases_line.isspace():
 374                     include = True
 375
 376         cases_file.close()
 377         result_file.close()
 378
 379     @staticmethod
 380     def file_is_empty(file_name):
 381         """Determine is a file is empty."""
 382         try:
 383             if os.stat(file_name).st_size > 0:
 384                 return False
 385         except Exception:  # pylint: disable=broad-except
 386             pass
 387
 388         return True
 389
 390     def _save_results(self, test_name, task_id):
 391         """ Generate and save task execution results"""
 392         # check for result directory and create it otherwise
 393         if not os.path.exists(self.results_dir):
 394             LOGGER.debug('%s does not exist, we create it.',
 395                          self.results_dir)
 396             os.makedirs(self.results_dir)
 397
 398         # put detailed result to log
 399         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 400         LOGGER.debug('running command: %s', cmd)
 401         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 402         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 403
 404         # save report as JSON
 405         report_json_name = '{}.json'.format(test_name)
 406         report_json_dir = os.path.join(self.results_dir, report_json_name)
 407         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 408                 "--out", report_json_dir])
 409         LOGGER.debug('running command: %s', cmd)
 410         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 411         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 412
 413         json_results = open(report_json_dir).read()
 414         self._append_summary(json_results, test_name)
 415
 416         # parse JSON operation result
 417         if self.task_succeed(json_results):
 418             LOGGER.info('Test scenario: "%s" OK.', test_name)
 419         else:
 420             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 421
 422     def run_task(self, test_name):
 423         """Run a task."""
 424         LOGGER.info('Starting test scenario "%s" ...', test_name)
 425         LOGGER.debug('running command: %s', self.run_cmd)
 426         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 427                                 stderr=subprocess.STDOUT)
 428         output = proc.communicate()[0]
 429
 430         task_id = self.get_task_id(output)
 431         LOGGER.debug('task_id : %s', task_id)
 432         if task_id is None:
 433             LOGGER.error("Failed to retrieve task_id")
 434             LOGGER.error("Result:\n%s", output)
 435             raise Exception("Failed to retrieve task id")
 436         self._save_results(test_name, task_id)
 437
 438     def _append_summary(self, json_raw, test_name):
 439         """Update statistics summary info."""
 440         nb_tests = 0
 441         nb_success = 0
 442         overall_duration = 0.0
 443         success = []
 444         failures = []
 445
 446         rally_report = json.loads(json_raw)
 447         for task in rally_report.get('tasks'):
 448             for subtask in task.get('subtasks'):
 449                 has_errors = False
 450                 for workload in subtask.get('workloads'):
 451                     if workload.get('full_duration'):
 452                         overall_duration += workload.get('full_duration')
 453
 454                     if workload.get('data'):
 455                         nb_tests += len(workload.get('data'))
 456
 457                     for result in workload.get('data'):
 458                         if not result.get('error'):
 459                             nb_success += 1
 460                         else:
 461                             has_errors = True
 462
 463                 if has_errors:
 464                     failures.append(subtask['title'])
 465                 else:
 466                     success.append(subtask['title'])
 467
 468         scenario_summary = {'test_name': test_name,
 469                             'overall_duration': overall_duration,
 470                             'nb_tests': nb_tests,
 471                             'nb_success': nb_success,
 472                             'success': success,
 473                             'failures': failures,
 474                             'task_status': self.task_succeed(json_raw)}
 475         self.summary.append(scenario_summary)
 476
 477     def prepare_run(self, **kwargs):
 478         """Prepare resources needed by test scenarios."""
 479         assert self.cloud
 480         LOGGER.debug('Validating run tests...')
 481         for test in kwargs.get('tests', self.TESTS):
 482             if test in self.TESTS:
 483                 self.tests.append(test)
 484             else:
 485                 raise Exception("Test name '%s' is invalid" % test)
 486
 487         if not os.path.exists(self.TASK_DIR):
 488             os.makedirs(self.TASK_DIR)
 489
 490         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 491         if not os.path.exists(task):
 492             LOGGER.error("Task file '%s' does not exist.", task)
 493             raise Exception("Task file '{}' does not exist.".
 494                             format(task))
 495         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 496         shutil.copyfile(task, self.task_file)
 497
 498         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 499         if not os.path.exists(task_macro):
 500             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 501             raise Exception("Task macro dir '{}' does not exist.".
 502                             format(task_macro))
 503         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 504         if os.path.exists(macro_dir):
 505             shutil.rmtree(macro_dir)
 506         shutil.copytree(task_macro, macro_dir)
 507
 508         self.update_keystone_default_role()
 509         self.compute_cnt = len(self.cloud.list_hypervisors())
 510         self.network_extensions = self.cloud.get_network_extensions()
 511         self.flavor_alt = self.create_flavor_alt()
 512         self.services = [service.name for service in
 513                          self.cloud.list_services()]
 514
 515         LOGGER.debug("flavor: %s", self.flavor_alt)
 516
 517     def prepare_task(self, test_name):
 518         """Prepare resources for test run."""
 519         file_name = self._prepare_test_list(test_name)
 520         if self.file_is_empty(file_name):
 521             LOGGER.info('No tests for scenario "%s"', test_name)
 522             return False
 523         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 524                          "--task", self.task_file, "--task-args",
 525                          str(self.build_task_args(test_name))])
 526         return True
 527
 528     def run_tests(self, **kwargs):
 529         """Execute tests."""
 530         optional = kwargs.get('optional', [])
 531         for test in self.tests:
 532             if test in self.services or test not in optional:
 533                 if self.prepare_task(test):
 534                     self.run_task(test)
 535
 536     def _generate_report(self):
 537         """Generate test execution summary report."""
 538         total_duration = 0.0
 539         total_nb_tests = 0
 540         total_nb_success = 0
 541         nb_modules = 0
 542         payload = []
 543
 544         res_table = prettytable.PrettyTable(
 545             padding_width=2,
 546             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 547         res_table.align['Module'] = "l"
 548         res_table.align['Duration'] = "r"
 549         res_table.align['Success'] = "r"
 550
 551         # for each scenario we draw a row for the table
 552         for item in self.summary:
 553             if item['task_status'] is True:
 554                 nb_modules += 1
 555             total_duration += item['overall_duration']
 556             total_nb_tests += item['nb_tests']
 557             total_nb_success += item['nb_success']
 558             try:
 559                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 560             except ZeroDivisionError:
 561                 success_avg = 0
 562             success_str = str("{:0.2f}".format(success_avg)) + '%'
 563             duration_str = time.strftime("%H:%M:%S",
 564                                          time.gmtime(item['overall_duration']))
 565             res_table.add_row([item['test_name'], duration_str,
 566                                item['nb_tests'], success_str])
 567             payload.append({'module': item['test_name'],
 568                             'details': {'duration': item['overall_duration'],
 569                                         'nb tests': item['nb_tests'],
 570                                         'success rate': success_str,
 571                                         'success': item['success'],
 572                                         'failures': item['failures']}})
 573
 574         total_duration_str = time.strftime("%H:%M:%S",
 575                                            time.gmtime(total_duration))
 576         try:
 577             self.result = 100 * total_nb_success / total_nb_tests
 578         except ZeroDivisionError:
 579             self.result = 100
 580         success_rate = "{:0.2f}".format(self.result)
 581         success_rate_str = str(success_rate) + '%'
 582         res_table.add_row(["", "", "", ""])
 583         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 584                            success_rate_str])
 585
 586         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 587         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 588                     self.case_name, success_rate, nb_modules,
 589                     len(self.summary))
 590         payload.append({'summary': {'duration': total_duration,
 591                                     'nb tests': total_nb_tests,
 592                                     'nb success': success_rate}})
 593         self.details = payload
 594
 595     @staticmethod
 596     def export_task(file_name, export_type="html"):
 597         """Export all task results (e.g. html or xunit report)
 598
 599         Raises:
 600             subprocess.CalledProcessError: if Rally doesn't return 0
 601
 602         Returns:
 603             None
 604         """
 605         cmd = ["rally", "task", "export", "--type", export_type,
 606                "--deployment",
 607                str(getattr(config.CONF, 'rally_deployment_name')),
 608                "--to", file_name]
 609         LOGGER.debug('running command: %s', cmd)
 610         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 611         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 612
 613     @staticmethod
 614     def verify_report(file_name, uuid, export_type="html"):
 615         """Generate the verifier report (e.g. html or xunit report)
 616
 617         Raises:
 618             subprocess.CalledProcessError: if Rally doesn't return 0
 619
 620         Returns:
 621             None
 622         """
 623         cmd = ["rally", "verify", "report", "--type", export_type,
 624                "--uuid", uuid, "--to", file_name]
 625         LOGGER.debug('running command: %s', cmd)
 626         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 627         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 628
 629     def clean(self):
 630         """Cleanup of OpenStack resources. Should be called on completion."""
 631         self.clean_rally_conf()
 632         self.clean_rally_logs()
 633         if self.flavor_alt:
 634             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 635         super(RallyBase, self).clean()
 636
 637     def is_successful(self):
 638         """The overall result of the test."""
 639         for item in self.summary:
 640             if item['task_status'] is False:
 641                 return testcase.TestCase.EX_TESTCASE_FAILED
 642
 643         return super(RallyBase, self).is_successful()
 644
 645     @staticmethod
 646     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 647         """Print rally logs in res dir"""
 648         if not os.path.exists(res_dir):
 649             os.makedirs(res_dir)
 650         rconfig = configparser.RawConfigParser()
 651         rconfig.read(rally_conf)
 652         rconfig.set('DEFAULT', 'debug', True)
 653         rconfig.set('DEFAULT', 'use_stderr', False)
 654         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 655         rconfig.set('DEFAULT', 'log_dir', res_dir)
 656         with open(rally_conf, 'w') as config_file:
 657             rconfig.write(config_file)
 658
 659     @staticmethod
 660     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 661         """Clean Rally config"""
 662         rconfig = configparser.RawConfigParser()
 663         rconfig.read(rally_conf)
 664         if rconfig.has_option('DEFAULT', 'use_stderr'):
 665             rconfig.remove_option('DEFAULT', 'use_stderr')
 666         if rconfig.has_option('DEFAULT', 'debug'):
 667             rconfig.remove_option('DEFAULT', 'debug')
 668         if rconfig.has_option('DEFAULT', 'log-file'):
 669             rconfig.remove_option('DEFAULT', 'log-file')
 670         if rconfig.has_option('DEFAULT', 'log_dir'):
 671             rconfig.remove_option('DEFAULT', 'log_dir')
 672         with open(rally_conf, 'w') as config_file:
 673             rconfig.write(config_file)
 674
 675     def run(self, **kwargs):
 676         """Run testcase."""
 677         self.start_time = time.time()
 678         try:
 679             assert super(RallyBase, self).run(
 680                 **kwargs) == testcase.TestCase.EX_OK
 681             self.update_rally_logs(self.res_dir)
 682             self.create_rally_deployment(environ=self.project.get_environ())
 683             self.prepare_run(**kwargs)
 684             self.run_tests(**kwargs)
 685             self._generate_report()
 686             self.export_task(
 687                 "{}/{}.html".format(self.results_dir, self.case_name))
 688             self.export_task(
 689                 "{}/{}.xml".format(self.results_dir, self.case_name),
 690                 export_type="junit-xml")
 691             res = testcase.TestCase.EX_OK
 692         except Exception:   # pylint: disable=broad-except
 693             LOGGER.exception('Error with run:')
 694             self.result = 0
 695             res = testcase.TestCase.EX_RUN_ERROR
 696         self.stop_time = time.time()
 697         return res
 698
 699
 700 class RallySanity(RallyBase):
 701     """Rally sanity testcase implementation."""
 702
 703     def __init__(self, **kwargs):
 704         """Initialize RallySanity object."""
 705         if "case_name" not in kwargs:
 706             kwargs["case_name"] = "rally_sanity"
 707         super(RallySanity, self).__init__(**kwargs)
 708         self.smoke = True
 709         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 710
 711
 712 class RallyFull(RallyBase):
 713     """Rally full testcase implementation."""
 714
 715     def __init__(self, **kwargs):
 716         """Initialize RallyFull object."""
 717         if "case_name" not in kwargs:
 718             kwargs["case_name"] = "rally_full"
 719         super(RallyFull, self).__init__(**kwargs)
 720         self.smoke = False
 721         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 722
 723
 724 class RallyJobs(RallyBase):
 725     """Rally OpenStack CI testcase implementation."""
 726
 727     TESTS = ["neutron"]
 728
 729     def __init__(self, **kwargs):
 730         """Initialize RallyJobs object."""
 731         if "case_name" not in kwargs:
 732             kwargs["case_name"] = "rally_jobs"
 733         super(RallyJobs, self).__init__(**kwargs)
 734         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 735         self.task_yaml = None
 736
 737     def prepare_run(self, **kwargs):
 738         """Create resources needed by test scenarios."""
 739         super(RallyJobs, self).prepare_run(**kwargs)
 740         with open(os.path.join(self.RALLY_DIR,
 741                                'rally_jobs.yaml'), 'r') as task_file:
 742             self.task_yaml = yaml.safe_load(task_file)
 743
 744         for task in self.task_yaml:
 745             if task not in self.tests:
 746                 raise Exception("Test '%s' not in '%s'" %
 747                                 (task, self.tests))
 748
 749     def apply_blacklist(self, case_file_name, result_file_name):
 750         # pylint: disable=too-many-branches
 751         """Apply blacklist."""
 752         LOGGER.debug("Applying blacklist...")
 753         black_tests = list(set(self.excl_func() +
 754                                self.excl_scenario()))
 755         if black_tests:
 756             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 757
 758         template = YAML(typ='jinja2')
 759         with open(case_file_name, 'r') as fname:
 760             cases = template.load(fname)
 761         if cases.get("version", 1) == 1:
 762             # scenarios in dictionary
 763             for name in cases.keys():
 764                 if self.in_iterable_re(name, black_tests):
 765                     cases.pop(name)
 766         else:
 767             # workloads in subtasks
 768             for sind, subtask in enumerate(cases.get('subtasks', [])):
 769                 idx = []
 770                 for wind, workload in enumerate(subtask.get('workloads', [])):
 771                     scenario = workload.get('scenario', {})
 772                     for name in scenario.keys():
 773                         if self.in_iterable_re(name, black_tests):
 774                             idx.append(wind)
 775                             break
 776                 for wind in reversed(idx):
 777                     cases['subtasks'][sind]['workloads'].pop(wind)
 778             # scenarios in subtasks
 779             idx = []
 780             for sind, subtask in enumerate(cases.get('subtasks', [])):
 781                 scenario = subtask.get('scenario', {})
 782                 for name in scenario.keys():
 783                     if self.in_iterable_re(name, black_tests):
 784                         idx.append(sind)
 785                         break
 786             for sind in reversed(idx):
 787                 cases['subtasks'].pop(sind)
 788
 789         with open(result_file_name, 'w') as fname:
 790             template.dump(cases, fname)
 791
 792     def build_task_args(self, test_name):
 793         """Build arguments for the Rally task."""
 794         task_args = {}
 795         if self.ext_net:
 796             task_args['floating_network'] = str(self.ext_net.name)
 797         else:
 798             task_args['floating_network'] = ''
 799         return task_args
 800
 801     @staticmethod
 802     def _remove_plugins_extra():
 803         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 804         try:
 805             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 806             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 807         except Exception:  # pylint: disable=broad-except
 808             pass
 809
 810     def prepare_task(self, test_name):
 811         """Prepare resources for test run."""
 812         self._remove_plugins_extra()
 813         jobs_dir = os.path.join(
 814             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 815         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 816         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 817                         os.path.join(inst_dir, 'plugins'))
 818         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 819                         os.path.join(inst_dir, 'extra'))
 820
 821         task_name = self.task_yaml.get(test_name).get("task")
 822         task = os.path.join(jobs_dir, task_name)
 823         if not os.path.exists(task):
 824             raise Exception("The scenario '%s' does not exist." % task)
 825         LOGGER.debug('Scenario fetched from : %s', task)
 826
 827         if not os.path.exists(self.TEMP_DIR):
 828             os.makedirs(self.TEMP_DIR)
 829         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 830         self.apply_blacklist(task, task_file_name)
 831         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 832                          "--task-args", str(self.build_task_args(test_name))])
 833         return True
 834
 835     def clean(self):
 836         self._remove_plugins_extra()
 837         super(RallyJobs, self).clean()