functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican']
  45
  46     rally_conf_path = "/etc/rally/rally.conf"
  47     rally_aar4_patch_path = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     rally_dir = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     rally_scenario_dir = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     template_dir = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     support_dir = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     users_amount = 2
  58     tenants_amount = 3
  59     iterations_amount = 10
  60     concurrency = 4
  61     volume_version = 3
  62     volume_service_type = "volumev3"
  63     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  64     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     temp_dir = os.path.join(task_dir, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69     allow_no_fip = True
  70     task_timeout = 3600
  71
  72     def __init__(self, **kwargs):
  73         """Initialize RallyBase object."""
  74         super(RallyBase, self).__init__(**kwargs)
  75         assert self.orig_cloud
  76         assert self.project
  77         if self.orig_cloud.get_role("admin"):
  78             role_name = "admin"
  79         elif self.orig_cloud.get_role("Admin"):
  80             role_name = "Admin"
  81         else:
  82             raise Exception("Cannot detect neither admin nor Admin")
  83         self.orig_cloud.grant_role(
  84             role_name, user=self.project.user.id,
  85             project=self.project.project.id,
  86             domain=self.project.domain.id)
  87         self.results_dir = os.path.join(
  88             getattr(config.CONF, 'dir_results'), self.case_name)
  89         self.task_file = ''
  90         self.creators = []
  91         self.summary = []
  92         self.scenario_dir = ''
  93         self.smoke = None
  94         self.start_time = None
  95         self.result = None
  96         self.details = None
  97         self.compute_cnt = 0
  98         self.flavor_alt = None
  99         self.tests = []
 100         self.run_cmd = ''
 101         self.network_extensions = []
 102         self.services = []
 103
 104     def build_task_args(self, test_name):
 105         """Build arguments for the Rally task."""
 106         task_args = {'service_list': [test_name]}
 107         task_args['image_name'] = str(self.image.name)
 108         task_args['flavor_name'] = str(self.flavor.name)
 109         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 110         task_args['glance_image_location'] = str(self.filename)
 111         task_args['glance_image_format'] = str(self.image_format)
 112         task_args['tmpl_dir'] = str(self.template_dir)
 113         task_args['sup_dir'] = str(self.support_dir)
 114         task_args['users_amount'] = self.users_amount
 115         task_args['tenants_amount'] = self.tenants_amount
 116         task_args['use_existing_users'] = False
 117         task_args['iterations'] = self.iterations_amount
 118         task_args['concurrency'] = self.concurrency
 119         task_args['smoke'] = self.smoke
 120         task_args['volume_version'] = self.volume_version
 121         task_args['volume_service_type'] = self.volume_service_type
 122         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 123
 124         if self.ext_net:
 125             task_args['floating_network'] = str(self.ext_net.name)
 126         else:
 127             task_args['floating_network'] = ''
 128
 129         if self.network:
 130             task_args['netid'] = str(self.network.id)
 131         else:
 132             task_args['netid'] = ''
 133
 134         return task_args
 135
 136     def _prepare_test_list(self, test_name):
 137         """Build the list of test cases to be executed."""
 138         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 139         scenario_file_name = os.path.join(self.rally_scenario_dir,
 140                                           test_yaml_file_name)
 141
 142         if not os.path.exists(scenario_file_name):
 143             scenario_file_name = os.path.join(self.scenario_dir,
 144                                               test_yaml_file_name)
 145
 146             if not os.path.exists(scenario_file_name):
 147                 raise Exception("The scenario '%s' does not exist."
 148                                 % scenario_file_name)
 149
 150         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 151         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 152
 153         if not os.path.exists(self.temp_dir):
 154             os.makedirs(self.temp_dir)
 155
 156         self.apply_blacklist(scenario_file_name, test_file_name)
 157         return test_file_name
 158
 159     @staticmethod
 160     def get_verifier_deployment_id():
 161         """
 162         Returns deployment id for active Rally deployment
 163         """
 164         cmd = ("rally deployment list | awk '/" +
 165                getattr(config.CONF, 'rally_deployment_name') +
 166                "/ {print $2}'")
 167         proc = subprocess.Popen(cmd, shell=True,
 168                                 stdout=subprocess.PIPE,
 169                                 stderr=subprocess.STDOUT)
 170         deployment_uuid = proc.stdout.readline().rstrip()
 171         return deployment_uuid.decode("utf-8")
 172
 173     @staticmethod
 174     def create_rally_deployment(environ=None):
 175         """Create new rally deployment"""
 176         # set the architecture to default
 177         pod_arch = env.get("POD_ARCH")
 178         arch_filter = ['aarch64']
 179
 180         if pod_arch and pod_arch in arch_filter:
 181             LOGGER.info("Apply aarch64 specific to rally config...")
 182             with open(RallyBase.rally_aar4_patch_path, "r") as pfile:
 183                 rally_patch_conf = pfile.read()
 184
 185             for line in fileinput.input(RallyBase.rally_conf_path):
 186                 print(line, end=' ')
 187                 if "cirros|testvm" in line:
 188                     print(rally_patch_conf)
 189
 190         LOGGER.info("Creating Rally environment...")
 191         try:
 192             cmd = ['rally', 'deployment', 'destroy',
 193                    '--deployment',
 194                    str(getattr(config.CONF, 'rally_deployment_name'))]
 195             output = subprocess.check_output(cmd)
 196             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 197         except subprocess.CalledProcessError:
 198             pass
 199
 200         cmd = ['rally', 'deployment', 'create', '--fromenv',
 201                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 202         output = subprocess.check_output(cmd, env=environ)
 203         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 204
 205         cmd = ['rally', 'deployment', 'check']
 206         output = subprocess.check_output(cmd)
 207         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 208         return RallyBase.get_verifier_deployment_id()
 209
 210     @staticmethod
 211     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 212         """Set keystone_default_role in rally.conf"""
 213         if env.get("NEW_USER_ROLE").lower() != "member":
 214             rconfig = configparser.RawConfigParser()
 215             rconfig.read(rally_conf)
 216             if not rconfig.has_section('openstack'):
 217                 rconfig.add_section('openstack')
 218             rconfig.set(
 219                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 220             with open(rally_conf, 'w') as config_file:
 221                 rconfig.write(config_file)
 222
 223     @staticmethod
 224     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 225         """Clean Rally config"""
 226         if env.get("NEW_USER_ROLE").lower() != "member":
 227             rconfig = configparser.RawConfigParser()
 228             rconfig.read(rally_conf)
 229             if rconfig.has_option('openstack', 'keystone_default_role'):
 230                 rconfig.remove_option('openstack', 'keystone_default_role')
 231             with open(rally_conf, 'w') as config_file:
 232                 rconfig.write(config_file)
 233
 234     @staticmethod
 235     def get_task_id(cmd_raw):
 236         """
 237         Get task id from command rally result.
 238
 239         :param cmd_raw:
 240         :return: task_id as string
 241         """
 242         taskid_re = re.compile('^Task +(.*): started$')
 243         for line in cmd_raw.splitlines(True):
 244             line = line.strip()
 245             match = taskid_re.match(line.decode("utf-8"))
 246             if match:
 247                 return match.group(1)
 248         return None
 249
 250     @staticmethod
 251     def task_succeed(json_raw):
 252         """
 253         Parse JSON from rally JSON results.
 254
 255         :param json_raw:
 256         :return: Bool
 257         """
 258         rally_report = json.loads(json_raw)
 259         tasks = rally_report.get('tasks')
 260         if tasks:
 261             for task in tasks:
 262                 if task.get('status') != 'finished' or \
 263                    task.get('pass_sla') is not True:
 264                     return False
 265         else:
 266             return False
 267         return True
 268
 269     def _migration_supported(self):
 270         """Determine if migration is supported."""
 271         if self.compute_cnt > 1:
 272             return True
 273         return False
 274
 275     def _network_trunk_supported(self):
 276         """Determine if network trunk service is available"""
 277         if 'trunk' in self.network_extensions:
 278             return True
 279         return False
 280
 281     @staticmethod
 282     def excl_scenario():
 283         """Exclude scenario."""
 284         black_tests = []
 285         try:
 286             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 287                 black_list_yaml = yaml.safe_load(black_list_file)
 288
 289             deploy_scenario = env.get('DEPLOY_SCENARIO')
 290             if (bool(deploy_scenario) and
 291                     'scenario' in black_list_yaml.keys()):
 292                 for item in black_list_yaml['scenario']:
 293                     scenarios = item['scenarios']
 294                     in_it = RallyBase.in_iterable_re
 295                     if in_it(deploy_scenario, scenarios):
 296                         tests = item['tests']
 297                         black_tests.extend(tests)
 298         except Exception:  # pylint: disable=broad-except
 299             LOGGER.debug("Scenario exclusion not applied.")
 300
 301         return black_tests
 302
 303     @staticmethod
 304     def in_iterable_re(needle, haystack):
 305         """
 306         Check if given needle is in the iterable haystack, using regex.
 307
 308         :param needle: string to be matched
 309         :param haystack: iterable of strings (optionally regex patterns)
 310         :return: True if needle is eqial to any of the elements in haystack,
 311                  or if a nonempty regex pattern in haystack is found in needle.
 312         """
 313         # match without regex
 314         if needle in haystack:
 315             return True
 316
 317         for pattern in haystack:
 318             # match if regex pattern is set and found in the needle
 319             if pattern and re.search(pattern, needle) is not None:
 320                 return True
 321
 322         return False
 323
 324     def excl_func(self):
 325         """Exclude functionalities."""
 326         black_tests = []
 327         func_list = []
 328
 329         try:
 330             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 331                 black_list_yaml = yaml.safe_load(black_list_file)
 332
 333             if env.get('BLOCK_MIGRATION').lower() == 'true':
 334                 func_list.append("block_migration")
 335             if not self._migration_supported():
 336                 func_list.append("no_migration")
 337             if not self._network_trunk_supported():
 338                 func_list.append("no_net_trunk_service")
 339             if not self.ext_net:
 340                 func_list.append("no_floating_ip")
 341
 342             if 'functionality' in black_list_yaml.keys():
 343                 for item in black_list_yaml['functionality']:
 344                     functions = item['functions']
 345                     for func in func_list:
 346                         if func in functions:
 347                             tests = item['tests']
 348                             black_tests.extend(tests)
 349         except Exception:  # pylint: disable=broad-except
 350             LOGGER.debug("Functionality exclusion not applied.")
 351
 352         return black_tests
 353
 354     def apply_blacklist(self, case_file_name, result_file_name):
 355         """Apply blacklist."""
 356         LOGGER.debug("Applying blacklist...")
 357         cases_file = open(case_file_name, 'r')
 358         result_file = open(result_file_name, 'w')
 359
 360         black_tests = list(set(self.excl_func() +
 361                                self.excl_scenario()))
 362
 363         if black_tests:
 364             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 365
 366         include = True
 367         for cases_line in cases_file:
 368             if include:
 369                 for black_tests_line in black_tests:
 370                     if re.search(black_tests_line,
 371                                  cases_line.strip().rstrip(':')):
 372                         include = False
 373                         break
 374                 else:
 375                     result_file.write(str(cases_line))
 376             else:
 377                 if cases_line.isspace():
 378                     include = True
 379
 380         cases_file.close()
 381         result_file.close()
 382
 383     @staticmethod
 384     def file_is_empty(file_name):
 385         """Determine is a file is empty."""
 386         try:
 387             if os.stat(file_name).st_size > 0:
 388                 return False
 389         except Exception:  # pylint: disable=broad-except
 390             pass
 391
 392         return True
 393
 394     def _save_results(self, test_name, task_id):
 395         """ Generate and save task execution results"""
 396         # check for result directory and create it otherwise
 397         if not os.path.exists(self.results_dir):
 398             LOGGER.debug('%s does not exist, we create it.',
 399                          self.results_dir)
 400             os.makedirs(self.results_dir)
 401
 402         # put detailed result to log
 403         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 404         LOGGER.debug('running command: %s', cmd)
 405         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 406         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 407
 408         # save report as JSON
 409         report_json_name = '{}.json'.format(test_name)
 410         report_json_dir = os.path.join(self.results_dir, report_json_name)
 411         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 412                 "--out", report_json_dir])
 413         LOGGER.debug('running command: %s', cmd)
 414         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 415         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 416
 417         json_results = open(report_json_dir).read()
 418         self._append_summary(json_results, test_name)
 419
 420         # parse JSON operation result
 421         if self.task_succeed(json_results):
 422             LOGGER.info('Test scenario: "%s" OK.', test_name)
 423         else:
 424             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 425
 426     def run_task(self, test_name):
 427         """Run a task."""
 428         LOGGER.info('Starting test scenario "%s" ...', test_name)
 429         LOGGER.debug('running command: %s', self.run_cmd)
 430         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 431                                 stderr=subprocess.STDOUT)
 432         try:
 433             output = proc.communicate(timeout=self.task_timeout)[0]
 434         except subprocess.TimeoutExpired:
 435             proc.kill()
 436             proc.communicate()
 437             LOGGER.error("Failed to complete run task")
 438             raise Exception("Failed to complete run task")
 439         task_id = self.get_task_id(output)
 440         LOGGER.debug('task_id : %s', task_id)
 441         if task_id is None:
 442             LOGGER.error("Failed to retrieve task_id")
 443             LOGGER.error("Result:\n%s", output.decode("utf-8"))
 444             raise Exception("Failed to retrieve task id")
 445         self._save_results(test_name, task_id)
 446
 447     def _append_summary(self, json_raw, test_name):
 448         # pylint: disable=too-many-locals
 449         """Update statistics summary info."""
 450         nb_tests = 0
 451         nb_success = 0
 452         overall_duration = 0.0
 453         success = []
 454         failures = []
 455
 456         rally_report = json.loads(json_raw)
 457         for task in rally_report.get('tasks'):
 458             for subtask in task.get('subtasks'):
 459                 has_errors = False
 460                 for workload in subtask.get('workloads'):
 461                     if workload.get('full_duration'):
 462                         overall_duration += workload.get('full_duration')
 463
 464                     if workload.get('data'):
 465                         nb_tests += len(workload.get('data'))
 466
 467                     for result in workload.get('data'):
 468                         if not result.get('error'):
 469                             nb_success += 1
 470                         else:
 471                             has_errors = True
 472
 473                 if has_errors:
 474                     failures.append(subtask['title'])
 475                 else:
 476                     success.append(subtask['title'])
 477
 478         scenario_summary = {'test_name': test_name,
 479                             'overall_duration': overall_duration,
 480                             'nb_tests': nb_tests,
 481                             'nb_success': nb_success,
 482                             'success': success,
 483                             'failures': failures,
 484                             'task_status': self.task_succeed(json_raw)}
 485         self.summary.append(scenario_summary)
 486
 487     def prepare_run(self, **kwargs):
 488         """Prepare resources needed by test scenarios."""
 489         assert self.cloud
 490         LOGGER.debug('Validating run tests...')
 491         for test in kwargs.get('tests', self.stests):
 492             if test in self.stests:
 493                 self.tests.append(test)
 494             else:
 495                 raise Exception("Test name '%s' is invalid" % test)
 496
 497         if not os.path.exists(self.task_dir):
 498             os.makedirs(self.task_dir)
 499
 500         task = os.path.join(self.rally_dir, 'task.yaml')
 501         if not os.path.exists(task):
 502             LOGGER.error("Task file '%s' does not exist.", task)
 503             raise Exception("Task file '{}' does not exist.".
 504                             format(task))
 505         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 506         shutil.copyfile(task, self.task_file)
 507
 508         task_macro = os.path.join(self.rally_dir, 'macro')
 509         if not os.path.exists(task_macro):
 510             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 511             raise Exception("Task macro dir '{}' does not exist.".
 512                             format(task_macro))
 513         macro_dir = os.path.join(self.task_dir, 'macro')
 514         if os.path.exists(macro_dir):
 515             shutil.rmtree(macro_dir)
 516         shutil.copytree(task_macro, macro_dir)
 517
 518         self.update_keystone_default_role()
 519         self.compute_cnt = len(self.cloud.list_hypervisors())
 520         self.network_extensions = self.cloud.get_network_extensions()
 521         self.flavor_alt = self.create_flavor_alt()
 522         self.services = [service.name for service in
 523                          self.cloud.list_services()]
 524
 525         LOGGER.debug("flavor: %s", self.flavor_alt)
 526
 527     def prepare_task(self, test_name):
 528         """Prepare resources for test run."""
 529         file_name = self._prepare_test_list(test_name)
 530         if self.file_is_empty(file_name):
 531             LOGGER.info('No tests for scenario "%s"', test_name)
 532             return False
 533         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 534                          "--task", self.task_file, "--task-args",
 535                          str(self.build_task_args(test_name))])
 536         return True
 537
 538     def run_tests(self, **kwargs):
 539         """Execute tests."""
 540         optional = kwargs.get('optional', [])
 541         for test in self.tests:
 542             if test in self.services or test not in optional:
 543                 if self.prepare_task(test):
 544                     self.run_task(test)
 545
 546     def _generate_report(self):
 547         """Generate test execution summary report."""
 548         total_duration = 0.0
 549         total_nb_tests = 0
 550         total_nb_success = 0
 551         nb_modules = 0
 552         payload = []
 553
 554         res_table = prettytable.PrettyTable(
 555             padding_width=2,
 556             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 557         res_table.align['Module'] = "l"
 558         res_table.align['Duration'] = "r"
 559         res_table.align['Success'] = "r"
 560
 561         # for each scenario we draw a row for the table
 562         for item in self.summary:
 563             if item['task_status'] is True:
 564                 nb_modules += 1
 565             total_duration += item['overall_duration']
 566             total_nb_tests += item['nb_tests']
 567             total_nb_success += item['nb_success']
 568             try:
 569                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 570             except ZeroDivisionError:
 571                 success_avg = 0
 572             success_str = str("{:0.2f}".format(success_avg)) + '%'
 573             duration_str = time.strftime("%H:%M:%S",
 574                                          time.gmtime(item['overall_duration']))
 575             res_table.add_row([item['test_name'], duration_str,
 576                                item['nb_tests'], success_str])
 577             payload.append({'module': item['test_name'],
 578                             'details': {'duration': item['overall_duration'],
 579                                         'nb tests': item['nb_tests'],
 580                                         'success rate': success_str,
 581                                         'success': item['success'],
 582                                         'failures': item['failures']}})
 583
 584         total_duration_str = time.strftime("%H:%M:%S",
 585                                            time.gmtime(total_duration))
 586         try:
 587             self.result = 100 * total_nb_success / total_nb_tests
 588         except ZeroDivisionError:
 589             self.result = 100
 590         success_rate = "{:0.2f}".format(self.result)
 591         success_rate_str = str(success_rate) + '%'
 592         res_table.add_row(["", "", "", ""])
 593         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 594                            success_rate_str])
 595
 596         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 597         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 598                     self.case_name, success_rate, nb_modules,
 599                     len(self.summary))
 600         payload.append({'summary': {'duration': total_duration,
 601                                     'nb tests': total_nb_tests,
 602                                     'nb success': success_rate}})
 603         self.details = payload
 604
 605     @staticmethod
 606     def export_task(file_name, export_type="html"):
 607         """Export all task results (e.g. html or xunit report)
 608
 609         Raises:
 610             subprocess.CalledProcessError: if Rally doesn't return 0
 611
 612         Returns:
 613             None
 614         """
 615         cmd = ["rally", "task", "export", "--type", export_type,
 616                "--deployment",
 617                str(getattr(config.CONF, 'rally_deployment_name')),
 618                "--to", file_name]
 619         LOGGER.debug('running command: %s', cmd)
 620         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 621         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 622
 623     @staticmethod
 624     def verify_report(file_name, uuid, export_type="html"):
 625         """Generate the verifier report (e.g. html or xunit report)
 626
 627         Raises:
 628             subprocess.CalledProcessError: if Rally doesn't return 0
 629
 630         Returns:
 631             None
 632         """
 633         cmd = ["rally", "verify", "report", "--type", export_type,
 634                "--uuid", uuid, "--to", file_name]
 635         LOGGER.debug('running command: %s', cmd)
 636         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 637         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 638
 639     def clean(self):
 640         """Cleanup of OpenStack resources. Should be called on completion."""
 641         self.clean_rally_conf()
 642         self.clean_rally_logs()
 643         if self.flavor_alt:
 644             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 645         super(RallyBase, self).clean()
 646
 647     def is_successful(self):
 648         """The overall result of the test."""
 649         for item in self.summary:
 650             if item['task_status'] is False:
 651                 return testcase.TestCase.EX_TESTCASE_FAILED
 652
 653         return super(RallyBase, self).is_successful()
 654
 655     @staticmethod
 656     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 657         """Print rally logs in res dir"""
 658         if not os.path.exists(res_dir):
 659             os.makedirs(res_dir)
 660         rconfig = configparser.RawConfigParser()
 661         rconfig.read(rally_conf)
 662         rconfig.set('DEFAULT', 'debug', True)
 663         rconfig.set('DEFAULT', 'use_stderr', False)
 664         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 665         rconfig.set('DEFAULT', 'log_dir', res_dir)
 666         with open(rally_conf, 'w') as config_file:
 667             rconfig.write(config_file)
 668
 669     @staticmethod
 670     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 671         """Clean Rally config"""
 672         rconfig = configparser.RawConfigParser()
 673         rconfig.read(rally_conf)
 674         if rconfig.has_option('DEFAULT', 'use_stderr'):
 675             rconfig.remove_option('DEFAULT', 'use_stderr')
 676         if rconfig.has_option('DEFAULT', 'debug'):
 677             rconfig.remove_option('DEFAULT', 'debug')
 678         if rconfig.has_option('DEFAULT', 'log-file'):
 679             rconfig.remove_option('DEFAULT', 'log-file')
 680         if rconfig.has_option('DEFAULT', 'log_dir'):
 681             rconfig.remove_option('DEFAULT', 'log_dir')
 682         with open(rally_conf, 'w') as config_file:
 683             rconfig.write(config_file)
 684
 685     def run(self, **kwargs):
 686         """Run testcase."""
 687         self.start_time = time.time()
 688         try:
 689             assert super(RallyBase, self).run(
 690                 **kwargs) == testcase.TestCase.EX_OK
 691             self.update_rally_logs(self.res_dir)
 692             self.create_rally_deployment(environ=self.project.get_environ())
 693             self.prepare_run(**kwargs)
 694             self.run_tests(**kwargs)
 695             self._generate_report()
 696             self.export_task(
 697                 "{}/{}.html".format(self.results_dir, self.case_name))
 698             self.export_task(
 699                 "{}/{}.xml".format(self.results_dir, self.case_name),
 700                 export_type="junit-xml")
 701             res = testcase.TestCase.EX_OK
 702         except Exception:   # pylint: disable=broad-except
 703             LOGGER.exception('Error with run:')
 704             self.result = 0
 705             res = testcase.TestCase.EX_RUN_ERROR
 706         self.stop_time = time.time()
 707         return res
 708
 709
 710 class RallySanity(RallyBase):
 711     """Rally sanity testcase implementation."""
 712
 713     def __init__(self, **kwargs):
 714         """Initialize RallySanity object."""
 715         if "case_name" not in kwargs:
 716             kwargs["case_name"] = "rally_sanity"
 717         super(RallySanity, self).__init__(**kwargs)
 718         self.smoke = True
 719         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 720
 721
 722 class RallyFull(RallyBase):
 723     """Rally full testcase implementation."""
 724
 725     task_timeout = 7200
 726
 727     def __init__(self, **kwargs):
 728         """Initialize RallyFull object."""
 729         if "case_name" not in kwargs:
 730             kwargs["case_name"] = "rally_full"
 731         super(RallyFull, self).__init__(**kwargs)
 732         self.smoke = False
 733         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 734
 735
 736 class RallyJobs(RallyBase):
 737     """Rally OpenStack CI testcase implementation."""
 738
 739     stests = ["neutron"]
 740     task_timeout = 7200
 741
 742     def __init__(self, **kwargs):
 743         """Initialize RallyJobs object."""
 744         if "case_name" not in kwargs:
 745             kwargs["case_name"] = "rally_jobs"
 746         super(RallyJobs, self).__init__(**kwargs)
 747         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 748         self.task_yaml = None
 749
 750     def prepare_run(self, **kwargs):
 751         """Create resources needed by test scenarios."""
 752         super(RallyJobs, self).prepare_run(**kwargs)
 753         with open(os.path.join(self.rally_dir,
 754                                'rally_jobs.yaml'), 'r') as task_file:
 755             self.task_yaml = yaml.safe_load(task_file)
 756
 757         for task in self.task_yaml:
 758             if task not in self.tests:
 759                 raise Exception("Test '%s' not in '%s'" %
 760                                 (task, self.tests))
 761
 762     def apply_blacklist(self, case_file_name, result_file_name):
 763         # pylint: disable=too-many-branches
 764         """Apply blacklist."""
 765         LOGGER.debug("Applying blacklist...")
 766         black_tests = list(set(self.excl_func() +
 767                                self.excl_scenario()))
 768         if black_tests:
 769             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 770
 771         template = YAML(typ='jinja2')
 772         with open(case_file_name, 'r') as fname:
 773             cases = template.load(fname)
 774         if cases.get("version", 1) == 1:
 775             # scenarios in dictionary
 776             for name in cases.keys():
 777                 if self.in_iterable_re(name, black_tests):
 778                     cases.pop(name)
 779         else:
 780             # workloads in subtasks
 781             for sind, subtask in reversed(list(
 782                     enumerate(cases.get('subtasks', [])))):
 783                 for wind, workload in reversed(list(
 784                         enumerate(subtask.get('workloads', [])))):
 785                     scenario = workload.get('scenario', {})
 786                     for name in scenario.keys():
 787                         if self.in_iterable_re(name, black_tests):
 788                             cases['subtasks'][sind]['workloads'].pop(wind)
 789                             break
 790                 if 'workloads' in cases['subtasks'][sind]:
 791                     if not cases['subtasks'][sind]['workloads']:
 792                         cases['subtasks'].pop(sind)
 793             # scenarios in subtasks
 794             for sind, subtask in reversed(list(
 795                     enumerate(cases.get('subtasks', [])))):
 796                 scenario = subtask.get('scenario', {})
 797                 for name in scenario.keys():
 798                     if self.in_iterable_re(name, black_tests):
 799                         cases['subtasks'].pop(sind)
 800                         break
 801
 802         with open(result_file_name, 'w') as fname:
 803             template.dump(cases, fname)
 804
 805     def build_task_args(self, test_name):
 806         """Build arguments for the Rally task."""
 807         task_args = {}
 808         if self.ext_net:
 809             task_args['floating_network'] = str(self.ext_net.name)
 810         else:
 811             task_args['floating_network'] = ''
 812         task_args['image_name'] = str(self.image.name)
 813         task_args['flavor_name'] = str(self.flavor.name)
 814         return task_args
 815
 816     @staticmethod
 817     def _remove_plugins_extra():
 818         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 819         try:
 820             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 821             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 822         except Exception:  # pylint: disable=broad-except
 823             pass
 824
 825     def prepare_task(self, test_name):
 826         """Prepare resources for test run."""
 827         self._remove_plugins_extra()
 828         jobs_dir = os.path.join(
 829             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 830         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 831         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 832                         os.path.join(inst_dir, 'plugins'))
 833         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 834                         os.path.join(inst_dir, 'extra'))
 835
 836         task_name = self.task_yaml.get(test_name).get("task")
 837         task = os.path.join(jobs_dir, task_name)
 838         if not os.path.exists(task):
 839             raise Exception("The scenario '%s' does not exist." % task)
 840         LOGGER.debug('Scenario fetched from : %s', task)
 841
 842         if not os.path.exists(self.temp_dir):
 843             os.makedirs(self.temp_dir)
 844         task_file_name = os.path.join(self.temp_dir, task_name)
 845         self.apply_blacklist(task, task_file_name)
 846         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 847                          "--task-args", str(self.build_task_args(test_name))])
 848         return True
 849
 850     def clean(self):
 851         self._remove_plugins_extra()
 852         super(RallyJobs, self).clean()