functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican']
  45
  46     rally_conf_path = "/etc/rally/rally.conf"
  47     rally_aar4_patch_path = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     rally_dir = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     rally_scenario_dir = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     template_dir = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     support_dir = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     users_amount = 2
  58     tenants_amount = 3
  59     iterations_amount = 10
  60     concurrency = 4
  61     volume_version = 3
  62     volume_service_type = "volumev3"
  63     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  64     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     temp_dir = os.path.join(task_dir, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69     allow_no_fip = True
  70     task_timeout = '3600'
  71
  72     def __init__(self, **kwargs):
  73         """Initialize RallyBase object."""
  74         super(RallyBase, self).__init__(**kwargs)
  75         assert self.orig_cloud
  76         assert self.project
  77         if self.orig_cloud.get_role("admin"):
  78             role_name = "admin"
  79         elif self.orig_cloud.get_role("Admin"):
  80             role_name = "Admin"
  81         else:
  82             raise Exception("Cannot detect neither admin nor Admin")
  83         self.orig_cloud.grant_role(
  84             role_name, user=self.project.user.id,
  85             project=self.project.project.id,
  86             domain=self.project.domain.id)
  87         self.results_dir = os.path.join(
  88             getattr(config.CONF, 'dir_results'), self.case_name)
  89         self.task_file = ''
  90         self.creators = []
  91         self.summary = []
  92         self.scenario_dir = ''
  93         self.smoke = None
  94         self.start_time = None
  95         self.result = None
  96         self.details = None
  97         self.compute_cnt = 0
  98         self.flavor_alt = None
  99         self.tests = []
 100         self.run_cmd = ''
 101         self.network_extensions = []
 102         self.services = []
 103
 104     def build_task_args(self, test_name):
 105         """Build arguments for the Rally task."""
 106         task_args = {'service_list': [test_name]}
 107         task_args['image_name'] = str(self.image.name)
 108         task_args['flavor_name'] = str(self.flavor.name)
 109         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 110         task_args['glance_image_location'] = str(self.filename)
 111         task_args['glance_image_format'] = str(self.image_format)
 112         task_args['tmpl_dir'] = str(self.template_dir)
 113         task_args['sup_dir'] = str(self.support_dir)
 114         task_args['users_amount'] = self.users_amount
 115         task_args['tenants_amount'] = self.tenants_amount
 116         task_args['use_existing_users'] = False
 117         task_args['iterations'] = self.iterations_amount
 118         task_args['concurrency'] = self.concurrency
 119         task_args['smoke'] = self.smoke
 120         task_args['volume_version'] = self.volume_version
 121         task_args['volume_service_type'] = self.volume_service_type
 122         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 123
 124         if self.ext_net:
 125             task_args['floating_network'] = str(self.ext_net.name)
 126         else:
 127             task_args['floating_network'] = ''
 128
 129         if self.network:
 130             task_args['netid'] = str(self.network.id)
 131         else:
 132             task_args['netid'] = ''
 133
 134         return task_args
 135
 136     def _prepare_test_list(self, test_name):
 137         """Build the list of test cases to be executed."""
 138         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 139         scenario_file_name = os.path.join(self.rally_scenario_dir,
 140                                           test_yaml_file_name)
 141
 142         if not os.path.exists(scenario_file_name):
 143             scenario_file_name = os.path.join(self.scenario_dir,
 144                                               test_yaml_file_name)
 145
 146             if not os.path.exists(scenario_file_name):
 147                 raise Exception("The scenario '%s' does not exist."
 148                                 % scenario_file_name)
 149
 150         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 151         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 152
 153         if not os.path.exists(self.temp_dir):
 154             os.makedirs(self.temp_dir)
 155
 156         self.apply_blacklist(scenario_file_name, test_file_name)
 157         return test_file_name
 158
 159     @staticmethod
 160     def get_verifier_deployment_id():
 161         """
 162         Returns deployment id for active Rally deployment
 163         """
 164         cmd = ("rally deployment list | awk '/" +
 165                getattr(config.CONF, 'rally_deployment_name') +
 166                "/ {print $2}'")
 167         proc = subprocess.Popen(cmd, shell=True,
 168                                 stdout=subprocess.PIPE,
 169                                 stderr=subprocess.STDOUT)
 170         deployment_uuid = proc.stdout.readline().rstrip()
 171         return deployment_uuid.decode("utf-8")
 172
 173     @staticmethod
 174     def create_rally_deployment(environ=None):
 175         """Create new rally deployment"""
 176         # set the architecture to default
 177         pod_arch = env.get("POD_ARCH")
 178         arch_filter = ['aarch64']
 179
 180         if pod_arch and pod_arch in arch_filter:
 181             LOGGER.info("Apply aarch64 specific to rally config...")
 182             with open(RallyBase.rally_aar4_patch_path, "r") as pfile:
 183                 rally_patch_conf = pfile.read()
 184
 185             for line in fileinput.input(RallyBase.rally_conf_path):
 186                 print(line, end=' ')
 187                 if "cirros|testvm" in line:
 188                     print(rally_patch_conf)
 189
 190         LOGGER.info("Creating Rally environment...")
 191         try:
 192             cmd = ['rally', 'deployment', 'destroy',
 193                    '--deployment',
 194                    str(getattr(config.CONF, 'rally_deployment_name'))]
 195             output = subprocess.check_output(cmd)
 196             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 197         except subprocess.CalledProcessError:
 198             pass
 199
 200         cmd = ['rally', 'deployment', 'create', '--fromenv',
 201                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 202         output = subprocess.check_output(cmd, env=environ)
 203         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 204
 205         cmd = ['rally', 'deployment', 'check']
 206         output = subprocess.check_output(cmd)
 207         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 208         return RallyBase.get_verifier_deployment_id()
 209
 210     @staticmethod
 211     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 212         """Set keystone_default_role in rally.conf"""
 213         if env.get("NEW_USER_ROLE").lower() != "member":
 214             rconfig = configparser.RawConfigParser()
 215             rconfig.read(rally_conf)
 216             if not rconfig.has_section('openstack'):
 217                 rconfig.add_section('openstack')
 218             rconfig.set(
 219                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 220             with open(rally_conf, 'w') as config_file:
 221                 rconfig.write(config_file)
 222
 223     @staticmethod
 224     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 225         """Clean Rally config"""
 226         if env.get("NEW_USER_ROLE").lower() != "member":
 227             rconfig = configparser.RawConfigParser()
 228             rconfig.read(rally_conf)
 229             if rconfig.has_option('openstack', 'keystone_default_role'):
 230                 rconfig.remove_option('openstack', 'keystone_default_role')
 231             with open(rally_conf, 'w') as config_file:
 232                 rconfig.write(config_file)
 233
 234     @staticmethod
 235     def get_task_id(cmd_raw):
 236         """
 237         Get task id from command rally result.
 238
 239         :param cmd_raw:
 240         :return: task_id as string
 241         """
 242         taskid_re = re.compile('^Task +(.*): started$')
 243         for line in cmd_raw.splitlines(True):
 244             line = line.strip()
 245             match = taskid_re.match(line.decode("utf-8"))
 246             if match:
 247                 return match.group(1)
 248         return None
 249
 250     @staticmethod
 251     def task_succeed(json_raw):
 252         """
 253         Parse JSON from rally JSON results.
 254
 255         :param json_raw:
 256         :return: Bool
 257         """
 258         rally_report = json.loads(json_raw)
 259         tasks = rally_report.get('tasks')
 260         if tasks:
 261             for task in tasks:
 262                 if task.get('status') != 'finished' or \
 263                    task.get('pass_sla') is not True:
 264                     return False
 265         else:
 266             return False
 267         return True
 268
 269     def _migration_supported(self):
 270         """Determine if migration is supported."""
 271         if self.compute_cnt > 1:
 272             return True
 273         return False
 274
 275     def _network_trunk_supported(self):
 276         """Determine if network trunk service is available"""
 277         if 'trunk' in self.network_extensions:
 278             return True
 279         return False
 280
 281     @staticmethod
 282     def excl_scenario():
 283         """Exclude scenario."""
 284         black_tests = []
 285         try:
 286             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 287                 black_list_yaml = yaml.safe_load(black_list_file)
 288
 289             deploy_scenario = env.get('DEPLOY_SCENARIO')
 290             if (bool(deploy_scenario) and
 291                     'scenario' in black_list_yaml.keys()):
 292                 for item in black_list_yaml['scenario']:
 293                     scenarios = item['scenarios']
 294                     in_it = RallyBase.in_iterable_re
 295                     if in_it(deploy_scenario, scenarios):
 296                         tests = item['tests']
 297                         black_tests.extend(tests)
 298         except Exception:  # pylint: disable=broad-except
 299             LOGGER.debug("Scenario exclusion not applied.")
 300
 301         return black_tests
 302
 303     @staticmethod
 304     def in_iterable_re(needle, haystack):
 305         """
 306         Check if given needle is in the iterable haystack, using regex.
 307
 308         :param needle: string to be matched
 309         :param haystack: iterable of strings (optionally regex patterns)
 310         :return: True if needle is eqial to any of the elements in haystack,
 311                  or if a nonempty regex pattern in haystack is found in needle.
 312         """
 313         # match without regex
 314         if needle in haystack:
 315             return True
 316
 317         for pattern in haystack:
 318             # match if regex pattern is set and found in the needle
 319             if pattern and re.search(pattern, needle) is not None:
 320                 return True
 321
 322         return False
 323
 324     def excl_func(self):
 325         """Exclude functionalities."""
 326         black_tests = []
 327         func_list = []
 328
 329         try:
 330             with open(RallyBase.blacklist_file, 'r') as black_list_file:
 331                 black_list_yaml = yaml.safe_load(black_list_file)
 332
 333             if env.get('BLOCK_MIGRATION').lower() == 'true':
 334                 func_list.append("block_migration")
 335             if not self._migration_supported():
 336                 func_list.append("no_migration")
 337             if not self._network_trunk_supported():
 338                 func_list.append("no_net_trunk_service")
 339             if not self.ext_net:
 340                 func_list.append("no_floating_ip")
 341
 342             if 'functionality' in black_list_yaml.keys():
 343                 for item in black_list_yaml['functionality']:
 344                     functions = item['functions']
 345                     for func in func_list:
 346                         if func in functions:
 347                             tests = item['tests']
 348                             black_tests.extend(tests)
 349         except Exception:  # pylint: disable=broad-except
 350             LOGGER.debug("Functionality exclusion not applied.")
 351
 352         return black_tests
 353
 354     def apply_blacklist(self, case_file_name, result_file_name):
 355         """Apply blacklist."""
 356         LOGGER.debug("Applying blacklist...")
 357         cases_file = open(case_file_name, 'r')
 358         result_file = open(result_file_name, 'w')
 359
 360         black_tests = list(set(self.excl_func() +
 361                                self.excl_scenario()))
 362
 363         if black_tests:
 364             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 365
 366         include = True
 367         for cases_line in cases_file:
 368             if include:
 369                 for black_tests_line in black_tests:
 370                     if re.search(black_tests_line,
 371                                  cases_line.strip().rstrip(':')):
 372                         include = False
 373                         break
 374                 else:
 375                     result_file.write(str(cases_line))
 376             else:
 377                 if cases_line.isspace():
 378                     include = True
 379
 380         cases_file.close()
 381         result_file.close()
 382
 383     @staticmethod
 384     def file_is_empty(file_name):
 385         """Determine is a file is empty."""
 386         try:
 387             if os.stat(file_name).st_size > 0:
 388                 return False
 389         except Exception:  # pylint: disable=broad-except
 390             pass
 391
 392         return True
 393
 394     def _save_results(self, test_name, task_id):
 395         """ Generate and save task execution results"""
 396         # check for result directory and create it otherwise
 397         if not os.path.exists(self.results_dir):
 398             LOGGER.debug('%s does not exist, we create it.',
 399                          self.results_dir)
 400             os.makedirs(self.results_dir)
 401
 402         # put detailed result to log
 403         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 404         LOGGER.debug('running command: %s', cmd)
 405         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 406         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 407
 408         # save report as JSON
 409         report_json_name = '{}.json'.format(test_name)
 410         report_json_dir = os.path.join(self.results_dir, report_json_name)
 411         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 412                 "--out", report_json_dir])
 413         LOGGER.debug('running command: %s', cmd)
 414         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 415         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 416
 417         json_results = open(report_json_dir).read()
 418         self._append_summary(json_results, test_name)
 419
 420         # parse JSON operation result
 421         if self.task_succeed(json_results):
 422             LOGGER.info('Test scenario: "%s" OK.', test_name)
 423         else:
 424             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 425
 426     def run_task(self, test_name):
 427         """Run a task."""
 428         LOGGER.info('Starting test scenario "%s" ...', test_name)
 429         LOGGER.debug('running command: %s', self.run_cmd)
 430         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 431                                 stderr=subprocess.STDOUT)
 432         output = proc.communicate()[0]
 433
 434         task_id = self.get_task_id(output)
 435         LOGGER.debug('task_id : %s', task_id)
 436         if task_id is None:
 437             LOGGER.error("Failed to retrieve task_id")
 438             LOGGER.error("Result:\n%s", output.decode("utf-8"))
 439             raise Exception("Failed to retrieve task id")
 440         self._save_results(test_name, task_id)
 441
 442     def _append_summary(self, json_raw, test_name):
 443         # pylint: disable=too-many-locals
 444         """Update statistics summary info."""
 445         nb_tests = 0
 446         nb_success = 0
 447         overall_duration = 0.0
 448         success = []
 449         failures = []
 450
 451         rally_report = json.loads(json_raw)
 452         for task in rally_report.get('tasks'):
 453             for subtask in task.get('subtasks'):
 454                 has_errors = False
 455                 for workload in subtask.get('workloads'):
 456                     if workload.get('full_duration'):
 457                         overall_duration += workload.get('full_duration')
 458
 459                     if workload.get('data'):
 460                         nb_tests += len(workload.get('data'))
 461
 462                     for result in workload.get('data'):
 463                         if not result.get('error'):
 464                             nb_success += 1
 465                         else:
 466                             has_errors = True
 467
 468                 if has_errors:
 469                     failures.append(subtask['title'])
 470                 else:
 471                     success.append(subtask['title'])
 472
 473         scenario_summary = {'test_name': test_name,
 474                             'overall_duration': overall_duration,
 475                             'nb_tests': nb_tests,
 476                             'nb_success': nb_success,
 477                             'success': success,
 478                             'failures': failures,
 479                             'task_status': self.task_succeed(json_raw)}
 480         self.summary.append(scenario_summary)
 481
 482     def prepare_run(self, **kwargs):
 483         """Prepare resources needed by test scenarios."""
 484         assert self.cloud
 485         LOGGER.debug('Validating run tests...')
 486         for test in kwargs.get('tests', self.stests):
 487             if test in self.stests:
 488                 self.tests.append(test)
 489             else:
 490                 raise Exception("Test name '%s' is invalid" % test)
 491
 492         if not os.path.exists(self.task_dir):
 493             os.makedirs(self.task_dir)
 494
 495         task = os.path.join(self.rally_dir, 'task.yaml')
 496         if not os.path.exists(task):
 497             LOGGER.error("Task file '%s' does not exist.", task)
 498             raise Exception("Task file '{}' does not exist.".
 499                             format(task))
 500         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 501         shutil.copyfile(task, self.task_file)
 502
 503         task_macro = os.path.join(self.rally_dir, 'macro')
 504         if not os.path.exists(task_macro):
 505             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 506             raise Exception("Task macro dir '{}' does not exist.".
 507                             format(task_macro))
 508         macro_dir = os.path.join(self.task_dir, 'macro')
 509         if os.path.exists(macro_dir):
 510             shutil.rmtree(macro_dir)
 511         shutil.copytree(task_macro, macro_dir)
 512
 513         self.update_keystone_default_role()
 514         self.compute_cnt = len(self.cloud.list_hypervisors())
 515         self.network_extensions = self.cloud.get_network_extensions()
 516         self.flavor_alt = self.create_flavor_alt()
 517         self.services = [service.name for service in
 518                          self.cloud.list_services()]
 519
 520         LOGGER.debug("flavor: %s", self.flavor_alt)
 521
 522     def prepare_task(self, test_name):
 523         """Prepare resources for test run."""
 524         file_name = self._prepare_test_list(test_name)
 525         if self.file_is_empty(file_name):
 526             LOGGER.info('No tests for scenario "%s"', test_name)
 527             return False
 528         self.run_cmd = (["timeout", self.task_timeout,
 529                          "rally", "task", "start", "--abort-on-sla-failure",
 530                          "--task", self.task_file, "--task-args",
 531                          str(self.build_task_args(test_name))])
 532         return True
 533
 534     def run_tests(self, **kwargs):
 535         """Execute tests."""
 536         optional = kwargs.get('optional', [])
 537         for test in self.tests:
 538             if test in self.services or test not in optional:
 539                 if self.prepare_task(test):
 540                     self.run_task(test)
 541
 542     def _generate_report(self):
 543         """Generate test execution summary report."""
 544         total_duration = 0.0
 545         total_nb_tests = 0
 546         total_nb_success = 0
 547         nb_modules = 0
 548         payload = []
 549
 550         res_table = prettytable.PrettyTable(
 551             padding_width=2,
 552             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 553         res_table.align['Module'] = "l"
 554         res_table.align['Duration'] = "r"
 555         res_table.align['Success'] = "r"
 556
 557         # for each scenario we draw a row for the table
 558         for item in self.summary:
 559             if item['task_status'] is True:
 560                 nb_modules += 1
 561             total_duration += item['overall_duration']
 562             total_nb_tests += item['nb_tests']
 563             total_nb_success += item['nb_success']
 564             try:
 565                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 566             except ZeroDivisionError:
 567                 success_avg = 0
 568             success_str = str("{:0.2f}".format(success_avg)) + '%'
 569             duration_str = time.strftime("%H:%M:%S",
 570                                          time.gmtime(item['overall_duration']))
 571             res_table.add_row([item['test_name'], duration_str,
 572                                item['nb_tests'], success_str])
 573             payload.append({'module': item['test_name'],
 574                             'details': {'duration': item['overall_duration'],
 575                                         'nb tests': item['nb_tests'],
 576                                         'success rate': success_str,
 577                                         'success': item['success'],
 578                                         'failures': item['failures']}})
 579
 580         total_duration_str = time.strftime("%H:%M:%S",
 581                                            time.gmtime(total_duration))
 582         try:
 583             self.result = 100 * total_nb_success / total_nb_tests
 584         except ZeroDivisionError:
 585             self.result = 100
 586         success_rate = "{:0.2f}".format(self.result)
 587         success_rate_str = str(success_rate) + '%'
 588         res_table.add_row(["", "", "", ""])
 589         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 590                            success_rate_str])
 591
 592         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 593         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 594                     self.case_name, success_rate, nb_modules,
 595                     len(self.summary))
 596         payload.append({'summary': {'duration': total_duration,
 597                                     'nb tests': total_nb_tests,
 598                                     'nb success': success_rate}})
 599         self.details = payload
 600
 601     @staticmethod
 602     def export_task(file_name, export_type="html"):
 603         """Export all task results (e.g. html or xunit report)
 604
 605         Raises:
 606             subprocess.CalledProcessError: if Rally doesn't return 0
 607
 608         Returns:
 609             None
 610         """
 611         cmd = ["rally", "task", "export", "--type", export_type,
 612                "--deployment",
 613                str(getattr(config.CONF, 'rally_deployment_name')),
 614                "--to", file_name]
 615         LOGGER.debug('running command: %s', cmd)
 616         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 617         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 618
 619     @staticmethod
 620     def verify_report(file_name, uuid, export_type="html"):
 621         """Generate the verifier report (e.g. html or xunit report)
 622
 623         Raises:
 624             subprocess.CalledProcessError: if Rally doesn't return 0
 625
 626         Returns:
 627             None
 628         """
 629         cmd = ["rally", "verify", "report", "--type", export_type,
 630                "--uuid", uuid, "--to", file_name]
 631         LOGGER.debug('running command: %s', cmd)
 632         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 633         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 634
 635     def clean(self):
 636         """Cleanup of OpenStack resources. Should be called on completion."""
 637         self.clean_rally_conf()
 638         self.clean_rally_logs()
 639         if self.flavor_alt:
 640             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 641         super(RallyBase, self).clean()
 642
 643     def is_successful(self):
 644         """The overall result of the test."""
 645         for item in self.summary:
 646             if item['task_status'] is False:
 647                 return testcase.TestCase.EX_TESTCASE_FAILED
 648
 649         return super(RallyBase, self).is_successful()
 650
 651     @staticmethod
 652     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 653         """Print rally logs in res dir"""
 654         if not os.path.exists(res_dir):
 655             os.makedirs(res_dir)
 656         rconfig = configparser.RawConfigParser()
 657         rconfig.read(rally_conf)
 658         rconfig.set('DEFAULT', 'debug', True)
 659         rconfig.set('DEFAULT', 'use_stderr', False)
 660         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 661         rconfig.set('DEFAULT', 'log_dir', res_dir)
 662         with open(rally_conf, 'w') as config_file:
 663             rconfig.write(config_file)
 664
 665     @staticmethod
 666     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 667         """Clean Rally config"""
 668         rconfig = configparser.RawConfigParser()
 669         rconfig.read(rally_conf)
 670         if rconfig.has_option('DEFAULT', 'use_stderr'):
 671             rconfig.remove_option('DEFAULT', 'use_stderr')
 672         if rconfig.has_option('DEFAULT', 'debug'):
 673             rconfig.remove_option('DEFAULT', 'debug')
 674         if rconfig.has_option('DEFAULT', 'log-file'):
 675             rconfig.remove_option('DEFAULT', 'log-file')
 676         if rconfig.has_option('DEFAULT', 'log_dir'):
 677             rconfig.remove_option('DEFAULT', 'log_dir')
 678         with open(rally_conf, 'w') as config_file:
 679             rconfig.write(config_file)
 680
 681     def run(self, **kwargs):
 682         """Run testcase."""
 683         self.start_time = time.time()
 684         try:
 685             assert super(RallyBase, self).run(
 686                 **kwargs) == testcase.TestCase.EX_OK
 687             self.update_rally_logs(self.res_dir)
 688             self.create_rally_deployment(environ=self.project.get_environ())
 689             self.prepare_run(**kwargs)
 690             self.run_tests(**kwargs)
 691             self._generate_report()
 692             self.export_task(
 693                 "{}/{}.html".format(self.results_dir, self.case_name))
 694             self.export_task(
 695                 "{}/{}.xml".format(self.results_dir, self.case_name),
 696                 export_type="junit-xml")
 697             res = testcase.TestCase.EX_OK
 698         except Exception:   # pylint: disable=broad-except
 699             LOGGER.exception('Error with run:')
 700             self.result = 0
 701             res = testcase.TestCase.EX_RUN_ERROR
 702         self.stop_time = time.time()
 703         return res
 704
 705
 706 class RallySanity(RallyBase):
 707     """Rally sanity testcase implementation."""
 708
 709     def __init__(self, **kwargs):
 710         """Initialize RallySanity object."""
 711         if "case_name" not in kwargs:
 712             kwargs["case_name"] = "rally_sanity"
 713         super(RallySanity, self).__init__(**kwargs)
 714         self.smoke = True
 715         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 716
 717
 718 class RallyFull(RallyBase):
 719     """Rally full testcase implementation."""
 720
 721     def __init__(self, **kwargs):
 722         """Initialize RallyFull object."""
 723         if "case_name" not in kwargs:
 724             kwargs["case_name"] = "rally_full"
 725         super(RallyFull, self).__init__(**kwargs)
 726         self.smoke = False
 727         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 728
 729
 730 class RallyJobs(RallyBase):
 731     """Rally OpenStack CI testcase implementation."""
 732
 733     stests = ["neutron"]
 734     task_timeout = '7200'
 735
 736     def __init__(self, **kwargs):
 737         """Initialize RallyJobs object."""
 738         if "case_name" not in kwargs:
 739             kwargs["case_name"] = "rally_jobs"
 740         super(RallyJobs, self).__init__(**kwargs)
 741         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 742         self.task_yaml = None
 743
 744     def prepare_run(self, **kwargs):
 745         """Create resources needed by test scenarios."""
 746         super(RallyJobs, self).prepare_run(**kwargs)
 747         with open(os.path.join(self.rally_dir,
 748                                'rally_jobs.yaml'), 'r') as task_file:
 749             self.task_yaml = yaml.safe_load(task_file)
 750
 751         for task in self.task_yaml:
 752             if task not in self.tests:
 753                 raise Exception("Test '%s' not in '%s'" %
 754                                 (task, self.tests))
 755
 756     def apply_blacklist(self, case_file_name, result_file_name):
 757         # pylint: disable=too-many-branches
 758         """Apply blacklist."""
 759         LOGGER.debug("Applying blacklist...")
 760         black_tests = list(set(self.excl_func() +
 761                                self.excl_scenario()))
 762         if black_tests:
 763             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 764
 765         template = YAML(typ='jinja2')
 766         with open(case_file_name, 'r') as fname:
 767             cases = template.load(fname)
 768         if cases.get("version", 1) == 1:
 769             # scenarios in dictionary
 770             for name in cases.keys():
 771                 if self.in_iterable_re(name, black_tests):
 772                     cases.pop(name)
 773         else:
 774             # workloads in subtasks
 775             for sind, subtask in reversed(list(
 776                     enumerate(cases.get('subtasks', [])))):
 777                 for wind, workload in reversed(list(
 778                         enumerate(subtask.get('workloads', [])))):
 779                     scenario = workload.get('scenario', {})
 780                     for name in scenario.keys():
 781                         if self.in_iterable_re(name, black_tests):
 782                             cases['subtasks'][sind]['workloads'].pop(wind)
 783                             break
 784                 if 'workloads' in cases['subtasks'][sind]:
 785                     if not cases['subtasks'][sind]['workloads']:
 786                         cases['subtasks'].pop(sind)
 787             # scenarios in subtasks
 788             for sind, subtask in reversed(list(
 789                     enumerate(cases.get('subtasks', [])))):
 790                 scenario = subtask.get('scenario', {})
 791                 for name in scenario.keys():
 792                     if self.in_iterable_re(name, black_tests):
 793                         cases['subtasks'].pop(sind)
 794                         break
 795
 796         with open(result_file_name, 'w') as fname:
 797             template.dump(cases, fname)
 798
 799     def build_task_args(self, test_name):
 800         """Build arguments for the Rally task."""
 801         task_args = {}
 802         if self.ext_net:
 803             task_args['floating_network'] = str(self.ext_net.name)
 804         else:
 805             task_args['floating_network'] = ''
 806         task_args['image_name'] = str(self.image.name)
 807         task_args['flavor_name'] = str(self.flavor.name)
 808         return task_args
 809
 810     @staticmethod
 811     def _remove_plugins_extra():
 812         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 813         try:
 814             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 815             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 816         except Exception:  # pylint: disable=broad-except
 817             pass
 818
 819     def prepare_task(self, test_name):
 820         """Prepare resources for test run."""
 821         self._remove_plugins_extra()
 822         jobs_dir = os.path.join(
 823             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 824         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 825         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 826                         os.path.join(inst_dir, 'plugins'))
 827         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 828                         os.path.join(inst_dir, 'extra'))
 829
 830         task_name = self.task_yaml.get(test_name).get("task")
 831         task = os.path.join(jobs_dir, task_name)
 832         if not os.path.exists(task):
 833             raise Exception("The scenario '%s' does not exist." % task)
 834         LOGGER.debug('Scenario fetched from : %s', task)
 835
 836         if not os.path.exists(self.temp_dir):
 837             os.makedirs(self.temp_dir)
 838         task_file_name = os.path.join(self.temp_dir, task_name)
 839         self.apply_blacklist(task, task_file_name)
 840         self.run_cmd = (["timeout", self.task_timeout,
 841                          "rally", "task", "start", "--task", task_file_name,
 842                          "--task-args", str(self.build_task_args(test_name))])
 843         return True
 844
 845     def clean(self):
 846         self._remove_plugins_extra()
 847         super(RallyJobs, self).clean()