functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 from six.moves import configparser
  29 from xtesting.core import testcase
  30 import yaml
  31
  32 from functest.core import singlevm
  33 from functest.utils import config
  34 from functest.utils import env
  35
  36 LOGGER = logging.getLogger(__name__)
  37
  38
  39 class RallyBase(singlevm.VmReady2):
  40     """Base class form Rally testcases implementation."""
  41
  42     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  43     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  44              'keystone', 'neutron', 'nova', 'quotas']
  45
  46     RALLY_CONF_PATH = "/etc/rally/rally.conf"
  47     RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  48         'functest', 'ci/rally_aarch64_patch.conf')
  49     RALLY_DIR = pkg_resources.resource_filename(
  50         'functest', 'opnfv_tests/openstack/rally')
  51     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  52         'functest', 'opnfv_tests/openstack/rally/scenario')
  53     TEMPLATE_DIR = pkg_resources.resource_filename(
  54         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  55     SUPPORT_DIR = pkg_resources.resource_filename(
  56         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  57     USERS_AMOUNT = 2
  58     TENANTS_AMOUNT = 3
  59     ITERATIONS_AMOUNT = 10
  60     CONCURRENCY = 4
  61     VOLUME_VERSION = 3
  62     VOLUME_SERVICE_TYPE = "volumev3"
  63     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
  64     TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  65     TEMP_DIR = os.path.join(TASK_DIR, 'var')
  66
  67     visibility = 'public'
  68     shared_network = True
  69
  70     def __init__(self, **kwargs):
  71         """Initialize RallyBase object."""
  72         super(RallyBase, self).__init__(**kwargs)
  73         assert self.orig_cloud
  74         assert self.project
  75         if self.orig_cloud.get_role("admin"):
  76             role_name = "admin"
  77         elif self.orig_cloud.get_role("Admin"):
  78             role_name = "Admin"
  79         else:
  80             raise Exception("Cannot detect neither admin nor Admin")
  81         self.orig_cloud.grant_role(
  82             role_name, user=self.project.user.id,
  83             project=self.project.project.id,
  84             domain=self.project.domain.id)
  85         self.results_dir = os.path.join(
  86             getattr(config.CONF, 'dir_results'), self.case_name)
  87         self.task_file = ''
  88         self.creators = []
  89         self.summary = []
  90         self.scenario_dir = ''
  91         self.smoke = None
  92         self.start_time = None
  93         self.result = None
  94         self.details = None
  95         self.compute_cnt = 0
  96         self.flavor_alt = None
  97         self.tests = []
  98         self.run_cmd = ''
  99         self.network_extensions = []
 100         self.services = []
 101
 102     def build_task_args(self, test_name):
 103         """Build arguments for the Rally task."""
 104         task_args = {'service_list': [test_name]}
 105         task_args['image_name'] = str(self.image.name)
 106         task_args['flavor_name'] = str(self.flavor.name)
 107         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 108         task_args['glance_image_location'] = str(self.filename)
 109         task_args['glance_image_format'] = str(self.image_format)
 110         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 111         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 112         task_args['users_amount'] = self.USERS_AMOUNT
 113         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 114         task_args['use_existing_users'] = False
 115         task_args['iterations'] = self.ITERATIONS_AMOUNT
 116         task_args['concurrency'] = self.CONCURRENCY
 117         task_args['smoke'] = self.smoke
 118         task_args['volume_version'] = self.VOLUME_VERSION
 119         task_args['volume_service_type'] = self.VOLUME_SERVICE_TYPE
 120
 121         if self.ext_net:
 122             task_args['floating_network'] = str(self.ext_net.name)
 123         else:
 124             task_args['floating_network'] = ''
 125
 126         if self.network:
 127             task_args['netid'] = str(self.network.id)
 128         else:
 129             task_args['netid'] = ''
 130
 131         return task_args
 132
 133     def _prepare_test_list(self, test_name):
 134         """Build the list of test cases to be executed."""
 135         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 136         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 137                                           test_yaml_file_name)
 138
 139         if not os.path.exists(scenario_file_name):
 140             scenario_file_name = os.path.join(self.scenario_dir,
 141                                               test_yaml_file_name)
 142
 143             if not os.path.exists(scenario_file_name):
 144                 raise Exception("The scenario '%s' does not exist."
 145                                 % scenario_file_name)
 146
 147         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 148         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 149
 150         if not os.path.exists(self.TEMP_DIR):
 151             os.makedirs(self.TEMP_DIR)
 152
 153         self.apply_blacklist(scenario_file_name, test_file_name)
 154         return test_file_name
 155
 156     @staticmethod
 157     def get_verifier_deployment_id():
 158         """
 159         Returns deployment id for active Rally deployment
 160         """
 161         cmd = ("rally deployment list | awk '/" +
 162                getattr(config.CONF, 'rally_deployment_name') +
 163                "/ {print $2}'")
 164         proc = subprocess.Popen(cmd, shell=True,
 165                                 stdout=subprocess.PIPE,
 166                                 stderr=subprocess.STDOUT)
 167         deployment_uuid = proc.stdout.readline().rstrip()
 168         return deployment_uuid
 169
 170     @staticmethod
 171     def create_rally_deployment(environ=None):
 172         """Create new rally deployment"""
 173         # set the architecture to default
 174         pod_arch = env.get("POD_ARCH")
 175         arch_filter = ['aarch64']
 176
 177         if pod_arch and pod_arch in arch_filter:
 178             LOGGER.info("Apply aarch64 specific to rally config...")
 179             with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
 180                 rally_patch_conf = pfile.read()
 181
 182             for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
 183                 print(line, end=' ')
 184                 if "cirros|testvm" in line:
 185                     print(rally_patch_conf)
 186
 187         LOGGER.info("Creating Rally environment...")
 188         try:
 189             cmd = ['rally', 'deployment', 'destroy',
 190                    '--deployment',
 191                    str(getattr(config.CONF, 'rally_deployment_name'))]
 192             output = subprocess.check_output(cmd)
 193             LOGGER.info("%s\n%s", " ".join(cmd), output)
 194         except subprocess.CalledProcessError:
 195             pass
 196
 197         cmd = ['rally', 'deployment', 'create', '--fromenv',
 198                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 199         output = subprocess.check_output(cmd, env=environ)
 200         LOGGER.info("%s\n%s", " ".join(cmd), output)
 201
 202         cmd = ['rally', 'deployment', 'check']
 203         output = subprocess.check_output(cmd)
 204         LOGGER.info("%s\n%s", " ".join(cmd), output)
 205         return RallyBase.get_verifier_deployment_id()
 206
 207     @staticmethod
 208     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 209         """Set keystone_default_role in rally.conf"""
 210         if env.get("NEW_USER_ROLE").lower() != "member":
 211             rconfig = configparser.RawConfigParser()
 212             rconfig.read(rally_conf)
 213             if not rconfig.has_section('openstack'):
 214                 rconfig.add_section('openstack')
 215             rconfig.set(
 216                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 217             with open(rally_conf, 'wb') as config_file:
 218                 rconfig.write(config_file)
 219
 220     @staticmethod
 221     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 222         """Clean Rally config"""
 223         if env.get("NEW_USER_ROLE").lower() != "member":
 224             rconfig = configparser.RawConfigParser()
 225             rconfig.read(rally_conf)
 226             if rconfig.has_option('openstack', 'keystone_default_role'):
 227                 rconfig.remove_option('openstack', 'keystone_default_role')
 228             with open(rally_conf, 'wb') as config_file:
 229                 rconfig.write(config_file)
 230
 231     @staticmethod
 232     def get_task_id(cmd_raw):
 233         """
 234         Get task id from command rally result.
 235
 236         :param cmd_raw:
 237         :return: task_id as string
 238         """
 239         taskid_re = re.compile('^Task +(.*): started$')
 240         for line in cmd_raw.splitlines(True):
 241             line = line.strip()
 242             match = taskid_re.match(line)
 243             if match:
 244                 return match.group(1)
 245         return None
 246
 247     @staticmethod
 248     def task_succeed(json_raw):
 249         """
 250         Parse JSON from rally JSON results.
 251
 252         :param json_raw:
 253         :return: Bool
 254         """
 255         rally_report = json.loads(json_raw)
 256         tasks = rally_report.get('tasks')
 257         if tasks:
 258             for task in tasks:
 259                 if task.get('status') != 'finished' or \
 260                    task.get('pass_sla') is not True:
 261                     return False
 262         else:
 263             return False
 264         return True
 265
 266     def _migration_supported(self):
 267         """Determine if migration is supported."""
 268         if self.compute_cnt > 1:
 269             return True
 270         return False
 271
 272     def _network_trunk_supported(self):
 273         """Determine if network trunk service is available"""
 274         if 'trunk' in self.network_extensions:
 275             return True
 276         return False
 277
 278     @staticmethod
 279     def excl_scenario():
 280         """Exclude scenario."""
 281         black_tests = []
 282         try:
 283             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 284                 black_list_yaml = yaml.safe_load(black_list_file)
 285
 286             deploy_scenario = env.get('DEPLOY_SCENARIO')
 287             if (bool(deploy_scenario) and
 288                     'scenario' in black_list_yaml.keys()):
 289                 for item in black_list_yaml['scenario']:
 290                     scenarios = item['scenarios']
 291                     in_it = RallyBase.in_iterable_re
 292                     if in_it(deploy_scenario, scenarios):
 293                         tests = item['tests']
 294                         black_tests.extend(tests)
 295         except Exception:  # pylint: disable=broad-except
 296             LOGGER.debug("Scenario exclusion not applied.")
 297
 298         return black_tests
 299
 300     @staticmethod
 301     def in_iterable_re(needle, haystack):
 302         """
 303         Check if given needle is in the iterable haystack, using regex.
 304
 305         :param needle: string to be matched
 306         :param haystack: iterable of strings (optionally regex patterns)
 307         :return: True if needle is eqial to any of the elements in haystack,
 308                  or if a nonempty regex pattern in haystack is found in needle.
 309         """
 310         # match without regex
 311         if needle in haystack:
 312             return True
 313
 314         for pattern in haystack:
 315             # match if regex pattern is set and found in the needle
 316             if pattern and re.search(pattern, needle) is not None:
 317                 return True
 318
 319         return False
 320
 321     def excl_func(self):
 322         """Exclude functionalities."""
 323         black_tests = []
 324         func_list = []
 325
 326         try:
 327             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 328                 black_list_yaml = yaml.safe_load(black_list_file)
 329
 330             if not self._migration_supported():
 331                 func_list.append("no_migration")
 332             if not self._network_trunk_supported():
 333                 func_list.append("no_net_trunk_service")
 334
 335             if 'functionality' in black_list_yaml.keys():
 336                 for item in black_list_yaml['functionality']:
 337                     functions = item['functions']
 338                     for func in func_list:
 339                         if func in functions:
 340                             tests = item['tests']
 341                             black_tests.extend(tests)
 342         except Exception:  # pylint: disable=broad-except
 343             LOGGER.debug("Functionality exclusion not applied.")
 344
 345         return black_tests
 346
 347     def apply_blacklist(self, case_file_name, result_file_name):
 348         """Apply blacklist."""
 349         LOGGER.debug("Applying blacklist...")
 350         cases_file = open(case_file_name, 'r')
 351         result_file = open(result_file_name, 'w')
 352
 353         black_tests = list(set(self.excl_func() +
 354                                self.excl_scenario()))
 355
 356         if black_tests:
 357             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 358
 359         include = True
 360         for cases_line in cases_file:
 361             if include:
 362                 for black_tests_line in black_tests:
 363                     if re.search(black_tests_line,
 364                                  cases_line.strip().rstrip(':')):
 365                         include = False
 366                         break
 367                 else:
 368                     result_file.write(str(cases_line))
 369             else:
 370                 if cases_line.isspace():
 371                     include = True
 372
 373         cases_file.close()
 374         result_file.close()
 375
 376     @staticmethod
 377     def file_is_empty(file_name):
 378         """Determine is a file is empty."""
 379         try:
 380             if os.stat(file_name).st_size > 0:
 381                 return False
 382         except Exception:  # pylint: disable=broad-except
 383             pass
 384
 385         return True
 386
 387     def _save_results(self, test_name, task_id):
 388         """ Generate and save task execution results"""
 389         # check for result directory and create it otherwise
 390         if not os.path.exists(self.results_dir):
 391             LOGGER.debug('%s does not exist, we create it.',
 392                          self.results_dir)
 393             os.makedirs(self.results_dir)
 394
 395         # put detailed result to log
 396         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 397         LOGGER.debug('running command: %s', cmd)
 398         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 399         LOGGER.info("%s\n%s", " ".join(cmd), output)
 400
 401         # save report as JSON
 402         report_json_name = '{}.json'.format(test_name)
 403         report_json_dir = os.path.join(self.results_dir, report_json_name)
 404         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 405                 "--out", report_json_dir])
 406         LOGGER.debug('running command: %s', cmd)
 407         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 408         LOGGER.info("%s\n%s", " ".join(cmd), output)
 409
 410         json_results = open(report_json_dir).read()
 411         self._append_summary(json_results, test_name)
 412
 413         # parse JSON operation result
 414         if self.task_succeed(json_results):
 415             LOGGER.info('Test scenario: "%s" OK.', test_name)
 416         else:
 417             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 418
 419     def run_task(self, test_name):
 420         """Run a task."""
 421         LOGGER.info('Starting test scenario "%s" ...', test_name)
 422         LOGGER.debug('running command: %s', self.run_cmd)
 423         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 424                                 stderr=subprocess.STDOUT)
 425         output = proc.communicate()[0]
 426
 427         task_id = self.get_task_id(output)
 428         LOGGER.debug('task_id : %s', task_id)
 429         if task_id is None:
 430             LOGGER.error("Failed to retrieve task_id")
 431             LOGGER.error("Result:\n%s", output)
 432             raise Exception("Failed to retrieve task id")
 433         self._save_results(test_name, task_id)
 434
 435     def _append_summary(self, json_raw, test_name):
 436         """Update statistics summary info."""
 437         nb_tests = 0
 438         nb_success = 0
 439         overall_duration = 0.0
 440         success = []
 441         failures = []
 442
 443         rally_report = json.loads(json_raw)
 444         for task in rally_report.get('tasks'):
 445             for subtask in task.get('subtasks'):
 446                 has_errors = False
 447                 for workload in subtask.get('workloads'):
 448                     if workload.get('full_duration'):
 449                         overall_duration += workload.get('full_duration')
 450
 451                     if workload.get('data'):
 452                         nb_tests += len(workload.get('data'))
 453
 454                     for result in workload.get('data'):
 455                         if not result.get('error'):
 456                             nb_success += 1
 457                         else:
 458                             has_errors = True
 459
 460                 if has_errors:
 461                     failures.append(subtask['title'])
 462                 else:
 463                     success.append(subtask['title'])
 464
 465         scenario_summary = {'test_name': test_name,
 466                             'overall_duration': overall_duration,
 467                             'nb_tests': nb_tests,
 468                             'nb_success': nb_success,
 469                             'success': success,
 470                             'failures': failures,
 471                             'task_status': self.task_succeed(json_raw)}
 472         self.summary.append(scenario_summary)
 473
 474     def prepare_run(self, **kwargs):
 475         """Prepare resources needed by test scenarios."""
 476         assert self.cloud
 477         LOGGER.debug('Validating run tests...')
 478         for test in kwargs.get('tests', self.TESTS):
 479             if test in self.TESTS:
 480                 self.tests.append(test)
 481             else:
 482                 raise Exception("Test name '%s' is invalid" % test)
 483
 484         if not os.path.exists(self.TASK_DIR):
 485             os.makedirs(self.TASK_DIR)
 486
 487         task = os.path.join(self.RALLY_DIR, 'task.yaml')
 488         if not os.path.exists(task):
 489             LOGGER.error("Task file '%s' does not exist.", task)
 490             raise Exception("Task file '{}' does not exist.".
 491                             format(task))
 492         self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
 493         shutil.copyfile(task, self.task_file)
 494
 495         task_macro = os.path.join(self.RALLY_DIR, 'macro')
 496         if not os.path.exists(task_macro):
 497             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 498             raise Exception("Task macro dir '{}' does not exist.".
 499                             format(task_macro))
 500         macro_dir = os.path.join(self.TASK_DIR, 'macro')
 501         if os.path.exists(macro_dir):
 502             shutil.rmtree(macro_dir)
 503         shutil.copytree(task_macro, macro_dir)
 504
 505         self.update_keystone_default_role()
 506         self.compute_cnt = len(self.cloud.list_hypervisors())
 507         self.network_extensions = self.cloud.get_network_extensions()
 508         self.flavor_alt = self.create_flavor_alt()
 509         self.services = [service.name for service in
 510                          self.cloud.list_services()]
 511
 512         LOGGER.debug("flavor: %s", self.flavor_alt)
 513
 514     def prepare_task(self, test_name):
 515         """Prepare resources for test run."""
 516         file_name = self._prepare_test_list(test_name)
 517         if self.file_is_empty(file_name):
 518             LOGGER.info('No tests for scenario "%s"', test_name)
 519             return False
 520         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 521                          "--task", self.task_file, "--task-args",
 522                          str(self.build_task_args(test_name))])
 523         return True
 524
 525     def run_tests(self, **kwargs):
 526         """Execute tests."""
 527         optional = kwargs.get('optional', [])
 528         for test in self.tests:
 529             if test in self.services or test not in optional:
 530                 if self.prepare_task(test):
 531                     self.run_task(test)
 532
 533     def _generate_report(self):
 534         """Generate test execution summary report."""
 535         total_duration = 0.0
 536         total_nb_tests = 0
 537         total_nb_success = 0
 538         nb_modules = 0
 539         payload = []
 540
 541         res_table = prettytable.PrettyTable(
 542             padding_width=2,
 543             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 544         res_table.align['Module'] = "l"
 545         res_table.align['Duration'] = "r"
 546         res_table.align['Success'] = "r"
 547
 548         # for each scenario we draw a row for the table
 549         for item in self.summary:
 550             if item['task_status'] is True:
 551                 nb_modules += 1
 552             total_duration += item['overall_duration']
 553             total_nb_tests += item['nb_tests']
 554             total_nb_success += item['nb_success']
 555             try:
 556                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 557             except ZeroDivisionError:
 558                 success_avg = 0
 559             success_str = str("{:0.2f}".format(success_avg)) + '%'
 560             duration_str = time.strftime("%H:%M:%S",
 561                                          time.gmtime(item['overall_duration']))
 562             res_table.add_row([item['test_name'], duration_str,
 563                                item['nb_tests'], success_str])
 564             payload.append({'module': item['test_name'],
 565                             'details': {'duration': item['overall_duration'],
 566                                         'nb tests': item['nb_tests'],
 567                                         'success rate': success_str,
 568                                         'success': item['success'],
 569                                         'failures': item['failures']}})
 570
 571         total_duration_str = time.strftime("%H:%M:%S",
 572                                            time.gmtime(total_duration))
 573         try:
 574             self.result = 100 * total_nb_success / total_nb_tests
 575         except ZeroDivisionError:
 576             self.result = 100
 577         success_rate = "{:0.2f}".format(self.result)
 578         success_rate_str = str(success_rate) + '%'
 579         res_table.add_row(["", "", "", ""])
 580         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 581                            success_rate_str])
 582
 583         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 584         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 585                     self.case_name, success_rate, nb_modules,
 586                     len(self.summary))
 587         payload.append({'summary': {'duration': total_duration,
 588                                     'nb tests': total_nb_tests,
 589                                     'nb success': success_rate}})
 590         self.details = payload
 591
 592     @staticmethod
 593     def export_task(file_name, export_type="html"):
 594         """Export all task results (e.g. html or xunit report)
 595
 596         Raises:
 597             subprocess.CalledProcessError: if Rally doesn't return 0
 598
 599         Returns:
 600             None
 601         """
 602         cmd = ["rally", "task", "export", "--type", export_type,
 603                "--deployment",
 604                str(getattr(config.CONF, 'rally_deployment_name')),
 605                "--to", file_name]
 606         LOGGER.debug('running command: %s', cmd)
 607         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 608         LOGGER.info("%s\n%s", " ".join(cmd), output)
 609
 610     @staticmethod
 611     def verify_report(file_name, uuid, export_type="html"):
 612         """Generate the verifier report (e.g. html or xunit report)
 613
 614         Raises:
 615             subprocess.CalledProcessError: if Rally doesn't return 0
 616
 617         Returns:
 618             None
 619         """
 620         cmd = ["rally", "verify", "report", "--type", export_type,
 621                "--uuid", uuid, "--to", file_name]
 622         LOGGER.debug('running command: %s', cmd)
 623         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 624         LOGGER.info("%s\n%s", " ".join(cmd), output)
 625
 626     def clean(self):
 627         """Cleanup of OpenStack resources. Should be called on completion."""
 628         self.clean_rally_conf()
 629         if self.flavor_alt:
 630             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 631         super(RallyBase, self).clean()
 632
 633     def is_successful(self):
 634         """The overall result of the test."""
 635         for item in self.summary:
 636             if item['task_status'] is False:
 637                 return testcase.TestCase.EX_TESTCASE_FAILED
 638
 639         return super(RallyBase, self).is_successful()
 640
 641     def run(self, **kwargs):
 642         """Run testcase."""
 643         self.start_time = time.time()
 644         try:
 645             assert super(RallyBase, self).run(
 646                 **kwargs) == testcase.TestCase.EX_OK
 647             environ = dict(
 648                 os.environ,
 649                 OS_USERNAME=self.project.user.name,
 650                 OS_PROJECT_NAME=self.project.project.name,
 651                 OS_PROJECT_ID=self.project.project.id,
 652                 OS_PASSWORD=self.project.password)
 653             try:
 654                 del environ['OS_TENANT_NAME']
 655                 del environ['OS_TENANT_ID']
 656             except Exception:  # pylint: disable=broad-except
 657                 pass
 658             self.create_rally_deployment(environ=environ)
 659             self.prepare_run(**kwargs)
 660             self.run_tests(**kwargs)
 661             self._generate_report()
 662             self.export_task(
 663                 "{}/{}.html".format(self.results_dir, self.case_name))
 664             self.export_task(
 665                 "{}/{}.xml".format(self.results_dir, self.case_name),
 666                 export_type="junit-xml")
 667             res = testcase.TestCase.EX_OK
 668         except Exception as exc:   # pylint: disable=broad-except
 669             LOGGER.error('Error with run: %s', exc)
 670             self.result = 0
 671             res = testcase.TestCase.EX_RUN_ERROR
 672         self.stop_time = time.time()
 673         return res
 674
 675
 676 class RallySanity(RallyBase):
 677     """Rally sanity testcase implementation."""
 678
 679     def __init__(self, **kwargs):
 680         """Initialize RallySanity object."""
 681         if "case_name" not in kwargs:
 682             kwargs["case_name"] = "rally_sanity"
 683         super(RallySanity, self).__init__(**kwargs)
 684         self.smoke = True
 685         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 686
 687
 688 class RallyFull(RallyBase):
 689     """Rally full testcase implementation."""
 690
 691     def __init__(self, **kwargs):
 692         """Initialize RallyFull object."""
 693         if "case_name" not in kwargs:
 694             kwargs["case_name"] = "rally_full"
 695         super(RallyFull, self).__init__(**kwargs)
 696         self.smoke = False
 697         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
 698
 699
 700 class RallyJobs(RallyBase):
 701     """Rally OpenStack CI testcase implementation."""
 702
 703     TESTS = ["neutron"]
 704
 705     def __init__(self, **kwargs):
 706         """Initialize RallyJobs object."""
 707         if "case_name" not in kwargs:
 708             kwargs["case_name"] = "rally_jobs"
 709         super(RallyJobs, self).__init__(**kwargs)
 710         self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
 711         self.task_yaml = None
 712
 713     def prepare_run(self, **kwargs):
 714         """Create resources needed by test scenarios."""
 715         super(RallyJobs, self).prepare_run(**kwargs)
 716         with open(os.path.join(self.RALLY_DIR,
 717                                'rally_jobs.yaml'), 'r') as task_file:
 718             self.task_yaml = yaml.safe_load(task_file)
 719
 720         for task in self.task_yaml:
 721             if task not in self.tests:
 722                 raise Exception("Test '%s' not in '%s'" %
 723                                 (task, self.tests))
 724
 725     def apply_blacklist(self, case_file_name, result_file_name):
 726         # pylint: disable=too-many-branches
 727         """Apply blacklist."""
 728         LOGGER.debug("Applying blacklist...")
 729         black_tests = list(set(self.excl_func() +
 730                                self.excl_scenario()))
 731         if black_tests:
 732             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 733
 734         template = YAML(typ='jinja2')
 735         with open(case_file_name, 'r') as fname:
 736             cases = template.load(fname)
 737         if cases.get("version", 1) == 1:
 738             # scenarios in dictionary
 739             for name in cases.keys():
 740                 if self.in_iterable_re(name, black_tests):
 741                     cases.pop(name)
 742         else:
 743             # workloads in subtasks
 744             for sind, subtask in enumerate(cases.get('subtasks', [])):
 745                 idx = []
 746                 for wind, workload in enumerate(subtask.get('workloads', [])):
 747                     scenario = workload.get('scenario', {})
 748                     for name in scenario.keys():
 749                         if self.in_iterable_re(name, black_tests):
 750                             idx.append(wind)
 751                             break
 752                 for wind in reversed(idx):
 753                     cases['subtasks'][sind]['workloads'].pop(wind)
 754             # scenarios in subtasks
 755             idx = []
 756             for sind, subtask in enumerate(cases.get('subtasks', [])):
 757                 scenario = subtask.get('scenario', {})
 758                 for name in scenario.keys():
 759                     if self.in_iterable_re(name, black_tests):
 760                         idx.append(sind)
 761                         break
 762             for sind in reversed(idx):
 763                 cases['subtasks'].pop(sind)
 764
 765         with open(result_file_name, 'w') as fname:
 766             template.dump(cases, fname)
 767
 768     def build_task_args(self, test_name):
 769         """Build arguments for the Rally task."""
 770         task_args = {}
 771         if self.ext_net:
 772             task_args['floating_network'] = str(self.ext_net.name)
 773         else:
 774             task_args['floating_network'] = ''
 775         return task_args
 776
 777     @staticmethod
 778     def _remove_plugins_extra():
 779         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 780         try:
 781             shutil.rmtree(os.path.join(inst_dir, 'plugins'))
 782             shutil.rmtree(os.path.join(inst_dir, 'extra'))
 783         except Exception:  # pylint: disable=broad-except
 784             pass
 785
 786     def prepare_task(self, test_name):
 787         """Prepare resources for test run."""
 788         self._remove_plugins_extra()
 789         jobs_dir = os.path.join(
 790             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 791         inst_dir = getattr(config.CONF, 'dir_rally_inst')
 792         shutil.copytree(os.path.join(jobs_dir, 'plugins'),
 793                         os.path.join(inst_dir, 'plugins'))
 794         shutil.copytree(os.path.join(jobs_dir, 'extra'),
 795                         os.path.join(inst_dir, 'extra'))
 796
 797         task_name = self.task_yaml.get(test_name).get("task")
 798         task = os.path.join(jobs_dir, task_name)
 799         if not os.path.exists(task):
 800             raise Exception("The scenario '%s' does not exist." % task)
 801         LOGGER.debug('Scenario fetched from : %s', task)
 802
 803         if not os.path.exists(self.TEMP_DIR):
 804             os.makedirs(self.TEMP_DIR)
 805         task_file_name = os.path.join(self.TEMP_DIR, task_name)
 806         self.apply_blacklist(task, task_file_name)
 807         self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
 808                          "--task-args", str(self.build_task_args(test_name))])
 809         return True
 810
 811     def clean(self):
 812         self._remove_plugins_extra()
 813         super(RallyJobs, self).clean()