functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14 from __future__ import print_function
  15
  16 import fileinput
  17 import json
  18 import logging
  19 import os
  20 import re
  21 import shutil
  22 import subprocess
  23 import time
  24
  25 import pkg_resources
  26 import prettytable
  27 from ruamel.yaml import YAML
  28 import six
  29 from six.moves import configparser
  30 from xtesting.core import testcase
  31 import yaml
  32
  33 from functest.core import singlevm
  34 from functest.utils import config
  35 from functest.utils import env
  36 from functest.utils import functest_utils
  37
  38 LOGGER = logging.getLogger(__name__)
  39
  40
  41 class RallyBase(singlevm.VmReady2):
  42     """Base class form Rally testcases implementation."""
  43
  44     # pylint: disable=too-many-instance-attributes, too-many-public-methods
  45     stests = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  46               'keystone', 'neutron', 'nova', 'quotas', 'swift', 'barbican',
  47               'vm']
  48
  49     rally_conf_path = "/etc/rally/rally.conf"
  50     rally_aar4_patch_path = pkg_resources.resource_filename(
  51         'functest', 'ci/rally_aarch64_patch.conf')
  52     rally_dir = pkg_resources.resource_filename(
  53         'functest', 'opnfv_tests/openstack/rally')
  54     rally_scenario_dir = pkg_resources.resource_filename(
  55         'functest', 'opnfv_tests/openstack/rally/scenario')
  56     template_dir = pkg_resources.resource_filename(
  57         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  58     support_dir = pkg_resources.resource_filename(
  59         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  60     users_amount = 2
  61     tenants_amount = 3
  62     iterations_amount = 10
  63     concurrency = 4
  64     volume_version = 3
  65     volume_service_type = "volumev3"
  66     blacklist_file = os.path.join(rally_dir, "blacklist.yaml")
  67     task_dir = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
  68     temp_dir = os.path.join(task_dir, 'var')
  69
  70     visibility = 'public'
  71     shared_network = True
  72     task_timeout = 3600
  73     username = 'cirros'
  74
  75     def __init__(self, **kwargs):
  76         """Initialize RallyBase object."""
  77         super().__init__(**kwargs)
  78         assert self.orig_cloud
  79         assert self.project
  80         if self.orig_cloud.get_role("admin"):
  81             role_name = "admin"
  82         elif self.orig_cloud.get_role("Admin"):
  83             role_name = "Admin"
  84         else:
  85             raise Exception("Cannot detect neither admin nor Admin")
  86         self.orig_cloud.grant_role(
  87             role_name, user=self.project.user.id,
  88             project=self.project.project.id,
  89             domain=self.project.domain.id)
  90         self.results_dir = os.path.join(
  91             getattr(config.CONF, 'dir_results'), self.case_name)
  92         self.task_file = ''
  93         self.creators = []
  94         self.summary = []
  95         self.scenario_dir = ''
  96         self.smoke = None
  97         self.start_time = None
  98         self.result = None
  99         self.compute_cnt = 0
 100         self.flavor_alt = None
 101         self.tests = []
 102         self.run_cmd = ''
 103         self.network_extensions = []
 104         self.services = []
 105
 106     def build_task_args(self, test_name):
 107         """Build arguments for the Rally task."""
 108         task_args = {'service_list': [test_name]}
 109         task_args['image_name'] = str(self.image.name)
 110         task_args['flavor_name'] = str(self.flavor.name)
 111         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
 112         task_args['glance_image_location'] = str(self.filename)
 113         task_args['glance_image_format'] = str(self.image_format)
 114         task_args['tmpl_dir'] = str(self.template_dir)
 115         task_args['sup_dir'] = str(self.support_dir)
 116         task_args['users_amount'] = self.users_amount
 117         task_args['tenants_amount'] = self.tenants_amount
 118         task_args['use_existing_users'] = False
 119         task_args['iterations'] = self.iterations_amount
 120         task_args['concurrency'] = self.concurrency
 121         task_args['smoke'] = self.smoke
 122         task_args['volume_version'] = self.volume_version
 123         task_args['volume_service_type'] = self.volume_service_type
 124         task_args['block_migration'] = env.get("BLOCK_MIGRATION").lower()
 125         task_args['username'] = self.username
 126
 127         if self.ext_net:
 128             task_args['floating_network'] = str(self.ext_net.name)
 129         else:
 130             task_args['floating_network'] = ''
 131
 132         if self.network:
 133             task_args['netid'] = str(self.network.id)
 134         else:
 135             LOGGER.warning(
 136                 'No tenant network created. '
 137                 'Trying EXTERNAL_NETWORK as a fallback')
 138             if env.get("EXTERNAL_NETWORK"):
 139                 network = self.cloud.get_network(env.get("EXTERNAL_NETWORK"))
 140                 task_args['netid'] = str(network.id) if network else ''
 141             else:
 142                 task_args['netid'] = ''
 143
 144         return task_args
 145
 146     def _prepare_test_list(self, test_name):
 147         """Build the list of test cases to be executed."""
 148         test_yaml_file_name = f'opnfv-{test_name}.yaml'
 149         scenario_file_name = os.path.join(self.rally_scenario_dir,
 150                                           test_yaml_file_name)
 151
 152         if not os.path.exists(scenario_file_name):
 153             scenario_file_name = os.path.join(self.scenario_dir,
 154                                               test_yaml_file_name)
 155
 156             if not os.path.exists(scenario_file_name):
 157                 raise Exception(
 158                     f"The scenario '{scenario_file_name}' does not exist.")
 159
 160         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 161         test_file_name = os.path.join(self.temp_dir, test_yaml_file_name)
 162
 163         if not os.path.exists(self.temp_dir):
 164             os.makedirs(self.temp_dir)
 165
 166         self.apply_blacklist(scenario_file_name, test_file_name)
 167         return test_file_name
 168
 169     @staticmethod
 170     def get_verifier_deployment_id():
 171         """
 172         Returns deployment id for active Rally deployment
 173         """
 174         cmd = ("rally deployment list | awk '/" +
 175                getattr(config.CONF, 'rally_deployment_name') +
 176                "/ {print $2}'")
 177         with subprocess.Popen(
 178                 cmd, shell=True, stdout=subprocess.PIPE,
 179                 stderr=subprocess.STDOUT) as proc:
 180             deployment_uuid = proc.stdout.readline().rstrip()
 181         return deployment_uuid.decode("utf-8")
 182
 183     @staticmethod
 184     def create_rally_deployment(environ=None):
 185         # pylint: disable=unexpected-keyword-arg
 186         """Create new rally deployment"""
 187         # set the architecture to default
 188         pod_arch = env.get("POD_ARCH")
 189         arch_filter = ['aarch64']
 190
 191         if pod_arch and pod_arch in arch_filter:
 192             LOGGER.info("Apply aarch64 specific to rally config...")
 193             with open(
 194                     RallyBase.rally_aar4_patch_path, "r",
 195                     encoding='utf-8') as pfile:
 196                 rally_patch_conf = pfile.read()
 197
 198             for line in fileinput.input(RallyBase.rally_conf_path):
 199                 print(line, end=' ')
 200                 if "cirros|testvm" in line:
 201                     print(rally_patch_conf)
 202
 203         LOGGER.info("Creating Rally environment...")
 204         try:
 205             cmd = ['rally', 'deployment', 'destroy',
 206                    '--deployment',
 207                    str(getattr(config.CONF, 'rally_deployment_name'))]
 208             output = subprocess.check_output(cmd)
 209             LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 210         except subprocess.CalledProcessError:
 211             pass
 212
 213         cmd = ['rally', 'deployment', 'create', '--fromenv',
 214                '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
 215         output = subprocess.check_output(cmd, env=environ)
 216         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 217
 218         cmd = ['rally', 'deployment', 'check']
 219         output = subprocess.check_output(cmd)
 220         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 221         return RallyBase.get_verifier_deployment_id()
 222
 223     @staticmethod
 224     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 225         """Set keystone_default_role in rally.conf"""
 226         if env.get("NEW_USER_ROLE").lower() != "member":
 227             rconfig = configparser.RawConfigParser()
 228             rconfig.read(rally_conf)
 229             if not rconfig.has_section('openstack'):
 230                 rconfig.add_section('openstack')
 231             rconfig.set(
 232                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 233             with open(rally_conf, 'w', encoding='utf-8') as config_file:
 234                 rconfig.write(config_file)
 235
 236     @staticmethod
 237     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 238         """Clean Rally config"""
 239         if env.get("NEW_USER_ROLE").lower() != "member":
 240             rconfig = configparser.RawConfigParser()
 241             rconfig.read(rally_conf)
 242             if rconfig.has_option('openstack', 'keystone_default_role'):
 243                 rconfig.remove_option('openstack', 'keystone_default_role')
 244             with open(rally_conf, 'w', encoding='utf-8') as config_file:
 245                 rconfig.write(config_file)
 246
 247     @staticmethod
 248     def get_task_id(tag):
 249         """
 250         Get task id from command rally result.
 251
 252         :param tag:
 253         :return: task_id as string
 254         """
 255         cmd = ["rally", "task", "list", "--tag", tag, "--uuids-only"]
 256         output = subprocess.check_output(cmd).decode("utf-8").rstrip()
 257         LOGGER.info("%s: %s", " ".join(cmd), output)
 258         return output
 259
 260     @staticmethod
 261     def task_succeed(json_raw):
 262         """
 263         Parse JSON from rally JSON results.
 264
 265         :param json_raw:
 266         :return: Bool
 267         """
 268         rally_report = json.loads(json_raw)
 269         tasks = rally_report.get('tasks')
 270         if tasks:
 271             for task in tasks:
 272                 if task.get('status') != 'finished' or \
 273                    task.get('pass_sla') is not True:
 274                     return False
 275         else:
 276             return False
 277         return True
 278
 279     def _migration_supported(self):
 280         """Determine if migration is supported."""
 281         if self.compute_cnt > 1:
 282             return True
 283         return False
 284
 285     def _network_trunk_supported(self):
 286         """Determine if network trunk service is available"""
 287         if 'trunk' in self.network_extensions:
 288             return True
 289         return False
 290
 291     @staticmethod
 292     def excl_scenario():
 293         """Exclude scenario."""
 294         black_tests = []
 295         try:
 296             with open(
 297                     RallyBase.blacklist_file, 'r',
 298                     encoding='utf-8') as black_list_file:
 299                 black_list_yaml = yaml.safe_load(black_list_file)
 300
 301             deploy_scenario = env.get('DEPLOY_SCENARIO')
 302             if (bool(deploy_scenario) and
 303                     'scenario' in black_list_yaml.keys()):
 304                 for item in black_list_yaml['scenario']:
 305                     scenarios = item['scenarios']
 306                     in_it = RallyBase.in_iterable_re
 307                     if in_it(deploy_scenario, scenarios):
 308                         tests = item['tests']
 309                         black_tests.extend(tests)
 310         except Exception:  # pylint: disable=broad-except
 311             LOGGER.debug("Scenario exclusion not applied.")
 312
 313         return black_tests
 314
 315     @staticmethod
 316     def in_iterable_re(needle, haystack):
 317         """
 318         Check if given needle is in the iterable haystack, using regex.
 319
 320         :param needle: string to be matched
 321         :param haystack: iterable of strings (optionally regex patterns)
 322         :return: True if needle is eqial to any of the elements in haystack,
 323                  or if a nonempty regex pattern in haystack is found in needle.
 324         """
 325         # match without regex
 326         if needle in haystack:
 327             return True
 328
 329         for pattern in haystack:
 330             # match if regex pattern is set and found in the needle
 331             if pattern and re.search(pattern, needle) is not None:
 332                 return True
 333
 334         return False
 335
 336     def excl_func(self):
 337         """Exclude functionalities."""
 338         black_tests = []
 339         func_list = []
 340
 341         try:
 342             with open(
 343                     RallyBase.blacklist_file, 'r',
 344                     encoding='utf-8') as black_list_file:
 345                 black_list_yaml = yaml.safe_load(black_list_file)
 346
 347             if env.get('BLOCK_MIGRATION').lower() == 'true':
 348                 func_list.append("block_migration")
 349             if not self._migration_supported():
 350                 func_list.append("no_migration")
 351             if not self._network_trunk_supported():
 352                 func_list.append("no_net_trunk_service")
 353             if not self.ext_net:
 354                 func_list.append("no_floating_ip")
 355
 356             if 'functionality' in black_list_yaml.keys():
 357                 for item in black_list_yaml['functionality']:
 358                     functions = item['functions']
 359                     for func in func_list:
 360                         if func in functions:
 361                             tests = item['tests']
 362                             black_tests.extend(tests)
 363         except Exception:  # pylint: disable=broad-except
 364             LOGGER.debug("Functionality exclusion not applied.")
 365
 366         return black_tests
 367
 368     def apply_blacklist(self, case_file_name, result_file_name):
 369         """Apply blacklist."""
 370         LOGGER.debug("Applying blacklist...")
 371         with open(case_file_name, 'r', encoding='utf-8') as cases_file, open(
 372                 result_file_name, 'w', encoding='utf-8') as result_file:
 373             black_tests = list(set(self.excl_func() + self.excl_scenario()))
 374             if black_tests:
 375                 LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 376
 377             include = True
 378             for cases_line in cases_file:
 379                 if include:
 380                     for black_tests_line in black_tests:
 381                         if re.search(black_tests_line,
 382                                      cases_line.strip().rstrip(':')):
 383                             include = False
 384                             break
 385                     else:
 386                         result_file.write(str(cases_line))
 387                 else:
 388                     if cases_line.isspace():
 389                         include = True
 390
 391     @staticmethod
 392     def file_is_empty(file_name):
 393         """Determine is a file is empty."""
 394         try:
 395             if os.stat(file_name).st_size > 0:
 396                 return False
 397         except Exception:  # pylint: disable=broad-except
 398             pass
 399
 400         return True
 401
 402     def _save_results(self, test_name, task_id):
 403         """ Generate and save task execution results"""
 404         # check for result directory and create it otherwise
 405         if not os.path.exists(self.results_dir):
 406             LOGGER.debug('%s does not exist, we create it.',
 407                          self.results_dir)
 408             os.makedirs(self.results_dir)
 409
 410         # put detailed result to log
 411         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 412         LOGGER.debug('running command: %s', cmd)
 413         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 414         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 415
 416         # save report as JSON
 417         report_json_name = f'{test_name}.json'
 418         report_json_dir = os.path.join(self.results_dir, report_json_name)
 419         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 420                 "--out", report_json_dir])
 421         LOGGER.debug('running command: %s', cmd)
 422         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 423         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 424
 425         with open(report_json_dir, encoding='utf-8') as json_file:
 426             json_results = json_file.read()
 427         self._append_summary(json_results, test_name)
 428
 429         # parse JSON operation result
 430         if self.task_succeed(json_results):
 431             LOGGER.info('Test scenario: "%s" OK.', test_name)
 432         else:
 433             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 434
 435     def run_task(self, test_name):
 436         """Run a task."""
 437         LOGGER.info('Starting test scenario "%s" ...', test_name)
 438         LOGGER.debug('running command: %s', self.run_cmd)
 439         if six.PY3:
 440             subprocess.call(
 441                 self.run_cmd, timeout=self.task_timeout,
 442                 stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 443         else:
 444             with open(os.devnull, 'wb') as devnull:
 445                 subprocess.call(self.run_cmd, stdout=devnull, stderr=devnull)
 446         task_id = self.get_task_id(test_name)
 447         LOGGER.debug('task_id : %s', task_id)
 448         if not task_id:
 449             LOGGER.error("Failed to retrieve task_id")
 450             raise Exception("Failed to retrieve task id")
 451         self._save_results(test_name, task_id)
 452
 453     def _append_summary(self, json_raw, test_name):
 454         # pylint: disable=too-many-locals
 455         """Update statistics summary info."""
 456         nb_tests = 0
 457         nb_success = 0
 458         overall_duration = 0.0
 459         success = []
 460         failures = []
 461
 462         rally_report = json.loads(json_raw)
 463         for task in rally_report.get('tasks'):
 464             for subtask in task.get('subtasks'):
 465                 has_errors = False
 466                 for workload in subtask.get('workloads'):
 467                     if workload.get('full_duration'):
 468                         overall_duration += workload.get('full_duration')
 469
 470                     if workload.get('data'):
 471                         nb_tests += len(workload.get('data'))
 472
 473                     for result in workload.get('data'):
 474                         if not result.get('error'):
 475                             nb_success += 1
 476                         else:
 477                             has_errors = True
 478
 479                 if has_errors:
 480                     failures.append(subtask['title'])
 481                 else:
 482                     success.append(subtask['title'])
 483
 484         scenario_summary = {'test_name': test_name,
 485                             'overall_duration': overall_duration,
 486                             'nb_tests': nb_tests,
 487                             'nb_success': nb_success,
 488                             'success': success,
 489                             'failures': failures,
 490                             'task_status': self.task_succeed(json_raw)}
 491         self.summary.append(scenario_summary)
 492
 493     def prepare_run(self, **kwargs):
 494         """Prepare resources needed by test scenarios."""
 495         assert self.cloud
 496         LOGGER.debug('Validating run tests...')
 497         for test in kwargs.get('tests', self.stests):
 498             if test in self.stests:
 499                 self.tests.append(test)
 500             else:
 501                 raise Exception(f"Test name '{test}' is invalid")
 502
 503         if not os.path.exists(self.task_dir):
 504             os.makedirs(self.task_dir)
 505
 506         task = os.path.join(self.rally_dir, 'task.yaml')
 507         if not os.path.exists(task):
 508             LOGGER.error("Task file '%s' does not exist.", task)
 509             raise Exception(f"Task file '{task}' does not exist.")
 510         self.task_file = os.path.join(self.task_dir, 'task.yaml')
 511         shutil.copyfile(task, self.task_file)
 512
 513         task_macro = os.path.join(self.rally_dir, 'macro')
 514         if not os.path.exists(task_macro):
 515             LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
 516             raise Exception(f"Task macro dir '{task_macro}' does not exist.")
 517         macro_dir = os.path.join(self.task_dir, 'macro')
 518         if os.path.exists(macro_dir):
 519             shutil.rmtree(macro_dir)
 520         shutil.copytree(task_macro, macro_dir)
 521
 522         self.update_keystone_default_role()
 523         self.compute_cnt = self.count_hypervisors()
 524         self.network_extensions = self.cloud.get_network_extensions()
 525         self.flavor_alt = self.create_flavor_alt()
 526         self.services = [service.name for service in
 527                          functest_utils.list_services(self.cloud)]
 528
 529         LOGGER.debug("flavor: %s", self.flavor_alt)
 530
 531     def prepare_task(self, test_name):
 532         """Prepare resources for test run."""
 533         file_name = self._prepare_test_list(test_name)
 534         if self.file_is_empty(file_name):
 535             LOGGER.info('No tests for scenario "%s"', test_name)
 536             return False
 537         self.run_cmd = (["rally", "task", "start", "--tag", test_name,
 538                          "--abort-on-sla-failure",
 539                          "--task", self.task_file, "--task-args",
 540                          str(self.build_task_args(test_name))])
 541         return True
 542
 543     def run_tests(self, **kwargs):
 544         """Execute tests."""
 545         optional = kwargs.get('optional', [])
 546         for test in self.tests:
 547             if test in self.services or test not in optional:
 548                 if self.prepare_task(test):
 549                     self.run_task(test)
 550
 551     def _generate_report(self):
 552         """Generate test execution summary report."""
 553         total_duration = 0.0
 554         total_nb_tests = 0
 555         total_nb_success = 0
 556         nb_modules = 0
 557         payload = []
 558
 559         res_table = prettytable.PrettyTable(
 560             padding_width=2,
 561             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 562         res_table.align['Module'] = "l"
 563         res_table.align['Duration'] = "r"
 564         res_table.align['Success'] = "r"
 565
 566         # for each scenario we draw a row for the table
 567         for item in self.summary:
 568             if item['task_status'] is True:
 569                 nb_modules += 1
 570             total_duration += item['overall_duration']
 571             total_nb_tests += item['nb_tests']
 572             total_nb_success += item['nb_success']
 573             try:
 574                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 575             except ZeroDivisionError:
 576                 success_avg = 0
 577             success_str = f"{success_avg:0.2f}%"
 578             duration_str = time.strftime("%H:%M:%S",
 579                                          time.gmtime(item['overall_duration']))
 580             res_table.add_row([item['test_name'], duration_str,
 581                                item['nb_tests'], success_str])
 582             payload.append({'module': item['test_name'],
 583                             'details': {'duration': item['overall_duration'],
 584                                         'nb tests': item['nb_tests'],
 585                                         'success rate': success_str,
 586                                         'success': item['success'],
 587                                         'failures': item['failures']}})
 588
 589         total_duration_str = time.strftime("%H:%M:%S",
 590                                            time.gmtime(total_duration))
 591         try:
 592             self.result = 100 * total_nb_success / total_nb_tests
 593         except ZeroDivisionError:
 594             self.result = 100
 595         success_rate = f"{self.result:0.2f}"
 596         success_rate_str = str(success_rate) + '%'
 597         res_table.add_row(["", "", "", ""])
 598         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 599                            success_rate_str])
 600
 601         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 602         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 603                     self.case_name, success_rate, nb_modules,
 604                     len(self.summary))
 605         self.details['summary'] = {'duration': total_duration,
 606                                    'nb tests': total_nb_tests,
 607                                    'nb success': success_rate}
 608         self.details["modules"] = payload
 609
 610     @staticmethod
 611     def export_task(file_name, export_type="html"):
 612         """Export all task results (e.g. html or xunit report)
 613
 614         Raises:
 615             subprocess.CalledProcessError: if Rally doesn't return 0
 616
 617         Returns:
 618             None
 619         """
 620         cmd = ["rally", "task", "export", "--type", export_type,
 621                "--deployment",
 622                str(getattr(config.CONF, 'rally_deployment_name')),
 623                "--to", file_name]
 624         LOGGER.debug('running command: %s', cmd)
 625         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 626         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 627
 628     @staticmethod
 629     def verify_report(file_name, uuid, export_type="html"):
 630         """Generate the verifier report (e.g. html or xunit report)
 631
 632         Raises:
 633             subprocess.CalledProcessError: if Rally doesn't return 0
 634
 635         Returns:
 636             None
 637         """
 638         cmd = ["rally", "verify", "report", "--type", export_type,
 639                "--uuid", uuid, "--to", file_name]
 640         LOGGER.debug('running command: %s', cmd)
 641         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 642         LOGGER.info("%s\n%s", " ".join(cmd), output.decode("utf-8"))
 643
 644     def clean(self):
 645         """Cleanup of OpenStack resources. Should be called on completion."""
 646         self.clean_rally_conf()
 647         self.clean_rally_logs()
 648         if self.flavor_alt:
 649             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 650         super().clean()
 651
 652     def is_successful(self):
 653         """The overall result of the test."""
 654         for item in self.summary:
 655             if item['task_status'] is False:
 656                 return testcase.TestCase.EX_TESTCASE_FAILED
 657
 658         return super().is_successful()
 659
 660     @staticmethod
 661     def update_rally_logs(res_dir, rally_conf='/etc/rally/rally.conf'):
 662         """Print rally logs in res dir"""
 663         if not os.path.exists(res_dir):
 664             os.makedirs(res_dir)
 665         rconfig = configparser.RawConfigParser()
 666         rconfig.read(rally_conf)
 667         rconfig.set('DEFAULT', 'debug', True)
 668         rconfig.set('DEFAULT', 'use_stderr', False)
 669         rconfig.set('DEFAULT', 'log-file', 'rally.log')
 670         rconfig.set('DEFAULT', 'log_dir', res_dir)
 671         with open(rally_conf, 'w', encoding='utf-8') as config_file:
 672             rconfig.write(config_file)
 673
 674     @staticmethod
 675     def clean_rally_logs(rally_conf='/etc/rally/rally.conf'):
 676         """Clean Rally config"""
 677         rconfig = configparser.RawConfigParser()
 678         rconfig.read(rally_conf)
 679         if rconfig.has_option('DEFAULT', 'use_stderr'):
 680             rconfig.remove_option('DEFAULT', 'use_stderr')
 681         if rconfig.has_option('DEFAULT', 'debug'):
 682             rconfig.remove_option('DEFAULT', 'debug')
 683         if rconfig.has_option('DEFAULT', 'log-file'):
 684             rconfig.remove_option('DEFAULT', 'log-file')
 685         if rconfig.has_option('DEFAULT', 'log_dir'):
 686             rconfig.remove_option('DEFAULT', 'log_dir')
 687         with open(rally_conf, 'w', encoding='utf-8') as config_file:
 688             rconfig.write(config_file)
 689
 690     def run(self, **kwargs):
 691         """Run testcase."""
 692         self.start_time = time.time()
 693         try:
 694             assert super().run(
 695                 **kwargs) == testcase.TestCase.EX_OK
 696             self.update_rally_logs(self.res_dir)
 697             self.create_rally_deployment(environ=self.project.get_environ())
 698             self.prepare_run(**kwargs)
 699             self.run_tests(**kwargs)
 700             self._generate_report()
 701             self.export_task(
 702                 f"{self.results_dir}/{self.case_name}.html")
 703             self.export_task(
 704                 f"{self.results_dir}/{self.case_name}.xml",
 705                 export_type="junit-xml")
 706             res = testcase.TestCase.EX_OK
 707         except Exception:   # pylint: disable=broad-except
 708             LOGGER.exception('Error with run:')
 709             self.result = 0
 710             res = testcase.TestCase.EX_RUN_ERROR
 711         self.stop_time = time.time()
 712         return res
 713
 714
 715 class RallySanity(RallyBase):
 716     """Rally sanity testcase implementation."""
 717
 718     def __init__(self, **kwargs):
 719         """Initialize RallySanity object."""
 720         if "case_name" not in kwargs:
 721             kwargs["case_name"] = "rally_sanity"
 722         super().__init__(**kwargs)
 723         self.smoke = True
 724         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'sanity')
 725
 726
 727 class RallyFull(RallyBase):
 728     """Rally full testcase implementation."""
 729
 730     task_timeout = 7200
 731
 732     def __init__(self, **kwargs):
 733         """Initialize RallyFull object."""
 734         if "case_name" not in kwargs:
 735             kwargs["case_name"] = "rally_full"
 736         super().__init__(**kwargs)
 737         self.smoke = False
 738         self.scenario_dir = os.path.join(self.rally_scenario_dir, 'full')
 739
 740
 741 class RallyJobs(RallyBase):
 742     """Rally OpenStack CI testcase implementation."""
 743
 744     stests = ["neutron"]
 745     task_timeout = 7200
 746
 747     def __init__(self, **kwargs):
 748         """Initialize RallyJobs object."""
 749         if "case_name" not in kwargs:
 750             kwargs["case_name"] = "rally_jobs"
 751         super().__init__(**kwargs)
 752         self.task_file = os.path.join(self.rally_dir, 'rally_jobs.yaml')
 753         self.task_yaml = None
 754
 755     def prepare_run(self, **kwargs):
 756         """Create resources needed by test scenarios."""
 757         super().prepare_run(**kwargs)
 758         with open(
 759                 os.path.join(self.rally_dir, 'rally_jobs.yaml'),
 760                 'r', encoding='utf-8') as task_file:
 761             self.task_yaml = yaml.safe_load(task_file)
 762
 763         for task in self.task_yaml:
 764             if task not in self.tests:
 765                 raise Exception(f"Test '{task}' not in '{self.tests}'")
 766
 767     def apply_blacklist(self, case_file_name, result_file_name):
 768         # pylint: disable=too-many-branches
 769         """Apply blacklist."""
 770         LOGGER.debug("Applying blacklist...")
 771         black_tests = list(set(self.excl_func() +
 772                                self.excl_scenario()))
 773         if black_tests:
 774             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 775
 776         template = YAML(typ='jinja2')
 777         with open(case_file_name, 'r', encoding='utf-8') as fname:
 778             cases = template.load(fname)
 779         if cases.get("version", 1) == 1:
 780             # scenarios in dictionary
 781             for name in cases.keys():
 782                 if self.in_iterable_re(name, black_tests):
 783                     cases.pop(name)
 784         else:
 785             # workloads in subtasks
 786             for sind, subtask in reversed(list(
 787                     enumerate(cases.get('subtasks', [])))):
 788                 for wind, workload in reversed(list(
 789                         enumerate(subtask.get('workloads', [])))):
 790                     scenario = workload.get('scenario', {})
 791                     for name in scenario.keys():
 792                         if self.in_iterable_re(name, black_tests):
 793                             cases['subtasks'][sind]['workloads'].pop(wind)
 794                             break
 795                 if 'workloads' in cases['subtasks'][sind]:
 796                     if not cases['subtasks'][sind]['workloads']:
 797                         cases['subtasks'].pop(sind)
 798             # scenarios in subtasks
 799             for sind, subtask in reversed(list(
 800                     enumerate(cases.get('subtasks', [])))):
 801                 scenario = subtask.get('scenario', {})
 802                 for name in scenario.keys():
 803                     if self.in_iterable_re(name, black_tests):
 804                         cases['subtasks'].pop(sind)
 805                         break
 806
 807         with open(result_file_name, 'w', encoding='utf-8') as fname:
 808             template.dump(cases, fname)
 809
 810     def build_task_args(self, test_name):
 811         """Build arguments for the Rally task."""
 812         task_args = {}
 813         if self.ext_net:
 814             task_args['floating_network'] = str(self.ext_net.name)
 815         else:
 816             task_args['floating_network'] = ''
 817         task_args['image_name'] = str(self.image.name)
 818         task_args['flavor_name'] = str(self.flavor.name)
 819         return task_args
 820
 821     def prepare_task(self, test_name):
 822         """Prepare resources for test run."""
 823         jobs_dir = os.path.join(
 824             getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
 825         task_name = self.task_yaml.get(test_name).get("task")
 826         task = os.path.join(jobs_dir, task_name)
 827         if not os.path.exists(task):
 828             raise Exception(f"The scenario '{task}' does not exist.")
 829         LOGGER.debug('Scenario fetched from : %s', task)
 830
 831         if not os.path.exists(self.temp_dir):
 832             os.makedirs(self.temp_dir)
 833         task_file_name = os.path.join(self.temp_dir, task_name)
 834         self.apply_blacklist(task, task_file_name)
 835         self.run_cmd = (["rally", "task", "start", "--tag", test_name,
 836                          "--task", task_file_name,
 837                          "--task-args", str(self.build_task_args(test_name))])
 838         return True