functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import subprocess
  20 import time
  21
  22 import pkg_resources
  23 import prettytable
  24 from six.moves import configparser
  25 from xtesting.core import testcase
  26 from xtesting.energy import energy
  27 import yaml
  28
  29 from functest.core import singlevm
  30 from functest.opnfv_tests.openstack.tempest import conf_utils
  31 from functest.utils import config
  32 from functest.utils import env
  33
  34 LOGGER = logging.getLogger(__name__)
  35
  36
  37 class RallyBase(singlevm.VmReady2):
  38     """Base class form Rally testcases implementation."""
  39
  40     # pylint: disable=too-many-instance-attributes
  41     TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
  42              'keystone', 'neutron', 'nova', 'quotas']
  43
  44     RALLY_DIR = pkg_resources.resource_filename(
  45         'functest', 'opnfv_tests/openstack/rally')
  46     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  47         'functest', 'opnfv_tests/openstack/rally/scenario')
  48     TEMPLATE_DIR = pkg_resources.resource_filename(
  49         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  50     SUPPORT_DIR = pkg_resources.resource_filename(
  51         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  52     USERS_AMOUNT = 2
  53     TENANTS_AMOUNT = 3
  54     ITERATIONS_AMOUNT = 10
  55     CONCURRENCY = 4
  56     RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally')
  57     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
  58     TEMP_DIR = os.path.join(RALLY_DIR, "var")
  59
  60     visibility = 'public'
  61     shared_network = True
  62
  63     def __init__(self, **kwargs):
  64         """Initialize RallyBase object."""
  65         super(RallyBase, self).__init__(**kwargs)
  66         assert self.orig_cloud
  67         assert self.project
  68         if self.orig_cloud.get_role("admin"):
  69             role_name = "admin"
  70         elif self.orig_cloud.get_role("Admin"):
  71             role_name = "Admin"
  72         else:
  73             raise Exception("Cannot detect neither admin nor Admin")
  74         self.orig_cloud.grant_role(
  75             role_name, user=self.project.user.id,
  76             project=self.project.project.id,
  77             domain=self.project.domain.id)
  78         self.creators = []
  79         self.summary = []
  80         self.scenario_dir = ''
  81         self.smoke = None
  82         self.test_name = None
  83         self.start_time = None
  84         self.result = None
  85         self.details = None
  86         self.compute_cnt = 0
  87         self.flavor_alt = None
  88         self.tests = []
  89         self.task_file = ''
  90         self.run_cmd = ''
  91
  92     def _build_task_args(self, test_file_name):
  93         """Build arguments for the Rally task."""
  94         task_args = {'service_list': [test_file_name]}
  95         task_args['image_name'] = str(self.image.name)
  96         task_args['flavor_name'] = str(self.flavor.name)
  97         task_args['flavor_alt_name'] = str(self.flavor_alt.name)
  98         task_args['glance_image_location'] = str(self.filename)
  99         task_args['glance_image_format'] = str(self.image_format)
 100         task_args['tmpl_dir'] = str(self.TEMPLATE_DIR)
 101         task_args['sup_dir'] = str(self.SUPPORT_DIR)
 102         task_args['users_amount'] = self.USERS_AMOUNT
 103         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 104         task_args['use_existing_users'] = False
 105         task_args['iterations'] = self.ITERATIONS_AMOUNT
 106         task_args['concurrency'] = self.CONCURRENCY
 107         task_args['smoke'] = self.smoke
 108
 109         if self.ext_net:
 110             task_args['floating_network'] = str(self.ext_net.name)
 111         else:
 112             task_args['floating_network'] = ''
 113
 114         if self.network:
 115             task_args['netid'] = str(self.network.id)
 116         else:
 117             task_args['netid'] = ''
 118
 119         return task_args
 120
 121     def _prepare_test_list(self, test_name):
 122         """Build the list of test cases to be executed."""
 123         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 124         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 125                                           test_yaml_file_name)
 126
 127         if not os.path.exists(scenario_file_name):
 128             scenario_file_name = os.path.join(self.scenario_dir,
 129                                               test_yaml_file_name)
 130
 131             if not os.path.exists(scenario_file_name):
 132                 raise Exception("The scenario '%s' does not exist."
 133                                 % scenario_file_name)
 134
 135         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 136         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 137
 138         if not os.path.exists(self.TEMP_DIR):
 139             os.makedirs(self.TEMP_DIR)
 140
 141         self._apply_blacklist(scenario_file_name, test_file_name)
 142         return test_file_name
 143
 144     @staticmethod
 145     def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
 146         """Set keystone_default_role in rally.conf"""
 147         if env.get("NEW_USER_ROLE").lower() != "member":
 148             rconfig = configparser.RawConfigParser()
 149             rconfig.read(rally_conf)
 150             if not rconfig.has_section('openstack'):
 151                 rconfig.add_section('openstack')
 152             rconfig.set(
 153                 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
 154             with open(rally_conf, 'wb') as config_file:
 155                 rconfig.write(config_file)
 156
 157     @staticmethod
 158     def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
 159         """Clean Rally config"""
 160         if env.get("NEW_USER_ROLE").lower() != "member":
 161             rconfig = configparser.RawConfigParser()
 162             rconfig.read(rally_conf)
 163             if rconfig.has_option('openstack', 'keystone_default_role'):
 164                 rconfig.remove_option('openstack', 'keystone_default_role')
 165             with open(rally_conf, 'wb') as config_file:
 166                 rconfig.write(config_file)
 167
 168     @staticmethod
 169     def get_task_id(cmd_raw):
 170         """
 171         Get task id from command rally result.
 172
 173         :param cmd_raw:
 174         :return: task_id as string
 175         """
 176         taskid_re = re.compile('^Task +(.*): started$')
 177         for line in cmd_raw.splitlines(True):
 178             line = line.strip()
 179             match = taskid_re.match(line)
 180             if match:
 181                 return match.group(1)
 182         return None
 183
 184     @staticmethod
 185     def task_succeed(json_raw):
 186         """
 187         Parse JSON from rally JSON results.
 188
 189         :param json_raw:
 190         :return: Bool
 191         """
 192         rally_report = json.loads(json_raw)
 193         tasks = rally_report.get('tasks')
 194         if tasks:
 195             for task in tasks:
 196                 if task.get('status') != 'finished' or \
 197                    task.get('pass_sla') is not True:
 198                     return False
 199         else:
 200             return False
 201         return True
 202
 203     def _migration_supported(self):
 204         """Determine if migration is supported."""
 205         if self.compute_cnt > 1:
 206             return True
 207
 208         return False
 209
 210     @staticmethod
 211     def excl_scenario():
 212         """Exclude scenario."""
 213         black_tests = []
 214         try:
 215             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 216                 black_list_yaml = yaml.safe_load(black_list_file)
 217
 218             deploy_scenario = env.get('DEPLOY_SCENARIO')
 219             if (bool(deploy_scenario) and
 220                     'scenario' in black_list_yaml.keys()):
 221                 for item in black_list_yaml['scenario']:
 222                     scenarios = item['scenarios']
 223                     in_it = RallyBase.in_iterable_re
 224                     if in_it(deploy_scenario, scenarios):
 225                         tests = item['tests']
 226                         black_tests.extend(tests)
 227         except Exception:  # pylint: disable=broad-except
 228             LOGGER.debug("Scenario exclusion not applied.")
 229
 230         return black_tests
 231
 232     @staticmethod
 233     def in_iterable_re(needle, haystack):
 234         """
 235         Check if given needle is in the iterable haystack, using regex.
 236
 237         :param needle: string to be matched
 238         :param haystack: iterable of strings (optionally regex patterns)
 239         :return: True if needle is eqial to any of the elements in haystack,
 240                  or if a nonempty regex pattern in haystack is found in needle.
 241         """
 242         # match without regex
 243         if needle in haystack:
 244             return True
 245
 246         for pattern in haystack:
 247             # match if regex pattern is set and found in the needle
 248             if pattern and re.search(pattern, needle) is not None:
 249                 return True
 250
 251         return False
 252
 253     def excl_func(self):
 254         """Exclude functionalities."""
 255         black_tests = []
 256         func_list = []
 257
 258         try:
 259             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 260                 black_list_yaml = yaml.safe_load(black_list_file)
 261
 262             if not self._migration_supported():
 263                 func_list.append("no_migration")
 264
 265             if 'functionality' in black_list_yaml.keys():
 266                 for item in black_list_yaml['functionality']:
 267                     functions = item['functions']
 268                     for func in func_list:
 269                         if func in functions:
 270                             tests = item['tests']
 271                             black_tests.extend(tests)
 272         except Exception:  # pylint: disable=broad-except
 273             LOGGER.debug("Functionality exclusion not applied.")
 274
 275         return black_tests
 276
 277     def _apply_blacklist(self, case_file_name, result_file_name):
 278         """Apply blacklist."""
 279         LOGGER.debug("Applying blacklist...")
 280         cases_file = open(case_file_name, 'r')
 281         result_file = open(result_file_name, 'w')
 282
 283         black_tests = list(set(self.excl_func() +
 284                                self.excl_scenario()))
 285
 286         if black_tests:
 287             LOGGER.debug("Blacklisted tests: %s", str(black_tests))
 288
 289         include = True
 290         for cases_line in cases_file:
 291             if include:
 292                 for black_tests_line in black_tests:
 293                     if re.search(black_tests_line,
 294                                  cases_line.strip().rstrip(':')):
 295                         include = False
 296                         break
 297                 else:
 298                     result_file.write(str(cases_line))
 299             else:
 300                 if cases_line.isspace():
 301                     include = True
 302
 303         cases_file.close()
 304         result_file.close()
 305
 306     @staticmethod
 307     def file_is_empty(file_name):
 308         """Determine is a file is empty."""
 309         try:
 310             if os.stat(file_name).st_size > 0:
 311                 return False
 312         except Exception:  # pylint: disable=broad-except
 313             pass
 314
 315         return True
 316
 317     def _save_results(self, test_name, task_id):
 318         """ Generate and save task execution results"""
 319         # check for result directory and create it otherwise
 320         if not os.path.exists(self.RESULTS_DIR):
 321             LOGGER.debug('%s does not exist, we create it.',
 322                          self.RESULTS_DIR)
 323             os.makedirs(self.RESULTS_DIR)
 324
 325         # put detailed result to log
 326         cmd = (["rally", "task", "detailed", "--uuid", task_id])
 327         LOGGER.debug('running command: %s', cmd)
 328         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 329         LOGGER.info("%s\n%s", " ".join(cmd), output)
 330
 331         # save report as JSON
 332         report_json_name = 'opnfv-{}.json'.format(test_name)
 333         report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
 334         cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
 335                 "--out", report_json_dir])
 336         LOGGER.debug('running command: %s', cmd)
 337         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 338         LOGGER.info("%s\n%s", " ".join(cmd), output)
 339
 340         # save report as HTML
 341         report_html_name = 'opnfv-{}.html'.format(test_name)
 342         report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
 343         cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
 344                 "--out", report_html_dir])
 345         LOGGER.debug('running command: %s', cmd)
 346         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 347         LOGGER.info("%s\n%s", " ".join(cmd), output)
 348
 349         json_results = open(report_json_dir).read()
 350         self._append_summary(json_results, test_name)
 351
 352         # parse JSON operation result
 353         if self.task_succeed(json_results):
 354             LOGGER.info('Test scenario: "%s" OK.', test_name)
 355         else:
 356             LOGGER.info('Test scenario: "%s" Failed.', test_name)
 357
 358     def run_task(self, test_name):
 359         """Run a task."""
 360         LOGGER.info('Starting test scenario "%s" ...', test_name)
 361         LOGGER.debug('running command: %s', self.run_cmd)
 362         proc = subprocess.Popen(self.run_cmd, stdout=subprocess.PIPE,
 363                                 stderr=subprocess.STDOUT)
 364         output = proc.communicate()[0]
 365
 366         task_id = self.get_task_id(output)
 367         LOGGER.debug('task_id : %s', task_id)
 368         if task_id is None:
 369             LOGGER.error("Failed to retrieve task_id")
 370             LOGGER.error("Result:\n%s", output)
 371             raise Exception("Failed to retrieve task id")
 372         self._save_results(test_name, task_id)
 373
 374     def _append_summary(self, json_raw, test_name):
 375         """Update statistics summary info."""
 376         nb_tests = 0
 377         nb_success = 0
 378         overall_duration = 0.0
 379
 380         rally_report = json.loads(json_raw)
 381         for task in rally_report.get('tasks'):
 382             for subtask in task.get('subtasks'):
 383                 for workload in subtask.get('workloads'):
 384                     if workload.get('full_duration'):
 385                         overall_duration += workload.get('full_duration')
 386
 387                     if workload.get('data'):
 388                         nb_tests += len(workload.get('data'))
 389
 390                     for result in workload.get('data'):
 391                         if not result.get('error'):
 392                             nb_success += 1
 393
 394         scenario_summary = {'test_name': test_name,
 395                             'overall_duration': overall_duration,
 396                             'nb_tests': nb_tests,
 397                             'nb_success': nb_success,
 398                             'task_status': self.task_succeed(json_raw)}
 399         self.summary.append(scenario_summary)
 400
 401     def prepare_run(self):
 402         """Prepare resources needed by test scenarios."""
 403         assert self.cloud
 404         LOGGER.debug('Validating the test name...')
 405         if self.test_name == 'all':
 406             self.tests = self.TESTS
 407         elif self.test_name in self.TESTS:
 408             self.tests = [self.test_name]
 409         else:
 410             raise Exception("Test name '%s' is invalid" % self.test_name)
 411
 412         self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
 413         if not os.path.exists(self.task_file):
 414             LOGGER.error("Task file '%s' does not exist.", self.task_file)
 415             raise Exception("Task file '{}' does not exist.".
 416                             format(self.task_file))
 417
 418         self.update_keystone_default_role()
 419         self.compute_cnt = len(self.cloud.list_hypervisors())
 420         self.flavor_alt = self.create_flavor_alt()
 421         LOGGER.debug("flavor: %s", self.flavor_alt)
 422
 423     def prepare_task(self, test_name):
 424         """Prepare resources for test run."""
 425         file_name = self._prepare_test_list(test_name)
 426         if self.file_is_empty(file_name):
 427             LOGGER.info('No tests for scenario "%s"', test_name)
 428             return False
 429         self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
 430                          "--task", self.task_file, "--task-args",
 431                          str(self._build_task_args(test_name))])
 432         return True
 433
 434     def run_tests(self):
 435         """Execute tests."""
 436         for test in self.tests:
 437             if self.prepare_task(test):
 438                 self.run_task(test)
 439
 440     def _generate_report(self):
 441         """Generate test execution summary report."""
 442         total_duration = 0.0
 443         total_nb_tests = 0
 444         total_nb_success = 0
 445         nb_modules = 0
 446         payload = []
 447
 448         res_table = prettytable.PrettyTable(
 449             padding_width=2,
 450             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 451         res_table.align['Module'] = "l"
 452         res_table.align['Duration'] = "r"
 453         res_table.align['Success'] = "r"
 454
 455         # for each scenario we draw a row for the table
 456         for item in self.summary:
 457             if item['task_status'] is True:
 458                 nb_modules += 1
 459             total_duration += item['overall_duration']
 460             total_nb_tests += item['nb_tests']
 461             total_nb_success += item['nb_success']
 462             try:
 463                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 464             except ZeroDivisionError:
 465                 success_avg = 0
 466             success_str = str("{:0.2f}".format(success_avg)) + '%'
 467             duration_str = time.strftime("%M:%S",
 468                                          time.gmtime(item['overall_duration']))
 469             res_table.add_row([item['test_name'], duration_str,
 470                                item['nb_tests'], success_str])
 471             payload.append({'module': item['test_name'],
 472                             'details': {'duration': item['overall_duration'],
 473                                         'nb tests': item['nb_tests'],
 474                                         'success': success_str}})
 475
 476         total_duration_str = time.strftime("%H:%M:%S",
 477                                            time.gmtime(total_duration))
 478         try:
 479             self.result = 100 * total_nb_success / total_nb_tests
 480         except ZeroDivisionError:
 481             self.result = 100
 482         success_rate = "{:0.2f}".format(self.result)
 483         success_rate_str = str(success_rate) + '%'
 484         res_table.add_row(["", "", "", ""])
 485         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 486                            success_rate_str])
 487
 488         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 489         LOGGER.info("Rally '%s' success_rate is %s%% in %s/%s modules",
 490                     self.case_name, success_rate, nb_modules,
 491                     len(self.summary))
 492         payload.append({'summary': {'duration': total_duration,
 493                                     'nb tests': total_nb_tests,
 494                                     'nb success': success_rate}})
 495         self.details = payload
 496
 497     def clean(self):
 498         """Cleanup of OpenStack resources. Should be called on completion."""
 499         self.clean_rally_conf()
 500         if self.flavor_alt:
 501             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 502         super(RallyBase, self).clean()
 503
 504     def is_successful(self):
 505         """The overall result of the test."""
 506         for item in self.summary:
 507             if item['task_status'] is False:
 508                 return testcase.TestCase.EX_TESTCASE_FAILED
 509
 510         return super(RallyBase, self).is_successful()
 511
 512     @energy.enable_recording
 513     def run(self, **kwargs):
 514         """Run testcase."""
 515         self.start_time = time.time()
 516         try:
 517             assert super(RallyBase, self).run(
 518                 **kwargs) == testcase.TestCase.EX_OK
 519             environ = dict(
 520                 os.environ,
 521                 OS_USERNAME=self.project.user.name,
 522                 OS_PROJECT_NAME=self.project.project.name,
 523                 OS_PROJECT_ID=self.project.project.id,
 524                 OS_PASSWORD=self.project.password)
 525             conf_utils.create_rally_deployment(environ=environ)
 526             self.prepare_run()
 527             self.run_tests()
 528             self._generate_report()
 529             res = testcase.TestCase.EX_OK
 530         except Exception as exc:   # pylint: disable=broad-except
 531             LOGGER.error('Error with run: %s', exc)
 532             self.result = 0
 533             res = testcase.TestCase.EX_RUN_ERROR
 534         self.stop_time = time.time()
 535         return res
 536
 537
 538 class RallySanity(RallyBase):
 539     """Rally sanity testcase implementation."""
 540
 541     def __init__(self, **kwargs):
 542         """Initialize RallySanity object."""
 543         if "case_name" not in kwargs:
 544             kwargs["case_name"] = "rally_sanity"
 545         super(RallySanity, self).__init__(**kwargs)
 546         self.test_name = 'all'
 547         self.smoke = True
 548         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 549
 550
 551 class RallyFull(RallyBase):
 552     """Rally full testcase implementation."""
 553
 554     def __init__(self, **kwargs):
 555         """Initialize RallyFull object."""
 556         if "case_name" not in kwargs:
 557             kwargs["case_name"] = "rally_full"
 558         super(RallyFull, self).__init__(**kwargs)
 559         self.test_name = 'all'
 560         self.smoke = False
 561         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')