functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 from __future__ import division
  12
  13 import json
  14 import logging
  15 import os
  16 import pkg_resources
  17 import re
  18 import subprocess
  19 import time
  20
  21 import iniparse
  22 import yaml
  23
  24 from functest.core import testcase
  25 from functest.utils.constants import CONST
  26 import functest.utils.openstack_utils as os_utils
  27
  28 logger = logging.getLogger(__name__)
  29
  30
  31 class RallyBase(testcase.OSGCTestCase):
  32     TESTS = ['authenticate', 'glance', 'cinder', 'heat', 'keystone',
  33              'neutron', 'nova', 'quotas', 'requests', 'vm', 'all']
  34     GLANCE_IMAGE_NAME = CONST.__getattribute__('openstack_image_name')
  35     GLANCE_IMAGE_FILENAME = CONST.__getattribute__('openstack_image_file_name')
  36     GLANCE_IMAGE_PATH = os.path.join(
  37         CONST.__getattribute__('dir_functest_images'),
  38         GLANCE_IMAGE_FILENAME)
  39     GLANCE_IMAGE_FORMAT = CONST.__getattribute__('openstack_image_disk_format')
  40     FLAVOR_NAME = "m1.tiny"
  41
  42     RALLY_DIR = pkg_resources.resource_filename(
  43         'functest', 'opnfv_tests/openstack/rally')
  44     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  45         'functest', 'opnfv_tests/openstack/rally/scenario')
  46     TEMPLATE_DIR = pkg_resources.resource_filename(
  47         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  48     SUPPORT_DIR = pkg_resources.resource_filename(
  49         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  50     USERS_AMOUNT = 2
  51     TENANTS_AMOUNT = 3
  52     ITERATIONS_AMOUNT = 10
  53     CONCURRENCY = 4
  54     RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'), 'rally')
  55     TEMPEST_CONF_FILE = os.path.join(CONST.__getattribute__('dir_results'),
  56                                      'tempest/tempest.conf')
  57     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
  58     TEMP_DIR = os.path.join(RALLY_DIR, "var")
  59
  60     CINDER_VOLUME_TYPE_NAME = "volume_test"
  61     RALLY_PRIVATE_NET_NAME = CONST.__getattribute__('rally_network_name')
  62     RALLY_PRIVATE_SUBNET_NAME = CONST.__getattribute__('rally_subnet_name')
  63     RALLY_PRIVATE_SUBNET_CIDR = CONST.__getattribute__('rally_subnet_cidr')
  64     RALLY_ROUTER_NAME = CONST.__getattribute__('rally_router_name')
  65
  66     def __init__(self, **kwargs):
  67         super(RallyBase, self).__init__(**kwargs)
  68         self.mode = ''
  69         self.summary = []
  70         self.scenario_dir = ''
  71         self.nova_client = os_utils.get_nova_client()
  72         self.neutron_client = os_utils.get_neutron_client()
  73         self.cinder_client = os_utils.get_cinder_client()
  74         self.network_dict = {}
  75         self.volume_type = None
  76         self.smoke = None
  77
  78     def _build_task_args(self, test_file_name):
  79         task_args = {'service_list': [test_file_name]}
  80         task_args['image_name'] = self.GLANCE_IMAGE_NAME
  81         task_args['flavor_name'] = self.FLAVOR_NAME
  82         task_args['glance_image_location'] = self.GLANCE_IMAGE_PATH
  83         task_args['glance_image_format'] = self.GLANCE_IMAGE_FORMAT
  84         task_args['tmpl_dir'] = self.TEMPLATE_DIR
  85         task_args['sup_dir'] = self.SUPPORT_DIR
  86         task_args['users_amount'] = self.USERS_AMOUNT
  87         task_args['tenants_amount'] = self.TENANTS_AMOUNT
  88         task_args['use_existing_users'] = False
  89         task_args['iterations'] = self.ITERATIONS_AMOUNT
  90         task_args['concurrency'] = self.CONCURRENCY
  91         task_args['smoke'] = self.smoke
  92
  93         ext_net = os_utils.get_external_net(self.neutron_client)
  94         if ext_net:
  95             task_args['floating_network'] = str(ext_net)
  96         else:
  97             task_args['floating_network'] = ''
  98
  99         net_id = self.network_dict['net_id']
 100         if net_id:
 101             task_args['netid'] = str(net_id)
 102         else:
 103             task_args['netid'] = ''
 104
 105         # get keystone auth endpoint
 106         task_args['request_url'] = CONST.__getattribute__('OS_AUTH_URL') or ''
 107
 108         return task_args
 109
 110     def _prepare_test_list(self, test_name):
 111         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 112         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 113                                           test_yaml_file_name)
 114
 115         if not os.path.exists(scenario_file_name):
 116             scenario_file_name = os.path.join(self.scenario_dir,
 117                                               test_yaml_file_name)
 118
 119             if not os.path.exists(scenario_file_name):
 120                 raise Exception("The scenario '%s' does not exist."
 121                                 % scenario_file_name)
 122
 123         logger.debug('Scenario fetched from : {}'.format(scenario_file_name))
 124         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 125
 126         if not os.path.exists(self.TEMP_DIR):
 127             os.makedirs(self.TEMP_DIR)
 128
 129         self.apply_blacklist(scenario_file_name, test_file_name)
 130         return test_file_name
 131
 132     @staticmethod
 133     def get_task_id(cmd_raw):
 134         """
 135         get task id from command rally result
 136         :param cmd_raw:
 137         :return: task_id as string
 138         """
 139         taskid_re = re.compile('^Task +(.*): started$')
 140         for line in cmd_raw.splitlines(True):
 141             line = line.strip()
 142             match = taskid_re.match(line)
 143             if match:
 144                 return match.group(1)
 145         return None
 146
 147     @staticmethod
 148     def task_succeed(json_raw):
 149         """
 150         Parse JSON from rally JSON results
 151         :param json_raw:
 152         :return: Bool
 153         """
 154         rally_report = json.loads(json_raw)
 155         for report in rally_report:
 156             if report is None or report.get('result') is None:
 157                 return False
 158
 159             for result in report.get('result'):
 160                 if result is None or len(result.get('error')) > 0:
 161                     return False
 162
 163         return True
 164
 165     @staticmethod
 166     def live_migration_supported():
 167         config = iniparse.ConfigParser()
 168         if (config.read(RallyBase.TEMPEST_CONF_FILE) and
 169                 config.has_section('compute-feature-enabled') and
 170                 config.has_option('compute-feature-enabled',
 171                                   'live_migration')):
 172             return config.getboolean('compute-feature-enabled',
 173                                      'live_migration')
 174
 175         return False
 176
 177     @staticmethod
 178     def get_cmd_output(proc):
 179         result = ""
 180         while proc.poll() is None:
 181             line = proc.stdout.readline()
 182             result += line
 183         return result
 184
 185     @staticmethod
 186     def excl_scenario():
 187         black_tests = []
 188         try:
 189             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 190                 black_list_yaml = yaml.safe_load(black_list_file)
 191
 192             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
 193             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 194             if (bool(installer_type) * bool(deploy_scenario)):
 195                 if 'scenario' in black_list_yaml.keys():
 196                     for item in black_list_yaml['scenario']:
 197                         scenarios = item['scenarios']
 198                         installers = item['installers']
 199                         if (deploy_scenario in scenarios and
 200                                 installer_type in installers):
 201                             tests = item['tests']
 202                             black_tests.extend(tests)
 203         except Exception:
 204             logger.debug("Scenario exclusion not applied.")
 205
 206         return black_tests
 207
 208     @staticmethod
 209     def excl_func():
 210         black_tests = []
 211         func_list = []
 212
 213         try:
 214             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 215                 black_list_yaml = yaml.safe_load(black_list_file)
 216
 217             if not RallyBase.live_migration_supported():
 218                 func_list.append("no_live_migration")
 219
 220             if 'functionality' in black_list_yaml.keys():
 221                 for item in black_list_yaml['functionality']:
 222                     functions = item['functions']
 223                     for func in func_list:
 224                         if func in functions:
 225                             tests = item['tests']
 226                             black_tests.extend(tests)
 227         except Exception:
 228             logger.debug("Functionality exclusion not applied.")
 229
 230         return black_tests
 231
 232     @staticmethod
 233     def apply_blacklist(case_file_name, result_file_name):
 234         logger.debug("Applying blacklist...")
 235         cases_file = open(case_file_name, 'r')
 236         result_file = open(result_file_name, 'w')
 237
 238         black_tests = list(set(RallyBase.excl_func() +
 239                            RallyBase.excl_scenario()))
 240
 241         include = True
 242         for cases_line in cases_file:
 243             if include:
 244                 for black_tests_line in black_tests:
 245                     if re.search(black_tests_line,
 246                                  cases_line.strip().rstrip(':')):
 247                         include = False
 248                         break
 249                 else:
 250                     result_file.write(str(cases_line))
 251             else:
 252                 if cases_line.isspace():
 253                     include = True
 254
 255         cases_file.close()
 256         result_file.close()
 257
 258     @staticmethod
 259     def file_is_empty(file_name):
 260         try:
 261             if os.stat(file_name).st_size > 0:
 262                 return False
 263         except:
 264             pass
 265
 266         return True
 267
 268     def _run_task(self, test_name):
 269         logger.info('Starting test scenario "{}" ...'.format(test_name))
 270
 271         task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
 272         if not os.path.exists(task_file):
 273             logger.error("Task file '%s' does not exist." % task_file)
 274             raise Exception("Task file '%s' does not exist." % task_file)
 275
 276         file_name = self._prepare_test_list(test_name)
 277         if self.file_is_empty(file_name):
 278             logger.info('No tests for scenario "{}"'.format(test_name))
 279             return
 280
 281         cmd_line = ("rally task start --abort-on-sla-failure "
 282                     "--task {0} "
 283                     "--task-args \"{1}\""
 284                     .format(task_file, self._build_task_args(test_name)))
 285         logger.debug('running command line: {}'.format(cmd_line))
 286
 287         p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
 288                              stderr=subprocess.STDOUT, shell=True)
 289         output = self._get_output(p, test_name)
 290         task_id = self.get_task_id(output)
 291         logger.debug('task_id : {}'.format(task_id))
 292
 293         if task_id is None:
 294             logger.error('Failed to retrieve task_id, validating task...')
 295             cmd_line = ("rally task validate "
 296                         "--task {0} "
 297                         "--task-args \"{1}\""
 298                         .format(task_file, self._build_task_args(test_name)))
 299             logger.debug('running command line: {}'.format(cmd_line))
 300             p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE,
 301                                  stderr=subprocess.STDOUT, shell=True)
 302             output = self.get_cmd_output(p)
 303             logger.error("Task validation result:" + "\n" + output)
 304             return
 305
 306         # check for result directory and create it otherwise
 307         if not os.path.exists(self.RESULTS_DIR):
 308             logger.debug('{} does not exist, we create it.'
 309                          .format(self.RESULTS_DIR))
 310             os.makedirs(self.RESULTS_DIR)
 311
 312         # write html report file
 313         report_html_name = 'opnfv-{}.html'.format(test_name)
 314         report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
 315         cmd_line = "rally task report {} --out {}".format(task_id,
 316                                                           report_html_dir)
 317
 318         logger.debug('running command line: {}'.format(cmd_line))
 319         os.popen(cmd_line)
 320
 321         # get and save rally operation JSON result
 322         cmd_line = "rally task results %s" % task_id
 323         logger.debug('running command line: {}'.format(cmd_line))
 324         cmd = os.popen(cmd_line)
 325         json_results = cmd.read()
 326         report_json_name = 'opnfv-{}.json'.format(test_name)
 327         report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
 328         with open(report_json_dir, 'w') as f:
 329             logger.debug('saving json file')
 330             f.write(json_results)
 331
 332         """ parse JSON operation result """
 333         if self.task_succeed(json_results):
 334             logger.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
 335         else:
 336             logger.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
 337
 338     def _get_output(self, proc, test_name):
 339         result = ""
 340         nb_tests = 0
 341         overall_duration = 0.0
 342         success = 0.0
 343         nb_totals = 0
 344
 345         while proc.poll() is None:
 346             line = proc.stdout.readline()
 347             if ("Load duration" in line or
 348                     "started" in line or
 349                     "finished" in line or
 350                     " Preparing" in line or
 351                     "+-" in line or
 352                     "|" in line):
 353                 result += line
 354             elif "test scenario" in line:
 355                 result += "\n" + line
 356             elif "Full duration" in line:
 357                 result += line + "\n\n"
 358
 359             # parse output for summary report
 360             if ("| " in line and
 361                     "| action" not in line and
 362                     "| Starting" not in line and
 363                     "| Completed" not in line and
 364                     "| ITER" not in line and
 365                     "|   " not in line and
 366                     "| total" not in line):
 367                 nb_tests += 1
 368             elif "| total" in line:
 369                 percentage = ((line.split('|')[8]).strip(' ')).strip('%')
 370                 try:
 371                     success += float(percentage)
 372                 except ValueError:
 373                     logger.info('Percentage error: %s, %s' %
 374                                 (percentage, line))
 375                 nb_totals += 1
 376             elif "Full duration" in line:
 377                 duration = line.split(': ')[1]
 378                 try:
 379                     overall_duration += float(duration)
 380                 except ValueError:
 381                     logger.info('Duration error: %s, %s' % (duration, line))
 382
 383         overall_duration = "{:10.2f}".format(overall_duration)
 384         if nb_totals == 0:
 385             success_avg = 0
 386         else:
 387             success_avg = "{:0.2f}".format(success / nb_totals)
 388
 389         scenario_summary = {'test_name': test_name,
 390                             'overall_duration': overall_duration,
 391                             'nb_tests': nb_tests,
 392                             'success': success_avg}
 393         self.summary.append(scenario_summary)
 394
 395         logger.debug("\n" + result)
 396
 397         return result
 398
 399     def _prepare_env(self):
 400         logger.debug('Validating the test name...')
 401         if not (self.test_name in self.TESTS):
 402             raise Exception("Test name '%s' is invalid" % self.test_name)
 403
 404         volume_types = os_utils.list_volume_types(self.cinder_client,
 405                                                   private=False)
 406         if volume_types:
 407             logger.debug("Using existing volume type(s)...")
 408         else:
 409             logger.debug('Creating volume type...')
 410             self.volume_type = os_utils.create_volume_type(
 411                 self.cinder_client, self.CINDER_VOLUME_TYPE_NAME)
 412             if self.volume_type is None:
 413                 raise Exception("Failed to create volume type '%s'" %
 414                                 self.CINDER_VOLUME_TYPE_NAME)
 415             logger.debug("Volume type '%s' is created succesfully." %
 416                          self.CINDER_VOLUME_TYPE_NAME)
 417
 418         logger.debug('Getting or creating image...')
 419         self.image_exists, self.image_id = os_utils.get_or_create_image(
 420             self.GLANCE_IMAGE_NAME,
 421             self.GLANCE_IMAGE_PATH,
 422             self.GLANCE_IMAGE_FORMAT)
 423         if self.image_id is None:
 424             raise Exception("Failed to get or create image '%s'" %
 425                             self.GLANCE_IMAGE_NAME)
 426
 427         logger.debug("Creating network '%s'..." % self.RALLY_PRIVATE_NET_NAME)
 428         self.network_dict = os_utils.create_shared_network_full(
 429             self.RALLY_PRIVATE_NET_NAME,
 430             self.RALLY_PRIVATE_SUBNET_NAME,
 431             self.RALLY_ROUTER_NAME,
 432             self.RALLY_PRIVATE_SUBNET_CIDR)
 433         if self.network_dict is None:
 434             raise Exception("Failed to create shared network '%s'" %
 435                             self.RALLY_PRIVATE_NET_NAME)
 436
 437     def _run_tests(self):
 438         if self.test_name == 'all':
 439             for test in self.TESTS:
 440                 if (test == 'all' or test == 'vm'):
 441                     continue
 442                 self._run_task(test)
 443         else:
 444             self._run_task(self.test_name)
 445
 446     def _generate_report(self):
 447         report = (
 448             "\n"
 449             "                                                              "
 450             "\n"
 451             "                     Rally Summary Report\n"
 452             "\n"
 453             "+===================+============+===============+===========+"
 454             "\n"
 455             "| Module            | Duration   | nb. Test Run  | Success   |"
 456             "\n"
 457             "+===================+============+===============+===========+"
 458             "\n")
 459         payload = []
 460
 461         # for each scenario we draw a row for the table
 462         total_duration = 0.0
 463         total_nb_tests = 0
 464         total_success = 0.0
 465         for s in self.summary:
 466             name = "{0:<17}".format(s['test_name'])
 467             duration = float(s['overall_duration'])
 468             total_duration += duration
 469             duration = time.strftime("%M:%S", time.gmtime(duration))
 470             duration = "{0:<10}".format(duration)
 471             nb_tests = "{0:<13}".format(s['nb_tests'])
 472             total_nb_tests += int(s['nb_tests'])
 473             success = "{0:<10}".format(str(s['success']) + '%')
 474             total_success += float(s['success'])
 475             report += ("" +
 476                        "| " + name + " | " + duration + " | " +
 477                        nb_tests + " | " + success + "|\n" +
 478                        "+-------------------+------------"
 479                        "+---------------+-----------+\n")
 480             payload.append({'module': name,
 481                             'details': {'duration': s['overall_duration'],
 482                                         'nb tests': s['nb_tests'],
 483                                         'success': s['success']}})
 484
 485         total_duration_str = time.strftime("%H:%M:%S",
 486                                            time.gmtime(total_duration))
 487         total_duration_str2 = "{0:<10}".format(total_duration_str)
 488         total_nb_tests_str = "{0:<13}".format(total_nb_tests)
 489
 490         try:
 491             self.result = total_success / len(self.summary)
 492         except ZeroDivisionError:
 493             self.result = 100
 494
 495         success_rate = "{:0.2f}".format(self.result)
 496         success_rate_str = "{0:<10}".format(str(success_rate) + '%')
 497         report += ("+===================+============"
 498                    "+===============+===========+")
 499         report += "\n"
 500         report += ("| TOTAL:            | " + total_duration_str2 + " | " +
 501                    total_nb_tests_str + " | " + success_rate_str + "|\n")
 502         report += ("+===================+============"
 503                    "+===============+===========+")
 504         report += "\n"
 505
 506         logger.info("\n" + report)
 507         payload.append({'summary': {'duration': total_duration,
 508                                     'nb tests': total_nb_tests,
 509                                     'nb success': success_rate}})
 510
 511         self.details = payload
 512
 513         logger.info("Rally '%s' success_rate is %s%%"
 514                     % (self.case_name, success_rate))
 515
 516     def _clean_up(self):
 517         if self.volume_type:
 518             logger.debug("Deleting volume type '%s'..." % self.volume_type)
 519             os_utils.delete_volume_type(self.cinder_client, self.volume_type)
 520
 521         if not self.image_exists:
 522             logger.debug("Deleting image '%s' with ID '%s'..."
 523                          % (self.GLANCE_IMAGE_NAME, self.image_id))
 524             if not os_utils.delete_glance_image(self.nova_client,
 525                                                 self.image_id):
 526                 logger.error("Error deleting the glance image")
 527
 528     def run(self):
 529         self.start_time = time.time()
 530         try:
 531             self._prepare_env()
 532             self._run_tests()
 533             self._generate_report()
 534             self._clean_up()
 535             res = testcase.TestCase.EX_OK
 536         except Exception as e:
 537             logger.error('Error with run: %s' % e)
 538             res = testcase.TestCase.EX_RUN_ERROR
 539
 540         self.stop_time = time.time()
 541         return res
 542
 543
 544 class RallySanity(RallyBase):
 545     def __init__(self, **kwargs):
 546         if "case_name" not in kwargs:
 547             kwargs["case_name"] = "rally_sanity"
 548         super(RallySanity, self).__init__(**kwargs)
 549         self.mode = 'sanity'
 550         self.test_name = 'all'
 551         self.smoke = True
 552         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 553
 554
 555 class RallyFull(RallyBase):
 556     def __init__(self, **kwargs):
 557         if "case_name" not in kwargs:
 558             kwargs["case_name"] = "rally_full"
 559         super(RallyFull, self).__init__(**kwargs)
 560         self.mode = 'full'
 561         self.test_name = 'all'
 562         self.smoke = False
 563         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')