functest/opnfv_tests/openstack/rally/rally.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Rally testcases implementation."""
  12
  13 from __future__ import division
  14
  15 import json
  16 import logging
  17 import os
  18 import re
  19 import subprocess
  20 import time
  21 import uuid
  22
  23 import pkg_resources
  24 import prettytable
  25 import yaml
  26
  27 from functest.core import testcase
  28 from functest.energy import energy
  29 from functest.opnfv_tests.openstack.snaps import snaps_utils
  30 from functest.opnfv_tests.openstack.tempest import conf_utils
  31 from functest.utils.constants import CONST
  32
  33 from snaps.config.flavor import FlavorConfig
  34 from snaps.config.image import ImageConfig
  35 from snaps.config.network import NetworkConfig, SubnetConfig
  36 from snaps.config.router import RouterConfig
  37
  38 from snaps.openstack.create_flavor import OpenStackFlavor
  39 from snaps.openstack.tests import openstack_tests
  40 from snaps.openstack.utils import deploy_utils
  41
  42 LOGGER = logging.getLogger(__name__)
  43
  44
  45 class RallyBase(testcase.TestCase):
  46     """Base class form Rally testcases implementation."""
  47
  48     TESTS = ['authenticate', 'glance', 'ceilometer', 'cinder', 'heat',
  49              'keystone', 'neutron', 'nova', 'quotas', 'vm', 'all']
  50     GLANCE_IMAGE_NAME = CONST.__getattribute__('openstack_image_name')
  51     GLANCE_IMAGE_FILENAME = CONST.__getattribute__('openstack_image_file_name')
  52     GLANCE_IMAGE_PATH = os.path.join(
  53         CONST.__getattribute__('dir_functest_images'),
  54         GLANCE_IMAGE_FILENAME)
  55     GLANCE_IMAGE_FORMAT = CONST.__getattribute__('openstack_image_disk_format')
  56     GLANCE_IMAGE_USERNAME = CONST.__getattribute__('openstack_image_username')
  57     GLANCE_IMAGE_EXTRA_PROPERTIES = {}
  58     if hasattr(CONST, 'openstack_extra_properties'):
  59         GLANCE_IMAGE_EXTRA_PROPERTIES = CONST.__getattribute__(
  60             'openstack_extra_properties')
  61     FLAVOR_NAME = CONST.__getattribute__('rally_flavor_name')
  62     FLAVOR_ALT_NAME = CONST.__getattribute__('rally_flavor_alt_name')
  63     FLAVOR_EXTRA_SPECS = None
  64     FLAVOR_RAM = 512
  65     FLAVOR_RAM_ALT = 1024
  66     if hasattr(CONST, 'flavor_extra_specs'):
  67         FLAVOR_EXTRA_SPECS = CONST.__getattribute__('flavor_extra_specs')
  68         FLAVOR_RAM = 1024
  69         FLAVOR_RAM_ALT = 2048
  70
  71     RALLY_DIR = pkg_resources.resource_filename(
  72         'functest', 'opnfv_tests/openstack/rally')
  73     RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
  74         'functest', 'opnfv_tests/openstack/rally/scenario')
  75     TEMPLATE_DIR = pkg_resources.resource_filename(
  76         'functest', 'opnfv_tests/openstack/rally/scenario/templates')
  77     SUPPORT_DIR = pkg_resources.resource_filename(
  78         'functest', 'opnfv_tests/openstack/rally/scenario/support')
  79     USERS_AMOUNT = 2
  80     TENANTS_AMOUNT = 3
  81     ITERATIONS_AMOUNT = 10
  82     CONCURRENCY = 4
  83     RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'), 'rally')
  84     BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
  85     TEMP_DIR = os.path.join(RALLY_DIR, "var")
  86
  87     RALLY_PRIVATE_NET_NAME = CONST.__getattribute__('rally_network_name')
  88     RALLY_PRIVATE_SUBNET_NAME = CONST.__getattribute__('rally_subnet_name')
  89     RALLY_PRIVATE_SUBNET_CIDR = CONST.__getattribute__('rally_subnet_cidr')
  90     RALLY_ROUTER_NAME = CONST.__getattribute__('rally_router_name')
  91
  92     def __init__(self, **kwargs):
  93         """Initialize RallyBase object."""
  94         super(RallyBase, self).__init__(**kwargs)
  95         if 'os_creds' in kwargs:
  96             self.os_creds = kwargs['os_creds']
  97         else:
  98             creds_override = None
  99             if hasattr(CONST, 'snaps_os_creds_override'):
 100                 creds_override = CONST.__getattribute__(
 101                     'snaps_os_creds_override')
 102
 103             self.os_creds = openstack_tests.get_credentials(
 104                 os_env_file=CONST.__getattribute__('openstack_creds'),
 105                 overrides=creds_override)
 106
 107         self.guid = '-' + str(uuid.uuid4())
 108
 109         self.creators = []
 110         self.mode = ''
 111         self.summary = []
 112         self.scenario_dir = ''
 113         self.image_name = None
 114         self.ext_net_name = None
 115         self.priv_net_id = None
 116         self.flavor_name = None
 117         self.flavor_alt_name = None
 118         self.smoke = None
 119         self.test_name = None
 120         self.start_time = None
 121         self.result = None
 122         self.details = None
 123         self.compute_cnt = 0
 124
 125     def _build_task_args(self, test_file_name):
 126         task_args = {'service_list': [test_file_name]}
 127         task_args['image_name'] = self.image_name
 128         task_args['flavor_name'] = self.flavor_name
 129         task_args['flavor_alt_name'] = self.flavor_alt_name
 130         task_args['glance_image_location'] = self.GLANCE_IMAGE_PATH
 131         task_args['glance_image_format'] = self.GLANCE_IMAGE_FORMAT
 132         task_args['tmpl_dir'] = self.TEMPLATE_DIR
 133         task_args['sup_dir'] = self.SUPPORT_DIR
 134         task_args['users_amount'] = self.USERS_AMOUNT
 135         task_args['tenants_amount'] = self.TENANTS_AMOUNT
 136         task_args['use_existing_users'] = False
 137         task_args['iterations'] = self.ITERATIONS_AMOUNT
 138         task_args['concurrency'] = self.CONCURRENCY
 139         task_args['smoke'] = self.smoke
 140
 141         ext_net = self.ext_net_name
 142         if ext_net:
 143             task_args['floating_network'] = str(ext_net)
 144         else:
 145             task_args['floating_network'] = ''
 146
 147         net_id = self.priv_net_id
 148         if net_id:
 149             task_args['netid'] = str(net_id)
 150         else:
 151             task_args['netid'] = ''
 152
 153         return task_args
 154
 155     def _prepare_test_list(self, test_name):
 156         test_yaml_file_name = 'opnfv-{}.yaml'.format(test_name)
 157         scenario_file_name = os.path.join(self.RALLY_SCENARIO_DIR,
 158                                           test_yaml_file_name)
 159
 160         if not os.path.exists(scenario_file_name):
 161             scenario_file_name = os.path.join(self.scenario_dir,
 162                                               test_yaml_file_name)
 163
 164             if not os.path.exists(scenario_file_name):
 165                 raise Exception("The scenario '%s' does not exist."
 166                                 % scenario_file_name)
 167
 168         LOGGER.debug('Scenario fetched from : %s', scenario_file_name)
 169         test_file_name = os.path.join(self.TEMP_DIR, test_yaml_file_name)
 170
 171         if not os.path.exists(self.TEMP_DIR):
 172             os.makedirs(self.TEMP_DIR)
 173
 174         self._apply_blacklist(scenario_file_name, test_file_name)
 175         return test_file_name
 176
 177     @staticmethod
 178     def get_task_id(cmd_raw):
 179         """
 180         Get task id from command rally result.
 181
 182         :param cmd_raw:
 183         :return: task_id as string
 184         """
 185         taskid_re = re.compile('^Task +(.*): started$')
 186         for line in cmd_raw.splitlines(True):
 187             line = line.strip()
 188             match = taskid_re.match(line)
 189             if match:
 190                 return match.group(1)
 191         return None
 192
 193     @staticmethod
 194     def task_succeed(json_raw):
 195         """
 196         Parse JSON from rally JSON results.
 197
 198         :param json_raw:
 199         :return: Bool
 200         """
 201         rally_report = json.loads(json_raw)
 202         for report in rally_report:
 203             if report is None or report.get('result') is None:
 204                 return False
 205
 206             for result in report.get('result'):
 207                 if result is None or len(result.get('error')) > 0:
 208                     return False
 209
 210         return True
 211
 212     def _migration_supported(self):
 213         """Determine if migration is supported."""
 214         if self.compute_cnt > 1:
 215             return True
 216
 217         return False
 218
 219     @staticmethod
 220     def get_cmd_output(proc):
 221         """Get command stdout."""
 222         result = ""
 223         for line in proc.stdout:
 224             result += line
 225         return result
 226
 227     @staticmethod
 228     def excl_scenario():
 229         """Exclude scenario."""
 230         black_tests = []
 231         try:
 232             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 233                 black_list_yaml = yaml.safe_load(black_list_file)
 234
 235             installer_type = CONST.__getattribute__('INSTALLER_TYPE')
 236             deploy_scenario = CONST.__getattribute__('DEPLOY_SCENARIO')
 237             if (bool(installer_type) and bool(deploy_scenario) and
 238                     'scenario' in black_list_yaml.keys()):
 239                 for item in black_list_yaml['scenario']:
 240                     scenarios = item['scenarios']
 241                     installers = item['installers']
 242                     in_it = RallyBase.in_iterable_re
 243                     if (in_it(deploy_scenario, scenarios) and
 244                             in_it(installer_type, installers)):
 245                         tests = item['tests']
 246                         black_tests.extend(tests)
 247         except Exception:
 248             LOGGER.debug("Scenario exclusion not applied.")
 249
 250         return black_tests
 251
 252     @staticmethod
 253     def in_iterable_re(needle, haystack):
 254         """
 255         Check if given needle is in the iterable haystack, using regex.
 256
 257         :param needle: string to be matched
 258         :param haystack: iterable of strings (optionally regex patterns)
 259         :return: True if needle is eqial to any of the elements in haystack,
 260                  or if a nonempty regex pattern in haystack is found in needle.
 261         """
 262         # match without regex
 263         if needle in haystack:
 264             return True
 265
 266         for pattern in haystack:
 267             # match if regex pattern is set and found in the needle
 268             if pattern and re.search(pattern, needle) is not None:
 269                 return True
 270         else:
 271             return False
 272
 273     def excl_func(self):
 274         """Exclude functionalities."""
 275         black_tests = []
 276         func_list = []
 277
 278         try:
 279             with open(RallyBase.BLACKLIST_FILE, 'r') as black_list_file:
 280                 black_list_yaml = yaml.safe_load(black_list_file)
 281
 282             if not self._migration_supported():
 283                 func_list.append("no_migration")
 284
 285             if 'functionality' in black_list_yaml.keys():
 286                 for item in black_list_yaml['functionality']:
 287                     functions = item['functions']
 288                     for func in func_list:
 289                         if func in functions:
 290                             tests = item['tests']
 291                             black_tests.extend(tests)
 292         except Exception:  # pylint: disable=broad-except
 293             LOGGER.debug("Functionality exclusion not applied.")
 294
 295         return black_tests
 296
 297     def _apply_blacklist(self, case_file_name, result_file_name):
 298         """Apply blacklist."""
 299         LOGGER.debug("Applying blacklist...")
 300         cases_file = open(case_file_name, 'r')
 301         result_file = open(result_file_name, 'w')
 302
 303         black_tests = list(set(self.excl_func() +
 304                                self.excl_scenario()))
 305
 306         if black_tests:
 307             LOGGER.debug("Blacklisted tests: " + str(black_tests))
 308
 309         include = True
 310         for cases_line in cases_file:
 311             if include:
 312                 for black_tests_line in black_tests:
 313                     if re.search(black_tests_line,
 314                                  cases_line.strip().rstrip(':')):
 315                         include = False
 316                         break
 317                 else:
 318                     result_file.write(str(cases_line))
 319             else:
 320                 if cases_line.isspace():
 321                     include = True
 322
 323         cases_file.close()
 324         result_file.close()
 325
 326     @staticmethod
 327     def file_is_empty(file_name):
 328         """Determine is a file is empty."""
 329         try:
 330             if os.stat(file_name).st_size > 0:
 331                 return False
 332         except Exception:  # pylint: disable=broad-except
 333             pass
 334
 335         return True
 336
 337     def _run_task(self, test_name):
 338         """Run a task."""
 339         LOGGER.info('Starting test scenario "%s" ...', test_name)
 340
 341         task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
 342         if not os.path.exists(task_file):
 343             LOGGER.error("Task file '%s' does not exist.", task_file)
 344             raise Exception("Task file '%s' does not exist.", task_file)
 345
 346         file_name = self._prepare_test_list(test_name)
 347         if self.file_is_empty(file_name):
 348             LOGGER.info('No tests for scenario "%s"', test_name)
 349             return
 350
 351         cmd = (["rally", "task", "start", "--abort-on-sla-failure", "--task",
 352                 task_file, "--task-args",
 353                 str(self._build_task_args(test_name))])
 354         LOGGER.debug('running command: %s', cmd)
 355
 356         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 357                                 stderr=subprocess.STDOUT)
 358         output = self.get_cmd_output(proc)
 359         task_id = self.get_task_id(output)
 360
 361         LOGGER.debug('task_id : %s', task_id)
 362
 363         if task_id is None:
 364             LOGGER.error('Failed to retrieve task_id, validating task...')
 365             cmd = (["rally", "task", "validate", "--task", task_file,
 366                     "--task-args", str(self._build_task_args(test_name))])
 367             LOGGER.debug('running command: %s', cmd)
 368             proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 369                                     stderr=subprocess.STDOUT)
 370             output = self.get_cmd_output(proc)
 371             LOGGER.error("Task validation result:" + "\n" + output)
 372             return
 373
 374         # check for result directory and create it otherwise
 375         if not os.path.exists(self.RESULTS_DIR):
 376             LOGGER.debug('%s does not exist, we create it.',
 377                          self.RESULTS_DIR)
 378             os.makedirs(self.RESULTS_DIR)
 379
 380         # get and save rally operation JSON result
 381         cmd = (["rally", "task", "detailed", task_id])
 382         LOGGER.debug('running command: %s', cmd)
 383         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 384                                 stderr=subprocess.STDOUT)
 385         json_detailed = self.get_cmd_output(proc)
 386         LOGGER.info('%s', json_detailed)
 387
 388         cmd = (["rally", "task", "results", task_id])
 389         LOGGER.debug('running command: %s', cmd)
 390         proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
 391                                 stderr=subprocess.STDOUT)
 392         json_results = self.get_cmd_output(proc)
 393         self._append_summary(json_results, test_name)
 394         report_json_name = 'opnfv-{}.json'.format(test_name)
 395         report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
 396         with open(report_json_dir, 'w') as r_file:
 397             LOGGER.debug('saving json file')
 398             r_file.write(json_results)
 399
 400         # write html report file
 401         report_html_name = 'opnfv-{}.html'.format(test_name)
 402         report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
 403         cmd = (["rally", "task", "report", task_id, "--out", report_html_dir])
 404         LOGGER.debug('running command: %s', cmd)
 405         subprocess.Popen(cmd, stdout=subprocess.PIPE,
 406                          stderr=subprocess.STDOUT)
 407
 408         # parse JSON operation result
 409         if self.task_succeed(json_results):
 410             LOGGER.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
 411         else:
 412             LOGGER.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
 413
 414     def _append_summary(self, json_raw, test_name):
 415         nb_tests = 0
 416         nb_success = 0
 417         overall_duration = 0.0
 418
 419         rally_report = json.loads(json_raw)
 420         for report in rally_report:
 421             if report.get('full_duration'):
 422                 overall_duration += report.get('full_duration')
 423
 424             if report.get('result'):
 425                 for result in report.get('result'):
 426                     nb_tests += 1
 427                     if not result.get('error'):
 428                         nb_success += 1
 429
 430         scenario_summary = {'test_name': test_name,
 431                             'overall_duration': overall_duration,
 432                             'nb_tests': nb_tests,
 433                             'nb_success': nb_success}
 434         self.summary.append(scenario_summary)
 435
 436     def _prepare_env(self):
 437         LOGGER.debug('Validating the test name...')
 438         if self.test_name not in self.TESTS:
 439             raise Exception("Test name '%s' is invalid" % self.test_name)
 440
 441         network_name = self.RALLY_PRIVATE_NET_NAME + self.guid
 442         subnet_name = self.RALLY_PRIVATE_SUBNET_NAME + self.guid
 443         router_name = self.RALLY_ROUTER_NAME + self.guid
 444         self.image_name = self.GLANCE_IMAGE_NAME + self.guid
 445         self.flavor_name = self.FLAVOR_NAME + self.guid
 446         self.flavor_alt_name = self.FLAVOR_ALT_NAME + self.guid
 447         self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds)
 448         self.compute_cnt = snaps_utils.get_active_compute_cnt(self.os_creds)
 449
 450         LOGGER.debug("Creating image '%s'...", self.image_name)
 451         image_creator = deploy_utils.create_image(
 452             self.os_creds, ImageConfig(
 453                 name=self.image_name,
 454                 image_file=self.GLANCE_IMAGE_PATH,
 455                 img_format=self.GLANCE_IMAGE_FORMAT,
 456                 image_user=self.GLANCE_IMAGE_USERNAME,
 457                 public=True,
 458                 extra_properties=self.GLANCE_IMAGE_EXTRA_PROPERTIES))
 459         if image_creator is None:
 460             raise Exception("Failed to create image")
 461         self.creators.append(image_creator)
 462
 463         LOGGER.debug("Creating network '%s'...", network_name)
 464
 465         rally_network_type = None
 466         rally_physical_network = None
 467         rally_segmentation_id = None
 468
 469         if hasattr(CONST, 'rally_network_type'):
 470             rally_network_type = CONST.__getattribute__(
 471                 'rally_network_type')
 472         if hasattr(CONST, 'rally_physical_network'):
 473             rally_physical_network = CONST.__getattribute__(
 474                 'rally_physical_network')
 475         if hasattr(CONST, 'rally_segmentation_id'):
 476             rally_segmentation_id = CONST.__getattribute__(
 477                 'rally_segmentation_id')
 478
 479         network_creator = deploy_utils.create_network(
 480             self.os_creds, NetworkConfig(
 481                 name=network_name,
 482                 shared=True,
 483                 network_type=rally_network_type,
 484                 physical_network=rally_physical_network,
 485                 segmentation_id=rally_segmentation_id,
 486                 subnet_settings=[SubnetConfig(
 487                     name=subnet_name,
 488                     cidr=self.RALLY_PRIVATE_SUBNET_CIDR)
 489                 ]))
 490         if network_creator is None:
 491             raise Exception("Failed to create private network")
 492         self.priv_net_id = network_creator.get_network().id
 493         self.creators.append(network_creator)
 494
 495         LOGGER.debug("Creating router '%s'...", router_name)
 496         router_creator = deploy_utils.create_router(
 497             self.os_creds, RouterConfig(
 498                 name=router_name,
 499                 external_gateway=self.ext_net_name,
 500                 internal_subnets=[subnet_name]))
 501         if router_creator is None:
 502             raise Exception("Failed to create router")
 503         self.creators.append(router_creator)
 504
 505         LOGGER.debug("Creating flavor '%s'...", self.flavor_name)
 506         flavor_creator = OpenStackFlavor(
 507             self.os_creds, FlavorConfig(
 508                 name=self.flavor_name, ram=self.FLAVOR_RAM, disk=1, vcpus=1,
 509                 metadata=self.FLAVOR_EXTRA_SPECS))
 510         if flavor_creator is None or flavor_creator.create() is None:
 511             raise Exception("Failed to create flavor")
 512         self.creators.append(flavor_creator)
 513
 514         LOGGER.debug("Creating flavor '%s'...", self.flavor_alt_name)
 515         flavor_alt_creator = OpenStackFlavor(
 516             self.os_creds, FlavorConfig(
 517                 name=self.flavor_alt_name, ram=self.FLAVOR_RAM_ALT, disk=1,
 518                 vcpus=1, metadata=self.FLAVOR_EXTRA_SPECS))
 519         if flavor_alt_creator is None or flavor_alt_creator.create() is None:
 520             raise Exception("Failed to create flavor")
 521         self.creators.append(flavor_alt_creator)
 522
 523     def _run_tests(self):
 524         if self.test_name == 'all':
 525             for test in self.TESTS:
 526                 if test == 'all' or test == 'vm':
 527                     continue
 528                 self._run_task(test)
 529         else:
 530             self._run_task(self.test_name)
 531
 532     def _generate_report(self):
 533         total_duration = 0.0
 534         total_nb_tests = 0
 535         total_nb_success = 0
 536         payload = []
 537
 538         res_table = prettytable.PrettyTable(
 539             padding_width=2,
 540             field_names=['Module', 'Duration', 'nb. Test Run', 'Success'])
 541         res_table.align['Module'] = "l"
 542         res_table.align['Duration'] = "r"
 543         res_table.align['Success'] = "r"
 544
 545         # for each scenario we draw a row for the table
 546         for item in self.summary:
 547             total_duration += item['overall_duration']
 548             total_nb_tests += item['nb_tests']
 549             total_nb_success += item['nb_success']
 550             try:
 551                 success_avg = 100 * item['nb_success'] / item['nb_tests']
 552             except ZeroDivisionError:
 553                 success_avg = 0
 554             success_str = str("{:0.2f}".format(success_avg)) + '%'
 555             duration_str = time.strftime("%M:%S",
 556                                          time.gmtime(item['overall_duration']))
 557             res_table.add_row([item['test_name'], duration_str,
 558                                item['nb_tests'], success_str])
 559             payload.append({'module': item['test_name'],
 560                             'details': {'duration': item['overall_duration'],
 561                                         'nb tests': item['nb_tests'],
 562                                         'success': success_str}})
 563
 564         total_duration_str = time.strftime("%H:%M:%S",
 565                                            time.gmtime(total_duration))
 566         try:
 567             self.result = 100 * total_nb_success / total_nb_tests
 568         except ZeroDivisionError:
 569             self.result = 100
 570         success_rate = "{:0.2f}".format(self.result)
 571         success_rate_str = str(success_rate) + '%'
 572         res_table.add_row(["", "", "", ""])
 573         res_table.add_row(["TOTAL:", total_duration_str, total_nb_tests,
 574                            success_rate_str])
 575
 576         LOGGER.info("Rally Summary Report:\n\n%s\n", res_table.get_string())
 577         LOGGER.info("Rally '%s' success_rate is %s%%",
 578                     self.case_name, success_rate)
 579         payload.append({'summary': {'duration': total_duration,
 580                                     'nb tests': total_nb_tests,
 581                                     'nb success': success_rate}})
 582         self.details = payload
 583
 584     def _clean_up(self):
 585         for creator in reversed(self.creators):
 586             try:
 587                 creator.clean()
 588             except Exception as e:
 589                 LOGGER.error('Unexpected error cleaning - %s', e)
 590
 591     @energy.enable_recording
 592     def run(self, **kwargs):
 593         """Run testcase."""
 594         self.start_time = time.time()
 595         try:
 596             conf_utils.create_rally_deployment()
 597             self._prepare_env()
 598             self._run_tests()
 599             self._generate_report()
 600             res = testcase.TestCase.EX_OK
 601         except Exception as exc:   # pylint: disable=broad-except
 602             LOGGER.error('Error with run: %s', exc)
 603             res = testcase.TestCase.EX_RUN_ERROR
 604         finally:
 605             self._clean_up()
 606
 607         self.stop_time = time.time()
 608         return res
 609
 610
 611 class RallySanity(RallyBase):
 612     """Rally sanity testcase implementation."""
 613
 614     def __init__(self, **kwargs):
 615         """Initialize RallySanity object."""
 616         if "case_name" not in kwargs:
 617             kwargs["case_name"] = "rally_sanity"
 618         super(RallySanity, self).__init__(**kwargs)
 619         self.mode = 'sanity'
 620         self.test_name = 'all'
 621         self.smoke = True
 622         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
 623
 624
 625 class RallyFull(RallyBase):
 626     """Rally full testcase implementation."""
 627
 628     def __init__(self, **kwargs):
 629         """Initialize RallyFull object."""
 630         if "case_name" not in kwargs:
 631             kwargs["case_name"] = "rally_full"
 632         super(RallyFull, self).__init__(**kwargs)
 633         self.mode = 'full'
 634         self.test_name = 'all'
 635         self.smoke = False
 636         self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')