testcases/OpenStack/tempest/run_tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Description:
   4 #    Runs tempest and pushes the results to the DB
   5 #
   6 # Authors:
   7 #    morgan.richomme@orange.com
   8 #    jose.lausuch@ericsson.com
   9 #    viktor.tikkanen@nokia.com
  10 #
  11 # All rights reserved. This program and the accompanying materials
  12 # are made available under the terms of the Apache License, Version 2.0
  13 # which accompanies this distribution, and is available at
  14 # http://www.apache.org/licenses/LICENSE-2.0
  15 #
  16 import argparse
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import sys
  22 import time
  23 import yaml
  24 import ConfigParser
  25
  26 import keystoneclient.v2_0.client as ksclient
  27 from glanceclient import client as glanceclient
  28 from neutronclient.v2_0 import client as neutronclient
  29
  30 import functest.utils.functest_logger as ft_logger
  31 import functest.utils.functest_utils as ft_utils
  32 import functest.utils.openstack_utils as os_utils
  33
  34 modes = ['full', 'smoke', 'baremetal', 'compute', 'data_processing',
  35          'identity', 'image', 'network', 'object_storage', 'orchestration',
  36          'telemetry', 'volume', 'custom', 'defcore', 'feature_multisite']
  37
  38 """ tests configuration """
  39 parser = argparse.ArgumentParser()
  40 parser.add_argument("-d", "--debug",
  41                     help="Debug mode",
  42                     action="store_true")
  43 parser.add_argument("-s", "--serial",
  44                     help="Run tests in one thread",
  45                     action="store_true")
  46 parser.add_argument("-m", "--mode",
  47                     help="Tempest test mode [smoke, all]",
  48                     default="smoke")
  49 parser.add_argument("-r", "--report",
  50                     help="Create json result file",
  51                     action="store_true")
  52 parser.add_argument("-n", "--noclean",
  53                     help="Don't clean the created resources for this test.",
  54                     action="store_true")
  55
  56 args = parser.parse_args()
  57
  58 """ logging configuration """
  59 logger = ft_logger.Logger("run_tempest").getLogger()
  60
  61 REPO_PATH = os.environ['repos_dir'] + '/functest/'
  62
  63
  64 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
  65     functest_yaml = yaml.safe_load(f)
  66 f.close()
  67 TEST_DB = functest_yaml.get("results").get("test_db_url")
  68
  69 MODE = "smoke"
  70 GLANCE_IMAGE_NAME = functest_yaml.get("general").get(
  71     "openstack").get("image_name")
  72 GLANCE_IMAGE_FILENAME = functest_yaml.get("general").get(
  73     "openstack").get("image_file_name")
  74 GLANCE_IMAGE_FORMAT = functest_yaml.get("general").get(
  75     "openstack").get("image_disk_format")
  76 GLANCE_IMAGE_PATH = functest_yaml.get("general").get("directories").get(
  77     "dir_functest_data") + "/" + GLANCE_IMAGE_FILENAME
  78 PRIVATE_NET_NAME = functest_yaml.get("tempest").get("private_net_name")
  79 PRIVATE_SUBNET_NAME = functest_yaml.get("tempest").get("private_subnet_name")
  80 PRIVATE_SUBNET_CIDR = functest_yaml.get("tempest").get("private_subnet_cidr")
  81 ROUTER_NAME = functest_yaml.get("tempest").get("router_name")
  82 TENANT_NAME = functest_yaml.get("tempest").get("identity").get("tenant_name")
  83 TENANT_DESCRIPTION = functest_yaml.get("tempest").get("identity").get(
  84     "tenant_description")
  85 USER_NAME = functest_yaml.get("tempest").get("identity").get("user_name")
  86 USER_PASSWORD = functest_yaml.get("tempest").get("identity").get(
  87     "user_password")
  88 DEPLOYMENT_MAME = functest_yaml.get("rally").get("deployment_name")
  89 RALLY_INSTALLATION_DIR = functest_yaml.get("general").get("directories").get(
  90     "dir_rally_inst")
  91 RESULTS_DIR = functest_yaml.get("general").get("directories").get(
  92     "dir_results")
  93 TEMPEST_RESULTS_DIR = RESULTS_DIR + '/tempest'
  94 TEST_LIST_DIR = functest_yaml.get("general").get("directories").get(
  95     "dir_tempest_cases")
  96 TEMPEST_CUSTOM = REPO_PATH + TEST_LIST_DIR + 'test_list.txt'
  97 TEMPEST_BLACKLIST = REPO_PATH + TEST_LIST_DIR + 'blacklist.txt'
  98 TEMPEST_DEFCORE = REPO_PATH + TEST_LIST_DIR + 'defcore_req.txt'
  99 TEMPEST_RAW_LIST = TEMPEST_RESULTS_DIR + '/test_raw_list.txt'
 100 TEMPEST_LIST = TEMPEST_RESULTS_DIR + '/test_list.txt'
 101
 102
 103 def get_info(file_result):
 104     test_run = ""
 105     duration = ""
 106     test_failed = ""
 107
 108     p = subprocess.Popen('cat tempest.log',
 109                          shell=True, stdout=subprocess.PIPE,
 110                          stderr=subprocess.STDOUT)
 111     for line in p.stdout.readlines():
 112         # print line,
 113         if (len(test_run) < 1):
 114             test_run = re.findall("[0-9]*\.[0-9]*s", line)
 115         if (len(duration) < 1):
 116             duration = re.findall("[0-9]*\ tests", line)
 117         regexp = r"(failures=[0-9]+)"
 118         if (len(test_failed) < 1):
 119             test_failed = re.findall(regexp, line)
 120
 121     logger.debug("test_run:" + test_run)
 122     logger.debug("duration:" + duration)
 123
 124
 125 def create_tempest_resources():
 126     ks_creds = os_utils.get_credentials("keystone")
 127     logger.debug("Creating tenant and user for Tempest suite")
 128     keystone = ksclient.Client(**ks_creds)
 129     tenant_id = os_utils.create_tenant(keystone,
 130                                        TENANT_NAME,
 131                                        TENANT_DESCRIPTION)
 132     if tenant_id == '':
 133         logger.error("Error : Failed to create %s tenant" % TENANT_NAME)
 134
 135     user_id = os_utils.create_user(keystone, USER_NAME, USER_PASSWORD,
 136                                    None, tenant_id)
 137     if user_id == '':
 138         logger.error("Error : Failed to create %s user" % USER_NAME)
 139
 140     logger.debug("Creating private network for Tempest suite")
 141     creds_neutron = os_utils.get_credentials("neutron")
 142     neutron_client = neutronclient.Client(**creds_neutron)
 143     network_dic = os_utils.create_network_full(logger,
 144                                                neutron_client,
 145                                                PRIVATE_NET_NAME,
 146                                                PRIVATE_SUBNET_NAME,
 147                                                ROUTER_NAME,
 148                                                PRIVATE_SUBNET_CIDR)
 149     if network_dic:
 150         if not os_utils.update_neutron_net(neutron_client,
 151                                            network_dic['net_id'],
 152                                            shared=True):
 153             logger.error("Failed to update private network...")
 154             exit(-1)
 155         else:
 156             logger.debug("Network '%s' is available..." % PRIVATE_NET_NAME)
 157     else:
 158         logger.error("Private network creation failed")
 159         exit(-1)
 160
 161     logger.debug("Creating image for Tempest suite")
 162     glance_endpoint = keystone.service_catalog.url_for(
 163         service_type='image', endpoint_type='publicURL')
 164     glance_client = glanceclient.Client(1, glance_endpoint,
 165                                         token=keystone.auth_token)
 166     # Check if the given image exists
 167     image_id = os_utils.get_image_id(glance_client, GLANCE_IMAGE_NAME)
 168     if image_id != '':
 169         logger.info("Using existing image '%s'..." % GLANCE_IMAGE_NAME)
 170     else:
 171         logger.info("Creating image '%s' from '%s'..." % (GLANCE_IMAGE_NAME,
 172                                                           GLANCE_IMAGE_PATH))
 173         image_id = os_utils.create_glance_image(glance_client,
 174                                                 GLANCE_IMAGE_NAME,
 175                                                 GLANCE_IMAGE_PATH)
 176         if not image_id:
 177             logger.error("Failed to create a Glance image...")
 178             exit(-1)
 179         logger.debug("Image '%s' with ID=%s created successfully."
 180                      % (GLANCE_IMAGE_NAME, image_id))
 181
 182
 183 def configure_tempest(deployment_dir):
 184     """
 185     Add/update needed parameters into tempest.conf file generated by Rally
 186     """
 187
 188     logger.debug("Generating tempest.conf file...")
 189     cmd = "rally verify genconfig"
 190     ft_utils.execute_command(cmd, logger)
 191
 192     logger.debug("Finding tempest.conf file...")
 193     tempest_conf_file = deployment_dir + "/tempest.conf"
 194     if not os.path.isfile(tempest_conf_file):
 195         logger.error("Tempest configuration file %s NOT found."
 196                      % tempest_conf_file)
 197         exit(-1)
 198
 199     logger.debug("Updating selected tempest.conf parameters...")
 200     config = ConfigParser.RawConfigParser()
 201     config.read(tempest_conf_file)
 202     config.set('compute', 'fixed_network_name', PRIVATE_NET_NAME)
 203     config.set('identity', 'tenant_name', TENANT_NAME)
 204     config.set('identity', 'username', USER_NAME)
 205     config.set('identity', 'password', USER_PASSWORD)
 206     with open(tempest_conf_file, 'wb') as config_file:
 207         config.write(config_file)
 208
 209     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
 210     shutil.copyfile(tempest_conf_file, TEMPEST_RESULTS_DIR + '/tempest.conf')
 211     return True
 212
 213
 214 def configure_tempest_feature(deployment_dir, mode):
 215     """
 216     Add/update needed parameters into tempest.conf file generated by Rally
 217     """
 218
 219     logger.debug("Finding tempest.conf file...")
 220     tempest_conf_file = deployment_dir + "/tempest.conf"
 221     if not os.path.isfile(tempest_conf_file):
 222         logger.error("Tempest configuration file %s NOT found."
 223                      % tempest_conf_file)
 224         exit(-1)
 225
 226     logger.debug("Updating selected tempest.conf parameters...")
 227     config = ConfigParser.RawConfigParser()
 228     config.read(tempest_conf_file)
 229     if mode == 'feature_multisite':
 230         config.set('service_available', 'kingbird', 'true')
 231         cmd = "openstack endpoint show kingbird | grep publicurl |\
 232                awk '{print $4}' | awk -F '/' '{print $3}'"
 233         kingbird_endpoint_url = os.popen(cmd).read()
 234         cmd = "openstack endpoint show kingbird | grep publicurl |\
 235                awk '{print $4}' | awk -F '/' '{print $4}'"
 236         kingbird_api_version = os.popen(cmd).read()
 237         try:
 238             config.add_section("kingbird")
 239         except:
 240             logger.info('kingbird section exist')
 241         config.set('kingbird', 'endpoint_type', 'publicURL')
 242         config.set('kingbird', 'TIME_TO_SYNC', '20')
 243         config.set('kingbird', 'endpoint_url', kingbird_endpoint_url)
 244         config.set('kingbird', 'api_version', kingbird_api_version)
 245     with open(tempest_conf_file, 'wb') as config_file:
 246         config.write(config_file)
 247
 248     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
 249     shutil.copyfile(tempest_conf_file, TEMPEST_RESULTS_DIR + '/tempest.conf')
 250     return True
 251
 252
 253 def read_file(filename):
 254     with open(filename) as src:
 255         return [line.strip() for line in src.readlines()]
 256
 257
 258 def generate_test_list(deployment_dir, mode):
 259     logger.debug("Generating test case list...")
 260     if mode == 'defcore':
 261         shutil.copyfile(TEMPEST_DEFCORE, TEMPEST_RAW_LIST)
 262     elif mode == 'custom':
 263         if os.path.isfile(TEMPEST_CUSTOM):
 264             shutil.copyfile(TEMPEST_CUSTOM, TEMPEST_RAW_LIST)
 265         else:
 266             logger.error("Tempest test list file %s NOT found."
 267                          % TEMPEST_CUSTOM)
 268             exit(-1)
 269     else:
 270         if mode == 'smoke':
 271             testr_mode = "smoke"
 272         elif mode == 'feature_multisite':
 273             testr_mode = " | grep kingbird "
 274         elif mode == 'full':
 275             testr_mode = ""
 276         else:
 277             testr_mode = 'tempest.api.' + mode
 278         cmd = ("cd " + deployment_dir + ";" + "testr list-tests " +
 279                testr_mode + ">" + TEMPEST_RAW_LIST + ";cd")
 280         ft_utils.execute_command(cmd, logger)
 281
 282
 283 def apply_tempest_blacklist():
 284     logger.debug("Applying tempest blacklist...")
 285     cases_file = read_file(TEMPEST_RAW_LIST)
 286     result_file = open(TEMPEST_LIST, 'w')
 287     try:
 288         black_file = read_file(TEMPEST_BLACKLIST)
 289     except:
 290         black_file = ''
 291         logger.debug("Tempest blacklist file does not exist.")
 292     for line in cases_file:
 293         if line not in black_file:
 294             result_file.write(str(line) + '\n')
 295     result_file.close()
 296
 297
 298 def run_tempest(OPTION):
 299     #
 300     # the "main" function of the script which launches Rally to run Tempest
 301     # :param option: tempest option (smoke, ..)
 302     # :return: void
 303     #
 304     logger.info("Starting Tempest test suite: '%s'." % OPTION)
 305     start_time = time.time()
 306     stop_time = start_time
 307     cmd_line = "rally verify start " + OPTION + " --system-wide"
 308
 309     header = ("Tempest environment:\n"
 310               "  Installer: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 311               (os.getenv('INSTALLER_TYPE', 'Unknown'),
 312                os.getenv('DEPLOY_SCENARIO', 'Unknown'),
 313                os.getenv('NODE_NAME', 'Unknown'),
 314                time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 315
 316     f_stdout = open(TEMPEST_RESULTS_DIR + "/tempest.log", 'w+')
 317     f_stderr = open(TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+')
 318     f_env = open(TEMPEST_RESULTS_DIR + "/environment.log", 'w+')
 319     f_env.write(header)
 320
 321     subprocess.call(cmd_line, shell=True, stdout=f_stdout, stderr=f_stderr)
 322
 323     f_stdout.close()
 324     f_stderr.close()
 325     f_env.close()
 326
 327     cmd_line = "rally verify show"
 328     ft_utils.execute_command(cmd_line, logger,
 329                              exit_on_error=True, info=True)
 330
 331     cmd_line = "rally verify list"
 332     logger.debug('Executing command : {}'.format(cmd_line))
 333     cmd = os.popen(cmd_line)
 334     output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|")
 335     # Format:
 336     # | UUID | Deployment UUID | smoke | tests | failures | Created at |
 337     # Duration | Status  |
 338     num_tests = output[4]
 339     num_failures = output[5]
 340     time_start = output[6]
 341     duration = output[7]
 342     # Compute duration (lets assume it does not take more than 60 min)
 343     dur_min = int(duration.split(':')[1])
 344     dur_sec_float = float(duration.split(':')[2])
 345     dur_sec_int = int(round(dur_sec_float, 0))
 346     dur_sec_int = dur_sec_int + 60 * dur_min
 347     stop_time = time.time()
 348     # Push results in payload of testcase
 349     if args.report:
 350         logger.debug("Pushing tempest results into DB...")
 351         # Note criteria hardcoded...TODO move to testcase.yaml
 352         status = "FAIL"
 353         try:
 354             diff = (int(num_tests) - int(num_failures))
 355             success_rate = 100 * diff / int(num_tests)
 356         except:
 357             success_rate = 0
 358
 359         # For Tempest we assume that the success rate is above 90%
 360         if success_rate >= 90:
 361             status = "PASS"
 362
 363         # add the test in error in the details sections
 364         # should be possible to do it during the test
 365         with open(TEMPEST_RESULTS_DIR + "/tempest.log", 'r') as myfile:
 366             output = myfile.read()
 367         error_logs = ""
 368
 369         for match in re.findall('(.*?)[. ]*FAILED', output):
 370                 error_logs += match
 371
 372         # Generate json results for DB
 373         json_results = {"timestart": time_start, "duration": dur_sec_int,
 374                         "tests": int(num_tests), "failures": int(num_failures),
 375                         "errors": error_logs}
 376         logger.info("Results: " + str(json_results))
 377         # split Tempest smoke and full
 378         if "smoke" in args.mode:
 379             case_name = "tempest_smoke_serial"
 380         else:
 381             case_name = "tempest_full_parallel"
 382
 383         try:
 384             ft_utils.push_results_to_db("functest",
 385                                         case_name,
 386                                         None,
 387                                         start_time,
 388                                         stop_time,
 389                                         status,
 390                                         json_results)
 391         except:
 392             logger.error("Error pushing results into Database '%s'"
 393                          % sys.exc_info()[0])
 394
 395
 396 def main():
 397     global MODE
 398
 399     if not (args.mode in modes):
 400         logger.error("Tempest mode not valid. "
 401                      "Possible values are:\n" + str(modes))
 402         exit(-1)
 403
 404     if not os.path.exists(TEMPEST_RESULTS_DIR):
 405         os.makedirs(TEMPEST_RESULTS_DIR)
 406
 407     deployment_dir = ft_utils.get_deployment_dir(logger)
 408     configure_tempest(deployment_dir)
 409     configure_tempest_feature(deployment_dir, args.mode)
 410     create_tempest_resources()
 411     generate_test_list(deployment_dir, args.mode)
 412     apply_tempest_blacklist()
 413
 414     MODE = "--tests-file " + TEMPEST_LIST
 415     if args.serial:
 416         MODE += " --concur 1"
 417
 418     run_tempest(MODE)
 419
 420
 421 if __name__ == '__main__':
 422     main()