testcases/OpenStack/tempest/run_tempest.py

   1 #!/usr/bin/env python
   2 #
   3 # Description:
   4 #    Runs tempest and pushes the results to the DB
   5 #
   6 # Authors:
   7 #    morgan.richomme@orange.com
   8 #    jose.lausuch@ericsson.com
   9 #    viktor.tikkanen@nokia.com
  10 #
  11 # All rights reserved. This program and the accompanying materials
  12 # are made available under the terms of the Apache License, Version 2.0
  13 # which accompanies this distribution, and is available at
  14 # http://www.apache.org/licenses/LICENSE-2.0
  15 #
  16 import argparse
  17 import os
  18 import re
  19 import shutil
  20 import subprocess
  21 import sys
  22 import time
  23 import yaml
  24 import ConfigParser
  25
  26 import keystoneclient.v2_0.client as ksclient
  27 from glanceclient import client as glanceclient
  28 from neutronclient.v2_0 import client as neutronclient
  29
  30 import functest.utils.functest_logger as ft_logger
  31 import functest.utils.functest_utils as ft_utils
  32 import functest.utils.openstack_utils as os_utils
  33
  34 modes = ['full', 'smoke', 'baremetal', 'compute', 'data_processing',
  35          'identity', 'image', 'network', 'object_storage', 'orchestration',
  36          'telemetry', 'volume', 'custom', 'defcore']
  37
  38 """ tests configuration """
  39 parser = argparse.ArgumentParser()
  40 parser.add_argument("-d", "--debug",
  41                     help="Debug mode",
  42                     action="store_true")
  43 parser.add_argument("-s", "--serial",
  44                     help="Run tests in one thread",
  45                     action="store_true")
  46 parser.add_argument("-m", "--mode",
  47                     help="Tempest test mode [smoke, all]",
  48                     default="smoke")
  49 parser.add_argument("-r", "--report",
  50                     help="Create json result file",
  51                     action="store_true")
  52 parser.add_argument("-n", "--noclean",
  53                     help="Don't clean the created resources for this test.",
  54                     action="store_true")
  55
  56 args = parser.parse_args()
  57
  58 """ logging configuration """
  59 logger = ft_logger.Logger("run_tempest").getLogger()
  60
  61 REPO_PATH = os.environ['repos_dir'] + '/functest/'
  62
  63
  64 with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
  65     functest_yaml = yaml.safe_load(f)
  66 f.close()
  67 TEST_DB = functest_yaml.get("results").get("test_db_url")
  68
  69 MODE = "smoke"
  70 GLANCE_IMAGE_NAME = functest_yaml.get("general").get(
  71     "openstack").get("image_name")
  72 GLANCE_IMAGE_FILENAME = functest_yaml.get("general").get(
  73     "openstack").get("image_file_name")
  74 GLANCE_IMAGE_FORMAT = functest_yaml.get("general").get(
  75     "openstack").get("image_disk_format")
  76 GLANCE_IMAGE_PATH = functest_yaml.get("general").get("directories").get(
  77     "dir_functest_data") + "/" + GLANCE_IMAGE_FILENAME
  78 PRIVATE_NET_NAME = functest_yaml.get("tempest").get("private_net_name")
  79 PRIVATE_SUBNET_NAME = functest_yaml.get("tempest").get("private_subnet_name")
  80 PRIVATE_SUBNET_CIDR = functest_yaml.get("tempest").get("private_subnet_cidr")
  81 ROUTER_NAME = functest_yaml.get("tempest").get("router_name")
  82 TENANT_NAME = functest_yaml.get("tempest").get("identity").get("tenant_name")
  83 TENANT_DESCRIPTION = functest_yaml.get("tempest").get("identity").get(
  84     "tenant_description")
  85 USER_NAME = functest_yaml.get("tempest").get("identity").get("user_name")
  86 USER_PASSWORD = functest_yaml.get("tempest").get("identity").get(
  87     "user_password")
  88 DEPLOYMENT_MAME = functest_yaml.get("rally").get("deployment_name")
  89 RALLY_INSTALLATION_DIR = functest_yaml.get("general").get("directories").get(
  90     "dir_rally_inst")
  91 RESULTS_DIR = functest_yaml.get("general").get("directories").get(
  92     "dir_results")
  93 TEMPEST_RESULTS_DIR = RESULTS_DIR + '/tempest'
  94 TEST_LIST_DIR = functest_yaml.get("general").get("directories").get(
  95     "dir_tempest_cases")
  96 TEMPEST_CUSTOM = REPO_PATH + TEST_LIST_DIR + 'test_list.txt'
  97 TEMPEST_BLACKLIST = REPO_PATH + TEST_LIST_DIR + 'blacklist.txt'
  98 TEMPEST_DEFCORE = REPO_PATH + TEST_LIST_DIR + 'defcore_req.txt'
  99 TEMPEST_RAW_LIST = TEMPEST_RESULTS_DIR + '/test_raw_list.txt'
 100 TEMPEST_LIST = TEMPEST_RESULTS_DIR + '/test_list.txt'
 101
 102
 103 def get_info(file_result):
 104     test_run = ""
 105     duration = ""
 106     test_failed = ""
 107
 108     p = subprocess.Popen('cat tempest.log',
 109                          shell=True, stdout=subprocess.PIPE,
 110                          stderr=subprocess.STDOUT)
 111     for line in p.stdout.readlines():
 112         # print line,
 113         if (len(test_run) < 1):
 114             test_run = re.findall("[0-9]*\.[0-9]*s", line)
 115         if (len(duration) < 1):
 116             duration = re.findall("[0-9]*\ tests", line)
 117         regexp = r"(failures=[0-9]+)"
 118         if (len(test_failed) < 1):
 119             test_failed = re.findall(regexp, line)
 120
 121     logger.debug("test_run:" + test_run)
 122     logger.debug("duration:" + duration)
 123
 124
 125 def create_tempest_resources():
 126     ks_creds = os_utils.get_credentials("keystone")
 127     logger.debug("Creating tenant and user for Tempest suite")
 128     keystone = ksclient.Client(**ks_creds)
 129     tenant_id = os_utils.create_tenant(keystone,
 130                                        TENANT_NAME,
 131                                        TENANT_DESCRIPTION)
 132     if tenant_id == '':
 133         logger.error("Error : Failed to create %s tenant" % TENANT_NAME)
 134
 135     user_id = os_utils.create_user(keystone, USER_NAME, USER_PASSWORD,
 136                                    None, tenant_id)
 137     if user_id == '':
 138         logger.error("Error : Failed to create %s user" % USER_NAME)
 139
 140     logger.debug("Creating private network for Tempest suite")
 141     creds_neutron = os_utils.get_credentials("neutron")
 142     neutron_client = neutronclient.Client(**creds_neutron)
 143     network_dic = os_utils.create_network_full(logger,
 144                                                neutron_client,
 145                                                PRIVATE_NET_NAME,
 146                                                PRIVATE_SUBNET_NAME,
 147                                                ROUTER_NAME,
 148                                                PRIVATE_SUBNET_CIDR)
 149     if network_dic:
 150         if not os_utils.update_neutron_net(neutron_client,
 151                                            network_dic['net_id'],
 152                                            shared=True):
 153             logger.error("Failed to update private network...")
 154             exit(-1)
 155         else:
 156             logger.debug("Network '%s' is available..." % PRIVATE_NET_NAME)
 157     else:
 158         logger.error("Private network creation failed")
 159         exit(-1)
 160
 161     logger.debug("Creating image for Tempest suite")
 162     glance_endpoint = keystone.service_catalog.url_for(
 163         service_type='image', endpoint_type='publicURL')
 164     glance_client = glanceclient.Client(1, glance_endpoint,
 165                                         token=keystone.auth_token)
 166     # Check if the given image exists
 167     image_id = os_utils.get_image_id(glance_client, GLANCE_IMAGE_NAME)
 168     if image_id != '':
 169         logger.info("Using existing image '%s'..." % GLANCE_IMAGE_NAME)
 170     else:
 171         logger.info("Creating image '%s' from '%s'..." % (GLANCE_IMAGE_NAME,
 172                                                           GLANCE_IMAGE_PATH))
 173         image_id = os_utils.create_glance_image(glance_client,
 174                                                 GLANCE_IMAGE_NAME,
 175                                                 GLANCE_IMAGE_PATH)
 176         if not image_id:
 177             logger.error("Failed to create a Glance image...")
 178             exit(-1)
 179         logger.debug("Image '%s' with ID=%s created successfully."
 180                      % (GLANCE_IMAGE_NAME, image_id))
 181
 182
 183 def configure_tempest(deployment_dir):
 184     """
 185     Add/update needed parameters into tempest.conf file generated by Rally
 186     """
 187
 188     logger.debug("Generating tempest.conf file...")
 189     cmd = "rally verify genconfig"
 190     ft_utils.execute_command(cmd, logger)
 191
 192     logger.debug("Finding tempest.conf file...")
 193     tempest_conf_file = deployment_dir + "/tempest.conf"
 194     if not os.path.isfile(tempest_conf_file):
 195         logger.error("Tempest configuration file %s NOT found."
 196                      % tempest_conf_file)
 197         exit(-1)
 198
 199     logger.debug("Updating selected tempest.conf parameters...")
 200     config = ConfigParser.RawConfigParser()
 201     config.read(tempest_conf_file)
 202     config.set('compute', 'fixed_network_name', PRIVATE_NET_NAME)
 203     config.set('identity', 'tenant_name', TENANT_NAME)
 204     config.set('identity', 'username', USER_NAME)
 205     config.set('identity', 'password', USER_PASSWORD)
 206     with open(tempest_conf_file, 'wb') as config_file:
 207         config.write(config_file)
 208
 209     # Copy tempest.conf to /home/opnfv/functest/results/tempest/
 210     shutil.copyfile(tempest_conf_file, TEMPEST_RESULTS_DIR + '/tempest.conf')
 211     return True
 212
 213
 214 def read_file(filename):
 215     with open(filename) as src:
 216         return [line.strip() for line in src.readlines()]
 217
 218
 219 def generate_test_list(deployment_dir, mode):
 220     logger.debug("Generating test case list...")
 221     if mode == 'defcore':
 222         shutil.copyfile(TEMPEST_DEFCORE, TEMPEST_RAW_LIST)
 223     elif mode == 'custom':
 224         if os.path.isfile(TEMPEST_CUSTOM):
 225             shutil.copyfile(TEMPEST_CUSTOM, TEMPEST_RAW_LIST)
 226         else:
 227             logger.error("Tempest test list file %s NOT found."
 228                          % TEMPEST_CUSTOM)
 229             exit(-1)
 230     else:
 231         if mode == 'smoke':
 232             testr_mode = "smoke"
 233         elif mode == 'full':
 234             testr_mode = ""
 235         else:
 236             testr_mode = 'tempest.api.' + mode
 237         cmd = ("cd " + deployment_dir + ";" + "testr list-tests " +
 238                testr_mode + ">" + TEMPEST_RAW_LIST + ";cd")
 239         ft_utils.execute_command(cmd, logger)
 240
 241
 242 def apply_tempest_blacklist():
 243     logger.debug("Applying tempest blacklist...")
 244     cases_file = read_file(TEMPEST_RAW_LIST)
 245     result_file = open(TEMPEST_LIST, 'w')
 246     try:
 247         black_file = read_file(TEMPEST_BLACKLIST)
 248     except:
 249         black_file = ''
 250         logger.debug("Tempest blacklist file does not exist.")
 251     for line in cases_file:
 252         if line not in black_file:
 253             result_file.write(str(line) + '\n')
 254     result_file.close()
 255
 256
 257 def run_tempest(OPTION):
 258     #
 259     # the "main" function of the script which launches Rally to run Tempest
 260     # :param option: tempest option (smoke, ..)
 261     # :return: void
 262     #
 263     logger.info("Starting Tempest test suite: '%s'." % OPTION)
 264     start_time = time.time()
 265     stop_time = start_time
 266     cmd_line = "rally verify start " + OPTION + " --system-wide"
 267
 268     header = ("Tempest environment:\n"
 269               "  Installer: %s\n  Scenario: %s\n  Node: %s\n  Date: %s\n" %
 270               (os.getenv('INSTALLER_TYPE', 'Unknown'),
 271                os.getenv('DEPLOY_SCENARIO', 'Unknown'),
 272                os.getenv('NODE_NAME', 'Unknown'),
 273                time.strftime("%a %b %d %H:%M:%S %Z %Y")))
 274
 275     f_stdout = open(TEMPEST_RESULTS_DIR + "/tempest.log", 'w+')
 276     f_stderr = open(TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+')
 277     f_env = open(TEMPEST_RESULTS_DIR + "/environment.log", 'w+')
 278     f_env.write(header)
 279
 280     subprocess.call(cmd_line, shell=True, stdout=f_stdout, stderr=f_stderr)
 281
 282     f_stdout.close()
 283     f_stderr.close()
 284     f_env.close()
 285
 286     cmd_line = "rally verify show"
 287     ft_utils.execute_command(cmd_line, logger,
 288                              exit_on_error=True, info=True)
 289
 290     cmd_line = "rally verify list"
 291     logger.debug('Executing command : {}'.format(cmd_line))
 292     cmd = os.popen(cmd_line)
 293     output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|")
 294     # Format:
 295     # | UUID | Deployment UUID | smoke | tests | failures | Created at |
 296     # Duration | Status  |
 297     num_tests = output[4]
 298     num_failures = output[5]
 299     time_start = output[6]
 300     duration = output[7]
 301     # Compute duration (lets assume it does not take more than 60 min)
 302     dur_min = int(duration.split(':')[1])
 303     dur_sec_float = float(duration.split(':')[2])
 304     dur_sec_int = int(round(dur_sec_float, 0))
 305     dur_sec_int = dur_sec_int + 60 * dur_min
 306     stop_time = time.time()
 307     # Push results in payload of testcase
 308     if args.report:
 309         logger.debug("Pushing tempest results into DB...")
 310         # Note criteria hardcoded...TODO move to testcase.yaml
 311         status = "FAIL"
 312         try:
 313             diff = (int(num_tests) - int(num_failures))
 314             success_rate = 100 * diff / int(num_tests)
 315         except:
 316             success_rate = 0
 317
 318         # For Tempest we assume that the success rate is above 90%
 319         if success_rate >= 90:
 320             status = "PASS"
 321
 322         # add the test in error in the details sections
 323         # should be possible to do it during the test
 324         with open(TEMPEST_RESULTS_DIR + "/tempest.log", 'r') as myfile:
 325             output = myfile.read()
 326         error_logs = ""
 327
 328         for match in re.findall('(.*?)[. ]*FAILED', output):
 329                 error_logs += match
 330
 331         # Generate json results for DB
 332         json_results = {"timestart": time_start, "duration": dur_sec_int,
 333                         "tests": int(num_tests), "failures": int(num_failures),
 334                         "errors": error_logs}
 335         logger.info("Results: " + str(json_results))
 336         # split Tempest smoke and full
 337         if "smoke" in args.mode:
 338             case_name = "tempest_smoke_serial"
 339         else:
 340             case_name = "tempest_full_parallel"
 341
 342         try:
 343             ft_utils.push_results_to_db("functest",
 344                                         case_name,
 345                                         None,
 346                                         start_time,
 347                                         stop_time,
 348                                         status,
 349                                         json_results)
 350         except:
 351             logger.error("Error pushing results into Database '%s'"
 352                          % sys.exc_info()[0])
 353
 354
 355 def main():
 356     global MODE
 357
 358     if not (args.mode in modes):
 359         logger.error("Tempest mode not valid. "
 360                      "Possible values are:\n" + str(modes))
 361         exit(-1)
 362
 363     if not os.path.exists(TEMPEST_RESULTS_DIR):
 364         os.makedirs(TEMPEST_RESULTS_DIR)
 365
 366     deployment_dir = ft_utils.get_deployment_dir(logger)
 367     configure_tempest(deployment_dir)
 368     create_tempest_resources()
 369     generate_test_list(deployment_dir, args.mode)
 370     apply_tempest_blacklist()
 371
 372     MODE = "--tests-file " + TEMPEST_LIST
 373     if args.serial:
 374         MODE += " --concur 1"
 375
 376     run_tempest(MODE)
 377
 378
 379 if __name__ == '__main__':
 380     main()