# 0.3 (19/10/2015) remove Tempest from run_rally
# and push result into test DB
#
-import argparse
-import iniparse
+""" tests configuration """
+
import json
import os
import re
import subprocess
import time
-import yaml
-from novaclient import client as novaclient
-from glanceclient import client as glanceclient
-from keystoneclient.v2_0 import client as keystoneclient
-from neutronclient.v2_0 import client as neutronclient
-from cinderclient import client as cinderclient
+import argparse
+import iniparse
+import yaml
import functest.utils.functest_logger as ft_logger
-import functest.utils.functest_utils as functest_utils
-import functest.utils.openstack_utils as openstack_utils
+import functest.utils.functest_utils as ft_utils
+import functest.utils.openstack_utils as os_utils
-""" tests configuration """
tests = ['authenticate', 'glance', 'cinder', 'heat', 'keystone',
'neutron', 'nova', 'quotas', 'requests', 'vm', 'all']
parser = argparse.ArgumentParser()
args = parser.parse_args()
-client_dict = {}
network_dict = {}
if args.verbose:
""" logging configuration """
logger = ft_logger.Logger("run_rally").getLogger()
-REPO_PATH = os.environ['repos_dir'] + '/functest/'
-if not os.path.exists(REPO_PATH):
- logger.error("Functest repository directory not found '%s'" % REPO_PATH)
- exit(-1)
-
-
-with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
- functest_yaml = yaml.safe_load(f)
-f.close()
HOME = os.environ['HOME'] + "/"
-RALLY_DIR = REPO_PATH + functest_yaml.get("general").get(
- "directories").get("dir_rally")
+RALLY_DIR = ft_utils.FUNCTEST_REPO + '/' + \
+ ft_utils.get_functest_config('general.directories.dir_rally')
+SANITY_MODE_DIR = RALLY_DIR + "scenario/sanity"
+FULL_MODE_DIR = RALLY_DIR + "scenario/full"
TEMPLATE_DIR = RALLY_DIR + "scenario/templates"
SUPPORT_DIR = RALLY_DIR + "scenario/support"
+TEMP_DIR = RALLY_DIR + "var"
+BLACKLIST_FILE = RALLY_DIR + "blacklist.txt"
FLAVOR_NAME = "m1.tiny"
USERS_AMOUNT = 2
ITERATIONS_AMOUNT = 10
CONCURRENCY = 4
-RESULTS_DIR = functest_yaml.get("general").get("directories").get(
- "dir_rally_res")
-TEMPEST_CONF_FILE = functest_yaml.get("general").get("directories").get(
- "dir_results") + '/tempest/tempest.conf'
-TEST_DB = functest_yaml.get("results").get("test_db_url")
-
-PRIVATE_NET_NAME = functest_yaml.get("rally").get("network_name")
-PRIVATE_SUBNET_NAME = functest_yaml.get("rally").get("subnet_name")
-PRIVATE_SUBNET_CIDR = functest_yaml.get("rally").get("subnet_cidr")
-ROUTER_NAME = functest_yaml.get("rally").get("router_name")
-
-GLANCE_IMAGE_NAME = functest_yaml.get("general").get("openstack").get(
- "image_name")
-GLANCE_IMAGE_FILENAME = functest_yaml.get("general").get("openstack").get(
- "image_file_name")
-GLANCE_IMAGE_FORMAT = functest_yaml.get("general").get("openstack").get(
- "image_disk_format")
-GLANCE_IMAGE_PATH = functest_yaml.get("general").get("directories").get(
- "dir_functest_data") + "/" + GLANCE_IMAGE_FILENAME
+RESULTS_DIR = \
+ ft_utils.get_functest_config('general.directories.dir_rally_res')
+TEMPEST_CONF_FILE = \
+ ft_utils.get_functest_config('general.directories.dir_results') + \
+ '/tempest/tempest.conf'
+TEST_DB = ft_utils.get_functest_config('results.test_db_url')
+
+PRIVATE_NET_NAME = ft_utils.get_functest_config('rally.network_name')
+PRIVATE_SUBNET_NAME = ft_utils.get_functest_config('rally.subnet_name')
+PRIVATE_SUBNET_CIDR = ft_utils.get_functest_config('rally.subnet_cidr')
+ROUTER_NAME = ft_utils.get_functest_config('rally.router_name')
+
+GLANCE_IMAGE_NAME = \
+ ft_utils.get_functest_config('general.openstack.image_name')
+GLANCE_IMAGE_FILENAME = \
+ ft_utils.get_functest_config('general.openstack.image_file_name')
+GLANCE_IMAGE_FORMAT = \
+ ft_utils.get_functest_config('general.openstack.image_disk_format')
+GLANCE_IMAGE_PATH = \
+ ft_utils.get_functest_config('general.directories.dir_functest_data') + \
+ "/" + GLANCE_IMAGE_FILENAME
CINDER_VOLUME_TYPE_NAME = "volume_test"
SUMMARY = []
+neutron_client = None
def get_task_id(cmd_raw):
task_args['image_name'] = GLANCE_IMAGE_NAME
task_args['flavor_name'] = FLAVOR_NAME
task_args['glance_image_location'] = GLANCE_IMAGE_PATH
+ task_args['glance_image_format'] = GLANCE_IMAGE_FORMAT
task_args['tmpl_dir'] = TEMPLATE_DIR
task_args['sup_dir'] = SUPPORT_DIR
task_args['users_amount'] = USERS_AMOUNT
task_args['tenants_amount'] = TENANTS_AMOUNT
+ task_args['use_existing_users'] = False
task_args['iterations'] = ITERATIONS_AMOUNT
task_args['concurrency'] = CONCURRENCY
if args.sanity:
- task_args['full_mode'] = False
task_args['smoke'] = True
else:
- task_args['full_mode'] = True
task_args['smoke'] = args.smoke
- ext_net = openstack_utils.get_external_net(client_dict['neutron'])
+ ext_net = os_utils.get_external_net(neutron_client)
if ext_net:
task_args['floating_network'] = str(ext_net)
else:
net_id = network_dict['net_id']
task_args['netid'] = str(net_id)
- task_args['live_migration'] = live_migration_supported()
+
+ auth_url = os.getenv('OS_AUTH_URL')
+ if auth_url is not None:
+ task_args['request_url'] = auth_url.rsplit(":", 1)[0]
+ else:
+ task_args['request_url'] = ''
return task_args
'success': success_avg}
SUMMARY.append(scenario_summary)
- logger.info("\n" + result)
+ logger.debug("\n" + result)
return result
return result
+def excl_scenario():
+ black_tests = []
+
+ try:
+ with open(BLACKLIST_FILE, 'r') as black_list_file:
+ black_list_yaml = yaml.safe_load(black_list_file)
+
+ installer_type = os.getenv('INSTALLER_TYPE')
+ deploy_scenario = os.getenv('DEPLOY_SCENARIO')
+ if (bool(installer_type) * bool(deploy_scenario)):
+ if 'scenario' in black_list_yaml.keys():
+ for item in black_list_yaml['scenario']:
+ scenarios = item['scenarios']
+ installers = item['installers']
+ if (deploy_scenario in scenarios and
+ installer_type in installers):
+ tests = item['tests']
+ black_tests.extend(tests)
+ except:
+ logger.debug("Scenario exclusion not applied.")
+
+ return black_tests
+
+
+def excl_func():
+ black_tests = []
+ func_list = []
+
+ try:
+ with open(BLACKLIST_FILE, 'r') as black_list_file:
+ black_list_yaml = yaml.safe_load(black_list_file)
+
+ if not live_migration_supported():
+ func_list.append("no_live_migration")
+
+ if 'functionality' in black_list_yaml.keys():
+ for item in black_list_yaml['functionality']:
+ functions = item['functions']
+ for func in func_list:
+ if func in functions:
+ tests = item['tests']
+ black_tests.extend(tests)
+ except:
+ logger.debug("Functionality exclusion not applied.")
+
+ return black_tests
+
+
+def apply_blacklist(case_file_name, result_file_name):
+ logger.debug("Applying blacklist...")
+ cases_file = open(case_file_name, 'r')
+ result_file = open(result_file_name, 'w')
+
+ black_tests = list(set(excl_func() + excl_scenario()))
+
+ include = True
+ for cases_line in cases_file:
+ if include:
+ for black_tests_line in black_tests:
+ if re.search(black_tests_line, cases_line.strip().rstrip(':')):
+ include = False
+ break
+ else:
+ result_file.write(str(cases_line))
+ else:
+ if cases_line.isspace():
+ include = True
+
+ cases_file.close()
+ result_file.close()
+
+
+def prepare_test_list(test_name):
+ scenario_file_name = '{}opnfv-{}.yaml'.format(RALLY_DIR + "scenario/",
+ test_name)
+ if not os.path.exists(scenario_file_name):
+ if args.sanity:
+ scenario_file_name = '{}opnfv-{}.yaml'.format(SANITY_MODE_DIR +
+ "/", test_name)
+ else:
+ scenario_file_name = '{}opnfv-{}.yaml'.format(FULL_MODE_DIR +
+ "/", test_name)
+ if not os.path.exists(scenario_file_name):
+ logger.info("The scenario '%s' does not exist."
+ % scenario_file_name)
+ exit(-1)
+
+ logger.debug('Scenario fetched from : {}'.format(scenario_file_name))
+ test_file_name = '{}opnfv-{}.yaml'.format(TEMP_DIR + "/", test_name)
+
+ if not os.path.exists(TEMP_DIR):
+ os.makedirs(TEMP_DIR)
+
+ apply_blacklist(scenario_file_name, test_file_name)
+ return test_file_name
+
+
+def file_is_empty(file_name):
+ try:
+ if os.stat(file_name).st_size > 0:
+ return False
+ except:
+ pass
+
+ return True
+
+
def run_task(test_name):
#
# the "main" function of the script who launch rally for a task
global SUMMARY
logger.info('Starting test scenario "{}" ...'.format(test_name))
start_time = time.time()
- stop_time = start_time
task_file = '{}task.yaml'.format(RALLY_DIR)
if not os.path.exists(task_file):
logger.error("Task file '%s' does not exist." % task_file)
exit(-1)
- test_file_name = '{}opnfv-{}.yaml'.format(RALLY_DIR + "scenario/",
- test_name)
- if not os.path.exists(test_file_name):
- logger.error("The scenario '%s' does not exist." % test_file_name)
- exit(-1)
-
- logger.debug('Scenario fetched from : {}'.format(test_file_name))
+ file_name = prepare_test_list(test_name)
+ if file_is_empty(file_name):
+ logger.info('No tests for scenario "{}"'.format(test_name))
+ return
cmd_line = ("rally task start --abort-on-sla-failure " +
"--task {} ".format(task_file) +
# check for result directory and create it otherwise
if not os.path.exists(RESULTS_DIR):
- logger.debug('%s does not exist, we create it.'.format(RESULTS_DIR))
+ logger.debug('{} does not exist, we create it.'.format(RESULTS_DIR))
os.makedirs(RESULTS_DIR)
# write html report file
json_data = json.load(json_file)
""" parse JSON operation result """
- status = "failed"
+ status = "FAIL"
if task_succeed(json_results):
logger.info('Test scenario: "{}" OK.'.format(test_name) + "\n")
- status = "passed"
+ status = "PASS"
else:
logger.info('Test scenario: "{}" Failed.'.format(test_name) + "\n")
if args.report:
stop_time = time.time()
logger.debug("Push Rally detailed results into DB")
- functest_utils.push_results_to_db("functest",
- "Rally_details",
- logger,
- start_time,
- stop_time,
- status,
- json_data)
+ ft_utils.push_results_to_db("functest",
+ "Rally_details",
+ start_time,
+ stop_time,
+ status,
+ json_data)
def main():
global SUMMARY
global network_dict
+ global neutron_client
+
+ nova_client = os_utils.get_nova_client()
+ neutron_client = os_utils.get_neutron_client()
+ cinder_client = os_utils.get_cinder_client()
+
start_time = time.time()
- stop_time = start_time
# configure script
if not (args.test_name in tests):
exit(-1)
SUMMARY = []
- creds_nova = openstack_utils.get_credentials("nova")
- nova_client = novaclient.Client('2', **creds_nova)
- creds_neutron = openstack_utils.get_credentials("neutron")
- neutron_client = neutronclient.Client(**creds_neutron)
- creds_keystone = openstack_utils.get_credentials("keystone")
- keystone_client = keystoneclient.Client(**creds_keystone)
- glance_endpoint = keystone_client.service_catalog.url_for(
- service_type='image', endpoint_type='publicURL')
- glance_client = glanceclient.Client(1, glance_endpoint,
- token=keystone_client.auth_token)
- creds_cinder = openstack_utils.get_credentials("cinder")
- cinder_client = cinderclient.Client('2', creds_cinder['username'],
- creds_cinder['api_key'],
- creds_cinder['project_id'],
- creds_cinder['auth_url'],
- service_type="volume")
-
- client_dict['neutron'] = neutron_client
-
- volume_types = openstack_utils.list_volume_types(cinder_client,
- private=False)
+
+ volume_types = os_utils.list_volume_types(cinder_client,
+ private=False)
if not volume_types:
- volume_type = openstack_utils.create_volume_type(
+ volume_type = os_utils.create_volume_type(
cinder_client, CINDER_VOLUME_TYPE_NAME)
if not volume_type:
logger.error("Failed to create volume type...")
else:
logger.debug("Using existing volume type(s)...")
- image_id = openstack_utils.get_image_id(glance_client, GLANCE_IMAGE_NAME)
- image_exists = False
-
- if image_id == '':
- logger.debug("Creating image '%s' from '%s'..." % (GLANCE_IMAGE_NAME,
- GLANCE_IMAGE_PATH))
- image_id = openstack_utils.create_glance_image(glance_client,
- GLANCE_IMAGE_NAME,
- GLANCE_IMAGE_PATH)
- if not image_id:
- logger.error("Failed to create the Glance image...")
- exit(-1)
- else:
- logger.debug("Image '%s' with ID '%s' created succesfully ."
- % (GLANCE_IMAGE_NAME, image_id))
- else:
- logger.debug("Using existing image '%s' with ID '%s'..."
- % (GLANCE_IMAGE_NAME, image_id))
- image_exists = True
+ image_exists, image_id = os_utils.get_or_create_image(GLANCE_IMAGE_NAME,
+ GLANCE_IMAGE_PATH,
+ GLANCE_IMAGE_FORMAT)
+ if not image_id:
+ exit(-1)
logger.debug("Creating network '%s'..." % PRIVATE_NET_NAME)
- network_dict = openstack_utils.create_network_full(logger,
- client_dict['neutron'],
- PRIVATE_NET_NAME,
+ network_dict = os_utils.create_shared_network_full(PRIVATE_NET_NAME,
PRIVATE_SUBNET_NAME,
ROUTER_NAME,
PRIVATE_SUBNET_CIDR)
if not network_dict:
- logger.error("Failed to create network...")
- exit(-1)
- else:
- if not openstack_utils.update_neutron_net(client_dict['neutron'],
- network_dict['net_id'],
- shared=True):
- logger.error("Failed to update network...")
- exit(-1)
- else:
- logger.debug("Network '%s' available..." % PRIVATE_NET_NAME)
+ exit(1)
if args.test_name == "all":
for test_name in tests:
total_duration_str = time.strftime("%H:%M:%S", time.gmtime(total_duration))
total_duration_str2 = "{0:<10}".format(total_duration_str)
total_nb_tests_str = "{0:<13}".format(total_nb_tests)
- total_success = "{:0.2f}".format(total_success / len(SUMMARY))
- total_success_str = "{0:<10}".format(str(total_success) + '%')
+
+ if len(SUMMARY):
+ success_rate = total_success / len(SUMMARY)
+ else:
+ success_rate = 100
+ success_rate = "{:0.2f}".format(success_rate)
+ success_rate_str = "{0:<10}".format(str(success_rate) + '%')
report += "+===================+============+===============+===========+"
report += "\n"
report += ("| TOTAL: | " + total_duration_str2 + " | " +
- total_nb_tests_str + " | " + total_success_str + "|\n")
+ total_nb_tests_str + " | " + success_rate_str + "|\n")
report += "+===================+============+===============+===========+"
report += "\n"
logger.info("\n" + report)
payload.append({'summary': {'duration': total_duration,
'nb tests': total_nb_tests,
- 'nb success': total_success}})
+ 'nb success': success_rate}})
- # Generate json results for DB
- # json_results = {"timestart": time_start, "duration": total_duration,
- # "tests": int(total_nb_tests),
- # "success": int(total_success)}
- # logger.info("Results: "+str(json_results))
+ if args.sanity:
+ case_name = "rally_sanity"
+ else:
+ case_name = "rally_full"
# Evaluation of the success criteria
- status = "failed"
- # for Rally we decided that the overall success rate must be above 90%
- if total_success >= 90:
- status = "passed"
+ status = ft_utils.check_success_rate(case_name, success_rate)
+
+ exit_code = -1
+ if status == "PASS":
+ exit_code = 0
if args.report:
logger.debug("Pushing Rally summary into DB...")
- functest_utils.push_results_to_db("functest",
- "Rally",
- logger,
- start_time,
- stop_time,
- status,
- payload)
+ ft_utils.push_results_to_db("functest",
+ case_name,
+ start_time,
+ stop_time,
+ status,
+ payload)
if args.noclean:
- exit(0)
+ exit(exit_code)
if not image_exists:
logger.debug("Deleting image '%s' with ID '%s'..."
% (GLANCE_IMAGE_NAME, image_id))
- if not openstack_utils.delete_glance_image(nova_client, image_id):
+ if not os_utils.delete_glance_image(nova_client, image_id):
logger.error("Error deleting the glance image")
if not volume_types:
logger.debug("Deleting volume type '%s'..."
% CINDER_VOLUME_TYPE_NAME)
- if not openstack_utils.delete_volume_type(cinder_client, volume_type):
+ if not os_utils.delete_volume_type(cinder_client, volume_type):
logger.error("Error in deleting volume type...")
+ exit(exit_code)
+
if __name__ == '__main__':
main()
Code Review / functest-xtesting.git / blobdiff