"""Rally testcases implementation."""
from __future__ import division
+from __future__ import print_function
+import fileinput
import json
import logging
import os
import re
+import shutil
import subprocess
import time
import pkg_resources
import prettytable
+from ruamel.yaml import YAML
+from six.moves import configparser
from xtesting.core import testcase
-from xtesting.energy import energy
import yaml
from functest.core import singlevm
-from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
from functest.utils import env
class RallyBase(singlevm.VmReady2):
"""Base class form Rally testcases implementation."""
- # pylint: disable=too-many-instance-attributes
+ # pylint: disable=too-many-instance-attributes, too-many-public-methods
TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
'keystone', 'neutron', 'nova', 'quotas']
+ RALLY_CONF_PATH = "/etc/rally/rally.conf"
+ RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
+ 'functest', 'ci/rally_aarch64_patch.conf')
RALLY_DIR = pkg_resources.resource_filename(
'functest', 'opnfv_tests/openstack/rally')
RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
TENANTS_AMOUNT = 3
ITERATIONS_AMOUNT = 10
CONCURRENCY = 4
- RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally')
- BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt")
- TEMP_DIR = os.path.join(RALLY_DIR, "var")
+ BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.yaml")
+ TASK_DIR = os.path.join(getattr(config.CONF, 'dir_rally_data'), 'task')
+ TEMP_DIR = os.path.join(TASK_DIR, 'var')
visibility = 'public'
shared_network = True
role_name, user=self.project.user.id,
project=self.project.project.id,
domain=self.project.domain.id)
+ self.results_dir = os.path.join(
+ getattr(config.CONF, 'dir_results'), self.case_name)
+ self.task_file = ''
self.creators = []
self.summary = []
self.scenario_dir = ''
self.smoke = None
- self.test_name = None
self.start_time = None
self.result = None
self.details = None
self.compute_cnt = 0
self.flavor_alt = None
self.tests = []
- self.task_file = ''
self.run_cmd = ''
+ self.network_extensions = []
+ self.services = []
- def _build_task_args(self, test_file_name):
+ def build_task_args(self, test_name):
"""Build arguments for the Rally task."""
- task_args = {'service_list': [test_file_name]}
+ task_args = {'service_list': [test_name]}
task_args['image_name'] = str(self.image.name)
task_args['flavor_name'] = str(self.flavor.name)
task_args['flavor_alt_name'] = str(self.flavor_alt.name)
if not os.path.exists(self.TEMP_DIR):
os.makedirs(self.TEMP_DIR)
- self._apply_blacklist(scenario_file_name, test_file_name)
+ self.apply_blacklist(scenario_file_name, test_file_name)
return test_file_name
+ @staticmethod
+ def get_verifier_deployment_id():
+ """
+ Returns deployment id for active Rally deployment
+ """
+ cmd = ("rally deployment list | awk '/" +
+ getattr(config.CONF, 'rally_deployment_name') +
+ "/ {print $2}'")
+ proc = subprocess.Popen(cmd, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ deployment_uuid = proc.stdout.readline().rstrip()
+ return deployment_uuid
+
+ @staticmethod
+ def create_rally_deployment(environ=None):
+ """Create new rally deployment"""
+ # set the architecture to default
+ pod_arch = env.get("POD_ARCH")
+ arch_filter = ['aarch64']
+
+ if pod_arch and pod_arch in arch_filter:
+ LOGGER.info("Apply aarch64 specific to rally config...")
+ with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
+ rally_patch_conf = pfile.read()
+
+ for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
+ print(line, end=' ')
+ if "cirros|testvm" in line:
+ print(rally_patch_conf)
+
+ LOGGER.info("Creating Rally environment...")
+ try:
+ cmd = ['rally', 'deployment', 'destroy',
+ '--deployment',
+ str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ except subprocess.CalledProcessError:
+ pass
+
+ cmd = ['rally', 'deployment', 'create', '--fromenv',
+ '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd, env=environ)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ cmd = ['rally', 'deployment', 'check']
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ return RallyBase.get_verifier_deployment_id()
+
+ @staticmethod
+ def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
+ """Set keystone_default_role in rally.conf"""
+ if env.get("NEW_USER_ROLE").lower() != "member":
+ rconfig = configparser.RawConfigParser()
+ rconfig.read(rally_conf)
+ if not rconfig.has_section('openstack'):
+ rconfig.add_section('openstack')
+ rconfig.set(
+ 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE"))
+ with open(rally_conf, 'wb') as config_file:
+ rconfig.write(config_file)
+
+ @staticmethod
+ def clean_rally_conf(rally_conf='/etc/rally/rally.conf'):
+ """Clean Rally config"""
+ if env.get("NEW_USER_ROLE").lower() != "member":
+ rconfig = configparser.RawConfigParser()
+ rconfig.read(rally_conf)
+ if rconfig.has_option('openstack', 'keystone_default_role'):
+ rconfig.remove_option('openstack', 'keystone_default_role')
+ with open(rally_conf, 'wb') as config_file:
+ rconfig.write(config_file)
+
@staticmethod
def get_task_id(cmd_raw):
"""
"""Determine if migration is supported."""
if self.compute_cnt > 1:
return True
+ return False
+ def _network_trunk_supported(self):
+ """Determine if network trunk service is available"""
+ if 'trunk' in self.network_extensions:
+ return True
return False
@staticmethod
if not self._migration_supported():
func_list.append("no_migration")
+ if not self._network_trunk_supported():
+ func_list.append("no_net_trunk_service")
if 'functionality' in black_list_yaml.keys():
for item in black_list_yaml['functionality']:
return black_tests
- def _apply_blacklist(self, case_file_name, result_file_name):
+ def apply_blacklist(self, case_file_name, result_file_name):
"""Apply blacklist."""
LOGGER.debug("Applying blacklist...")
cases_file = open(case_file_name, 'r')
def _save_results(self, test_name, task_id):
""" Generate and save task execution results"""
# check for result directory and create it otherwise
- if not os.path.exists(self.RESULTS_DIR):
+ if not os.path.exists(self.results_dir):
LOGGER.debug('%s does not exist, we create it.',
- self.RESULTS_DIR)
- os.makedirs(self.RESULTS_DIR)
+ self.results_dir)
+ os.makedirs(self.results_dir)
# put detailed result to log
cmd = (["rally", "task", "detailed", "--uuid", task_id])
LOGGER.info("%s\n%s", " ".join(cmd), output)
# save report as JSON
- report_json_name = 'opnfv-{}.json'.format(test_name)
- report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name)
+ report_json_name = '{}.json'.format(test_name)
+ report_json_dir = os.path.join(self.results_dir, report_json_name)
cmd = (["rally", "task", "report", "--json", "--uuid", task_id,
"--out", report_json_dir])
LOGGER.debug('running command: %s', cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info("%s\n%s", " ".join(cmd), output)
- # save report as HTML
- report_html_name = 'opnfv-{}.html'.format(test_name)
- report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name)
- cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
- "--out", report_html_dir])
- LOGGER.debug('running command: %s', cmd)
- output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
-
json_results = open(report_json_dir).read()
self._append_summary(json_results, test_name)
nb_tests = 0
nb_success = 0
overall_duration = 0.0
+ success = []
+ failures = []
rally_report = json.loads(json_raw)
for task in rally_report.get('tasks'):
for subtask in task.get('subtasks'):
+ has_errors = False
for workload in subtask.get('workloads'):
if workload.get('full_duration'):
overall_duration += workload.get('full_duration')
for result in workload.get('data'):
if not result.get('error'):
nb_success += 1
+ else:
+ has_errors = True
+
+ if has_errors:
+ failures.append(subtask['title'])
+ else:
+ success.append(subtask['title'])
scenario_summary = {'test_name': test_name,
'overall_duration': overall_duration,
'nb_tests': nb_tests,
'nb_success': nb_success,
+ 'success': success,
+ 'failures': failures,
'task_status': self.task_succeed(json_raw)}
self.summary.append(scenario_summary)
- def prepare_run(self):
+ def prepare_run(self, **kwargs):
"""Prepare resources needed by test scenarios."""
assert self.cloud
- LOGGER.debug('Validating the test name...')
- if self.test_name == 'all':
- self.tests = self.TESTS
- elif self.test_name in self.TESTS:
- self.tests = [self.test_name]
- else:
- raise Exception("Test name '%s' is invalid" % self.test_name)
+ LOGGER.debug('Validating run tests...')
+ for test in kwargs.get('tests', self.TESTS):
+ if test in self.TESTS:
+ self.tests.append(test)
+ else:
+ raise Exception("Test name '%s' is invalid" % test)
- self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml')
- if not os.path.exists(self.task_file):
- LOGGER.error("Task file '%s' does not exist.", self.task_file)
- raise Exception("Task file '{}' does not exist.".
- format(self.task_file))
+ if not os.path.exists(self.TASK_DIR):
+ os.makedirs(self.TASK_DIR)
+ task = os.path.join(self.RALLY_DIR, 'task.yaml')
+ if not os.path.exists(task):
+ LOGGER.error("Task file '%s' does not exist.", task)
+ raise Exception("Task file '{}' does not exist.".
+ format(task))
+ self.task_file = os.path.join(self.TASK_DIR, 'task.yaml')
+ shutil.copyfile(task, self.task_file)
+
+ task_macro = os.path.join(self.RALLY_DIR, 'macro')
+ if not os.path.exists(task_macro):
+ LOGGER.error("Task macro dir '%s' does not exist.", task_macro)
+ raise Exception("Task macro dir '{}' does not exist.".
+ format(task_macro))
+ macro_dir = os.path.join(self.TASK_DIR, 'macro')
+ if os.path.exists(macro_dir):
+ shutil.rmtree(macro_dir)
+ shutil.copytree(task_macro, macro_dir)
+
+ self.update_keystone_default_role()
self.compute_cnt = len(self.cloud.list_hypervisors())
+ self.network_extensions = self.cloud.get_network_extensions()
self.flavor_alt = self.create_flavor_alt()
+ self.services = [service.name for service in
+ self.cloud.list_services()]
+
LOGGER.debug("flavor: %s", self.flavor_alt)
def prepare_task(self, test_name):
return False
self.run_cmd = (["rally", "task", "start", "--abort-on-sla-failure",
"--task", self.task_file, "--task-args",
- str(self._build_task_args(test_name))])
+ str(self.build_task_args(test_name))])
return True
- def run_tests(self):
+ def run_tests(self, **kwargs):
"""Execute tests."""
+ optional = kwargs.get('optional', [])
for test in self.tests:
- if self.prepare_task(test):
- self.run_task(test)
+ if test in self.services or test not in optional:
+ if self.prepare_task(test):
+ self.run_task(test)
def _generate_report(self):
"""Generate test execution summary report."""
except ZeroDivisionError:
success_avg = 0
success_str = str("{:0.2f}".format(success_avg)) + '%'
- duration_str = time.strftime("%M:%S",
+ duration_str = time.strftime("%H:%M:%S",
time.gmtime(item['overall_duration']))
res_table.add_row([item['test_name'], duration_str,
item['nb_tests'], success_str])
payload.append({'module': item['test_name'],
'details': {'duration': item['overall_duration'],
'nb tests': item['nb_tests'],
- 'success': success_str}})
+ 'success rate': success_str,
+ 'success': item['success'],
+ 'failures': item['failures']}})
total_duration_str = time.strftime("%H:%M:%S",
time.gmtime(total_duration))
'nb success': success_rate}})
self.details = payload
+ @staticmethod
+ def export_task(file_name, export_type="html"):
+ """Export all task results (e.g. html or xunit report)
+
+ Raises:
+ subprocess.CalledProcessError: if Rally doesn't return 0
+
+ Returns:
+ None
+ """
+ cmd = ["rally", "task", "export", "--type", export_type,
+ "--deployment",
+ str(getattr(config.CONF, 'rally_deployment_name')),
+ "--to", file_name]
+ LOGGER.debug('running command: %s', cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ @staticmethod
+ def verify_report(file_name, uuid, export_type="html"):
+ """Generate the verifier report (e.g. html or xunit report)
+
+ Raises:
+ subprocess.CalledProcessError: if Rally doesn't return 0
+
+ Returns:
+ None
+ """
+ cmd = ["rally", "verify", "report", "--type", export_type,
+ "--uuid", uuid, "--to", file_name]
+ LOGGER.debug('running command: %s', cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
def clean(self):
"""Cleanup of OpenStack resources. Should be called on completion."""
+ self.clean_rally_conf()
if self.flavor_alt:
self.orig_cloud.delete_flavor(self.flavor_alt.id)
super(RallyBase, self).clean()
return super(RallyBase, self).is_successful()
- @energy.enable_recording
def run(self, **kwargs):
"""Run testcase."""
self.start_time = time.time()
OS_PROJECT_NAME=self.project.project.name,
OS_PROJECT_ID=self.project.project.id,
OS_PASSWORD=self.project.password)
- conf_utils.create_rally_deployment(environ=environ)
- self.prepare_run()
- self.run_tests()
+ try:
+ del environ['OS_TENANT_NAME']
+ del environ['OS_TENANT_ID']
+ except Exception: # pylint: disable=broad-except
+ pass
+ self.create_rally_deployment(environ=environ)
+ self.prepare_run(**kwargs)
+ self.run_tests(**kwargs)
self._generate_report()
+ self.export_task(
+ "{}/{}.html".format(self.results_dir, self.case_name))
+ self.export_task(
+ "{}/{}.xml".format(self.results_dir, self.case_name),
+ export_type="junit-xml")
res = testcase.TestCase.EX_OK
except Exception as exc: # pylint: disable=broad-except
LOGGER.error('Error with run: %s', exc)
if "case_name" not in kwargs:
kwargs["case_name"] = "rally_sanity"
super(RallySanity, self).__init__(**kwargs)
- self.test_name = 'all'
self.smoke = True
self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity')
if "case_name" not in kwargs:
kwargs["case_name"] = "rally_full"
super(RallyFull, self).__init__(**kwargs)
- self.test_name = 'all'
self.smoke = False
self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full')
+
+
+class RallyJobs(RallyBase):
+ """Rally OpenStack CI testcase implementation."""
+
+ TESTS = ["neutron"]
+
+ def __init__(self, **kwargs):
+ """Initialize RallyJobs object."""
+ if "case_name" not in kwargs:
+ kwargs["case_name"] = "rally_jobs"
+ super(RallyJobs, self).__init__(**kwargs)
+ self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml')
+ self.task_yaml = None
+
+ def prepare_run(self, **kwargs):
+ """Create resources needed by test scenarios."""
+ super(RallyJobs, self).prepare_run(**kwargs)
+ with open(os.path.join(self.RALLY_DIR,
+ 'rally_jobs.yaml'), 'r') as task_file:
+ self.task_yaml = yaml.safe_load(task_file)
+
+ for task in self.task_yaml:
+ if task not in self.tests:
+ raise Exception("Test '%s' not in '%s'" %
+ (task, self.tests))
+
+ def apply_blacklist(self, case_file_name, result_file_name):
+ # pylint: disable=too-many-branches
+ """Apply blacklist."""
+ LOGGER.debug("Applying blacklist...")
+ black_tests = list(set(self.excl_func() +
+ self.excl_scenario()))
+ if black_tests:
+ LOGGER.debug("Blacklisted tests: %s", str(black_tests))
+
+ template = YAML(typ='jinja2')
+ with open(case_file_name, 'r') as fname:
+ cases = template.load(fname)
+ if cases.get("version", 1) == 1:
+ # scenarios in dictionary
+ for name in cases.keys():
+ if self.in_iterable_re(name, black_tests):
+ cases.pop(name)
+ else:
+ # workloads in subtasks
+ for sind, subtask in enumerate(cases.get('subtasks', [])):
+ idx = []
+ for wind, workload in enumerate(subtask.get('workloads', [])):
+ scenario = workload.get('scenario', {})
+ for name in scenario.keys():
+ if self.in_iterable_re(name, black_tests):
+ idx.append(wind)
+ break
+ for wind in reversed(idx):
+ cases['subtasks'][sind]['workloads'].pop(wind)
+ # scenarios in subtasks
+ idx = []
+ for sind, subtask in enumerate(cases.get('subtasks', [])):
+ scenario = subtask.get('scenario', {})
+ for name in scenario.keys():
+ if self.in_iterable_re(name, black_tests):
+ idx.append(sind)
+ break
+ for sind in reversed(idx):
+ cases['subtasks'].pop(sind)
+
+ with open(result_file_name, 'w') as fname:
+ template.dump(cases, fname)
+
+ def build_task_args(self, test_name):
+ """Build arguments for the Rally task."""
+ task_args = {}
+ if self.ext_net:
+ task_args['floating_network'] = str(self.ext_net.name)
+ else:
+ task_args['floating_network'] = ''
+ return task_args
+
+ @staticmethod
+ def _remove_plugins_extra():
+ inst_dir = getattr(config.CONF, 'dir_rally_inst')
+ try:
+ shutil.rmtree(os.path.join(inst_dir, 'plugins'))
+ shutil.rmtree(os.path.join(inst_dir, 'extra'))
+ except Exception: # pylint: disable=broad-except
+ pass
+
+ def prepare_task(self, test_name):
+ """Prepare resources for test run."""
+ self._remove_plugins_extra()
+ jobs_dir = os.path.join(
+ getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs')
+ inst_dir = getattr(config.CONF, 'dir_rally_inst')
+ shutil.copytree(os.path.join(jobs_dir, 'plugins'),
+ os.path.join(inst_dir, 'plugins'))
+ shutil.copytree(os.path.join(jobs_dir, 'extra'),
+ os.path.join(inst_dir, 'extra'))
+
+ task_name = self.task_yaml.get(test_name).get("task")
+ task = os.path.join(jobs_dir, task_name)
+ if not os.path.exists(task):
+ raise Exception("The scenario '%s' does not exist." % task)
+ LOGGER.debug('Scenario fetched from : %s', task)
+
+ if not os.path.exists(self.TEMP_DIR):
+ os.makedirs(self.TEMP_DIR)
+ task_file_name = os.path.join(self.TEMP_DIR, task_name)
+ self.apply_blacklist(task, task_file_name)
+ self.run_cmd = (["rally", "task", "start", "--task", task_file_name,
+ "--task-args", str(self.build_task_args(test_name))])
+ return True
+
+ def clean(self):
+ self._remove_plugins_extra()
+ super(RallyJobs, self).clean()