"""Rally testcases implementation."""
from __future__ import division
+from __future__ import print_function
+import fileinput
import json
import logging
import os
import yaml
from functest.core import singlevm
-from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
from functest.utils import env
class RallyBase(singlevm.VmReady2):
"""Base class form Rally testcases implementation."""
- # pylint: disable=too-many-instance-attributes
+ # pylint: disable=too-many-instance-attributes, too-many-public-methods
TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat',
'keystone', 'neutron', 'nova', 'quotas']
+ RALLY_CONF_PATH = "/etc/rally/rally.conf"
+ RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
+ 'functest', 'ci/rally_aarch64_patch.conf')
RALLY_DIR = pkg_resources.resource_filename(
'functest', 'opnfv_tests/openstack/rally')
RALLY_SCENARIO_DIR = pkg_resources.resource_filename(
self.apply_blacklist(scenario_file_name, test_file_name)
return test_file_name
+ @staticmethod
+ def get_verifier_deployment_id():
+ """
+ Returns deployment id for active Rally deployment
+ """
+ cmd = ("rally deployment list | awk '/" +
+ getattr(config.CONF, 'rally_deployment_name') +
+ "/ {print $2}'")
+ proc = subprocess.Popen(cmd, shell=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ deployment_uuid = proc.stdout.readline().rstrip()
+ return deployment_uuid
+
+ @staticmethod
+ def create_rally_deployment(environ=None):
+ """Create new rally deployment"""
+ # set the architecture to default
+ pod_arch = env.get("POD_ARCH")
+ arch_filter = ['aarch64']
+
+ if pod_arch and pod_arch in arch_filter:
+ LOGGER.info("Apply aarch64 specific to rally config...")
+ with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile:
+ rally_patch_conf = pfile.read()
+
+ for line in fileinput.input(RallyBase.RALLY_CONF_PATH):
+ print(line, end=' ')
+ if "cirros|testvm" in line:
+ print(rally_patch_conf)
+
+ LOGGER.info("Creating Rally environment...")
+ try:
+ cmd = ['rally', 'deployment', 'destroy',
+ '--deployment',
+ str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ except subprocess.CalledProcessError:
+ pass
+
+ cmd = ['rally', 'deployment', 'create', '--fromenv',
+ '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
+ output = subprocess.check_output(cmd, env=environ)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ cmd = ['rally', 'deployment', 'check']
+ output = subprocess.check_output(cmd)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+ return RallyBase.get_verifier_deployment_id()
+
@staticmethod
def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'):
"""Set keystone_default_role in rally.conf"""
'nb success': success_rate}})
self.details = payload
- def generate_html_report(self):
- """Save all task reports as single HTML
+ @staticmethod
+ def export_task(file_name, export_type="html"):
+ """Export all task results (e.g. html or xunit report)
Raises:
subprocess.CalledProcessError: if Rally doesn't return 0
Returns:
None
"""
- cmd = ["rally", "task", "report", "--deployment",
+ cmd = ["rally", "task", "export", "--type", export_type,
+ "--deployment",
str(getattr(config.CONF, 'rally_deployment_name')),
- "--out", "{}/{}.html".format(self.results_dir, self.case_name)]
+ "--to", file_name]
+ LOGGER.debug('running command: %s', cmd)
+ output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+ LOGGER.info("%s\n%s", " ".join(cmd), output)
+
+ @staticmethod
+ def verify_report(file_name, uuid, export_type="html"):
+ """Generate the verifier report (e.g. html or xunit report)
+
+ Raises:
+ subprocess.CalledProcessError: if Rally doesn't return 0
+
+ Returns:
+ None
+ """
+ cmd = ["rally", "verify", "report", "--type", export_type,
+ "--uuid", uuid, "--to", file_name]
LOGGER.debug('running command: %s', cmd)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
LOGGER.info("%s\n%s", " ".join(cmd), output)
del environ['OS_TENANT_ID']
except Exception: # pylint: disable=broad-except
pass
- conf_utils.create_rally_deployment(environ=environ)
+ self.create_rally_deployment(environ=environ)
self.prepare_run(**kwargs)
self.run_tests(**kwargs)
self._generate_report()
- self.generate_html_report()
+ self.export_task(
+ "{}/{}.html".format(self.results_dir, self.case_name))
+ self.export_task(
+ "{}/{}.xml".format(self.results_dir, self.case_name),
+ export_type="junit-xml")
res = testcase.TestCase.EX_OK
except Exception as exc: # pylint: disable=broad-except
LOGGER.error('Error with run: %s', exc)
"""Tempest configuration utilities."""
-from __future__ import print_function
-
import json
import logging
-import fileinput
import os
import subprocess
from functest.utils import functest_utils
-RALLY_CONF_PATH = "/etc/rally/rally.conf"
-RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
- 'functest', 'ci/rally_aarch64_patch.conf')
GLANCE_IMAGE_PATH = os.path.join(
getattr(config.CONF, 'dir_functest_images'),
getattr(config.CONF, 'openstack_image_file_name'))
LOGGER = logging.getLogger(__name__)
-def create_rally_deployment(environ=None):
- """Create new rally deployment"""
- # set the architecture to default
- pod_arch = env.get("POD_ARCH")
- arch_filter = ['aarch64']
-
- if pod_arch and pod_arch in arch_filter:
- LOGGER.info("Apply aarch64 specific to rally config...")
- with open(RALLY_AARCH64_PATCH_PATH, "r") as pfile:
- rally_patch_conf = pfile.read()
-
- for line in fileinput.input(RALLY_CONF_PATH):
- print(line, end=' ')
- if "cirros|testvm" in line:
- print(rally_patch_conf)
-
- LOGGER.info("Creating Rally environment...")
-
- try:
- cmd = ['rally', 'deployment', 'destroy',
- '--deployment',
- str(getattr(config.CONF, 'rally_deployment_name'))]
- output = subprocess.check_output(cmd)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
- except subprocess.CalledProcessError:
- pass
-
- cmd = ['rally', 'deployment', 'create', '--fromenv',
- '--name', str(getattr(config.CONF, 'rally_deployment_name'))]
- output = subprocess.check_output(cmd, env=environ)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
-
- cmd = ['rally', 'deployment', 'check']
- output = subprocess.check_output(cmd)
- LOGGER.info("%s\n%s", " ".join(cmd), output)
- return get_verifier_deployment_id()
-
-
def create_verifier():
"""Create new verifier"""
LOGGER.info("Create verifier from existing repo...")
return verifier_uuid
-def get_verifier_deployment_id():
- """
- Returns deployment id for active Rally deployment
- """
- cmd = ("rally deployment list | awk '/" +
- getattr(config.CONF, 'rally_deployment_name') +
- "/ {print $2}'")
- proc = subprocess.Popen(cmd, shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT)
- deployment_uuid = proc.stdout.readline().rstrip()
- return deployment_uuid
-
-
def get_verifier_repo_dir(verifier_id):
"""
Returns installed verifier repo directory for Tempest
import yaml
from functest.core import singlevm
+from functest.opnfv_tests.openstack.rally import rally
from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
from functest.utils import env
LOGGER.info("Tempest %s success_rate is %s%%",
self.case_name, self.result)
- def generate_report(self):
- """Generate verification report."""
- html_file = os.path.join(self.res_dir,
- "tempest-report.html")
- cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
- self.verification_id, "--to", str(html_file)]
- subprocess.check_output(cmd)
-
def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'):
"""Set image name as tempest img_name_regex"""
rconfig = configparser.RawConfigParser()
del environ['OS_TENANT_ID']
except Exception: # pylint: disable=broad-except
pass
- self.deployment_id = conf_utils.create_rally_deployment(
+ self.deployment_id = rally.RallyBase.create_rally_deployment(
environ=environ)
if not self.deployment_id:
raise Exception("Deployment create failed")
self.apply_tempest_blacklist()
self.run_verifier_tests(**kwargs)
self.parse_verifier_result()
- self.generate_report()
+ rally.RallyBase.verify_report(
+ os.path.join(self.res_dir, "tempest-report.html"),
+ self.verification_id)
+ rally.RallyBase.verify_report(
+ os.path.join(self.res_dir, "tempest-report.xml"),
+ self.verification_id, "junit-xml")
res = testcase.TestCase.EX_OK
except Exception: # pylint: disable=broad-except
LOGGER.exception('Error with run')
return True
return False
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.get_verifier_deployment_id', return_value='foo')
+ @mock.patch('subprocess.check_output')
+ def test_create_rally_deployment(self, mock_exec, mock_get_id):
+ # pylint: disable=unused-argument
+ self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo')
+ calls = [
+ mock.call(['rally', 'deployment', 'destroy', '--deployment',
+ str(getattr(config.CONF, 'rally_deployment_name'))]),
+ mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
+ str(getattr(config.CONF, 'rally_deployment_name'))],
+ env=None),
+ mock.call(['rally', 'deployment', 'check'])]
+ mock_exec.assert_has_calls(calls)
+
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
self.rally_base.clean()
self.assertEqual(mock_delete_flavor.call_count, 1)
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'prepare_run')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'_generate_report')
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
- 'generate_html_report')
+ 'export_task')
def test_run_default(self, *args):
self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
for func in args:
func.assert_called()
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment', side_effect=Exception)
def test_run_exception_create_rally_dep(self, mock_create_rally_dep):
self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR)
mock_create_rally_dep.assert_called()
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'create_rally_deployment', return_value=mock.Mock())
@mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
'prepare_run', side_effect=Exception)
@mock.patch('subprocess.check_output',
side_effect=subprocess.CalledProcessError('', ''))
- def test_generate_html_report_ko(self, *args):
+ def test_export_task_ko(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
with self.assertRaises(subprocess.CalledProcessError):
- self.rally_base.generate_html_report()
- cmd = ["rally", "task", "report", "--deployment",
+ self.rally_base.export_task(file_name)
+ cmd = ["rally", "task", "export", "--type", "html", "--deployment",
str(getattr(config.CONF, 'rally_deployment_name')),
- "--out", "{}/{}.html".format(
- self.rally_base.results_dir, self.rally_base.case_name)]
+ "--to", file_name]
args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
@mock.patch('subprocess.check_output', return_value=None)
- def test_generate_html_report(self, *args):
- self.assertEqual(self.rally_base.generate_html_report(), None)
- cmd = ["rally", "task", "report", "--deployment",
+ def test_export_task(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ self.assertEqual(self.rally_base.export_task(file_name), None)
+ cmd = ["rally", "task", "export", "--type", "html", "--deployment",
str(getattr(config.CONF, 'rally_deployment_name')),
- "--out", "{}/{}.html".format(
- self.rally_base.results_dir, self.rally_base.case_name)]
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+ @mock.patch('subprocess.check_output',
+ side_effect=subprocess.CalledProcessError('', ''))
+ def test_verify_report_ko(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ with self.assertRaises(subprocess.CalledProcessError):
+ self.rally_base.verify_report(file_name, "1")
+ cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+ "--to", file_name]
+ args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+ @mock.patch('subprocess.check_output', return_value=None)
+ def test_verify_report(self, *args):
+ file_name = "{}/{}.html".format(
+ self.rally_base.results_dir, self.rally_base.case_name)
+ self.assertEqual(self.rally_base.verify_report(file_name, "1"), None)
+ cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1",
+ "--to", file_name]
args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
import mock
+from functest.opnfv_tests.openstack.rally import rally
from functest.opnfv_tests.openstack.tempest import conf_utils
from functest.utils import config
class OSTempestConfUtilsTesting(unittest.TestCase):
# pylint: disable=too-many-public-methods
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
- '.get_verifier_deployment_id', return_value='foo')
- @mock.patch('subprocess.check_output')
- def test_create_rally_deployment(self, mock_exec, mock_get_id):
- # pylint: disable=unused-argument
- self.assertEqual(conf_utils.create_rally_deployment(), 'foo')
- calls = [
- mock.call(['rally', 'deployment', 'destroy', '--deployment',
- str(getattr(config.CONF, 'rally_deployment_name'))]),
- mock.call(['rally', 'deployment', 'create', '--fromenv', '--name',
- str(getattr(config.CONF, 'rally_deployment_name'))],
- env=None),
- mock.call(['rally', 'deployment', 'check'])]
- mock_exec.assert_has_calls(calls)
-
@mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils'
'.LOGGER.debug')
def test_create_verifier(self, mock_logger_debug):
@mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
'create_verifier', return_value=mock.Mock())
- @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.'
- 'create_rally_deployment', return_value=mock.Mock())
+ @mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.create_rally_deployment', return_value=mock.Mock())
def test_get_verifier_id_default(self, mock_rally, mock_tempest):
# pylint: disable=unused-argument
setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name')
mock_stdout.configure_mock(**attrs)
mock_popen.return_value = mock_stdout
- self.assertEqual(conf_utils.get_verifier_deployment_id(),
+ self.assertEqual(rally.RallyBase.get_verifier_deployment_id(),
'test_deploy_id')
def test_get_verif_repo_dir_default(self):
from functest.opnfv_tests.openstack.tempest import tempest
from functest.opnfv_tests.openstack.tempest import conf_utils
-from functest.utils import config
class OSTempestTesting(unittest.TestCase):
with mock.patch('os_client_config.get_config'), \
mock.patch('shade.OpenStackCloud'), \
mock.patch('functest.core.tenantnetwork.NewProject'), \
- mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
- 'conf_utils.create_rally_deployment'), \
+ mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.create_rally_deployment'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.create_verifier'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.get_verifier_id',
return_value='test_deploy_id'), \
- mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
- 'conf_utils.get_verifier_deployment_id',
+ mock.patch('functest.opnfv_tests.openstack.rally.rally.'
+ 'RallyBase.get_verifier_deployment_id',
return_value='test_deploy_id'), \
mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'conf_utils.get_verifier_repo_dir',
mock_logger_info. \
assert_any_call("Starting Tempest test suite: '%s'.", cmd)
- @mock.patch('subprocess.check_output')
- def test_generate_report(self, mock_popen):
- self.tempestcommon.verification_id = "1234"
- html_file = os.path.join(
- os.path.join(
- getattr(config.CONF, 'dir_results'),
- self.tempestcommon.case_name),
- "tempest-report.html")
- cmd = ["rally", "verify", "report", "--type", "html", "--uuid",
- "1234", "--to", html_file]
- self.tempestcommon.generate_report()
- mock_popen.assert_called_once_with(cmd)
-
@mock.patch('functest.opnfv_tests.openstack.tempest.tempest.'
'os.path.exists', return_value=False)
@mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs',
'apply_tempest_blacklist'), \
mock.patch.object(self.tempestcommon, 'run_verifier_tests'), \
mock.patch.object(self.tempestcommon,
- 'parse_verifier_result'), \
- mock.patch.object(self.tempestcommon, 'generate_report'):
+ 'parse_verifier_result'):
self._test_run(testcase.TestCase.EX_OK)
args[0].assert_called_once_with()
Code Review / functest.git / commitdiff