Generate one single report for Rally testcases

It eases sharing one single report including all task results.
For the time being, rally_jobs only generates neutron.html.
Then it only adds one report for rally_full and rally_sanity.

We may consider generating only one json file but it asks for
a refactoring (result parsing) which can be done in a second
change.

Change-Id: I8fa75ae80750336950f355c2aedc3dd4dc621cc5
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py
index 7ff6cd2..b6b285a 100644 (file)
--- a/functest/opnfv_tests/openstack/rally/rally.py
+++ b/functest/opnfv_tests/openstack/rally/rally.py
@@ -349,15 +349,6 @@ class RallyBase(singlevm.VmReady2):
         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
         LOGGER.info("%s\n%s", " ".join(cmd), output)
 
-        # save report as HTML
-        report_html_name = '{}.html'.format(test_name)
-        report_html_dir = os.path.join(self.results_dir, report_html_name)
-        cmd = (["rally", "task", "report", "--html", "--uuid", task_id,
-                "--out", report_html_dir])
-        LOGGER.debug('running command: %s', cmd)
-        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
-        LOGGER.info("%s\n%s", " ".join(cmd), output)
-
         json_results = open(report_json_dir).read()
         self._append_summary(json_results, test_name)
 
@@ -527,6 +518,22 @@ class RallyBase(singlevm.VmReady2):
                                     'nb success': success_rate}})
         self.details = payload
 
+    def generate_html_report(self):
+        """Save all task reports as single HTML
+
+        Raises:
+            subprocess.CalledProcessError: if Rally doesn't return 0
+
+        Returns:
+            None
+        """
+        cmd = ["rally", "task", "report", "--deployment",
+               str(getattr(config.CONF, 'rally_deployment_name')),
+               "--out", "{}/{}.html".format(self.results_dir, self.case_name)]
+        LOGGER.debug('running command: %s', cmd)
+        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+        LOGGER.info("%s\n%s", " ".join(cmd), output)
+
     def clean(self):
         """Cleanup of OpenStack resources. Should be called on completion."""
         self.clean_rally_conf()
@@ -563,6 +570,7 @@ class RallyBase(singlevm.VmReady2):
             self.prepare_run()
             self.run_tests(**kwargs)
             self._generate_report()
+            self.generate_html_report()
             res = testcase.TestCase.EX_OK
         except Exception as exc:   # pylint: disable=broad-except
             LOGGER.error('Error with run: %s', exc)

diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py
index bd691b8..fa307be 100644 (file)
--- a/functest/tests/unit/openstack/rally/test_rally.py
+++ b/functest/tests/unit/openstack/rally/test_rally.py
@@ -10,6 +10,7 @@
 import json
 import logging
 import os
+import subprocess
 import unittest
 
 import mock
@@ -17,6 +18,7 @@ import munch
 from xtesting.core import testcase
 
 from functest.opnfv_tests.openstack.rally import rally
+from functest.utils import config
 
 
 class OSRallyTesting(unittest.TestCase):
@@ -324,6 +326,8 @@ class OSRallyTesting(unittest.TestCase):
                 'run_tests')
     @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
                 '_generate_report')
+    @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.'
+                'generate_html_report')
     def test_run_default(self, *args):
         self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK)
         for func in args:
@@ -370,6 +374,26 @@ class OSRallyTesting(unittest.TestCase):
             self.assertEqual(self.rally_base.is_successful(), 424)
             mock_super(rally.RallyBase, self).is_successful.assert_called()
 
+    @mock.patch('subprocess.check_output',
+                side_effect=subprocess.CalledProcessError('', ''))
+    def test_generate_html_report_ko(self, *args):
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.rally_base.generate_html_report()
+        cmd = ["rally", "task", "report", "--deployment",
+               str(getattr(config.CONF, 'rally_deployment_name')),
+               "--out", "{}/{}.html".format(
+                   self.rally_base.results_dir, self.rally_base.case_name)]
+        args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
+    @mock.patch('subprocess.check_output', return_value=None)
+    def test_generate_html_report(self, *args):
+        self.assertEqual(self.rally_base.generate_html_report(), None)
+        cmd = ["rally", "task", "report", "--deployment",
+               str(getattr(config.CONF, 'rally_deployment_name')),
+               "--out", "{}/{}.html".format(
+                   self.rally_base.results_dir, self.rally_base.case_name)]
+        args[0].assert_called_with(cmd, stderr=subprocess.STDOUT)
+
 
 if __name__ == "__main__":
     logging.disable(logging.CRITICAL)