Do not allow testcase skipping in refstack

Set the result of refstack_defcore as failed if there were skipped testcases
in a run.

Change-Id: Id782c2dee511cefbafd5b4440f386fd66f934a47
Signed-off-by: Juha Kosonen <juha.kosonen@nokia.com>
(cherry picked from commit cfd79ec4c39e215a25fc3d99fc5368e7d874f7bf)

diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 67e1573..49ad8a8 100644 (file)
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -60,6 +60,7 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
                 description: >-
                     This test case runs a group of Rally jobs used in
                     OpenStack gating

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 912f35b..36a19c0 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -229,6 +229,7 @@ tiers:
                 project_name: functest
                 criteria: 100
                 blocking: false
+                deny_skipping: true
                 description: >-
                     This test case runs a group of Rally jobs used in
                     OpenStack gating

diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 597c711..57fedb7 100644 (file)
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -84,6 +84,7 @@ class TempestCommon(singlevm.VmReady2):
                 'neutron_extensions']
         except Exception:  # pylint: disable=broad-except
             pass
+        self.deny_skipping = kwargs.get("deny_skipping", False)
 
     def check_services(self):
         """Check the mandatory services."""
@@ -488,3 +489,10 @@ class TempestCommon(singlevm.VmReady2):
         if self.flavor_alt:
             self.orig_cloud.delete_flavor(self.flavor_alt.id)
         super(TempestCommon, self).clean()
+
+    def is_successful(self):
+        """The overall result of the test."""
+        skips = self.details.get("skipped_number", 0)
+        if skips > 0 and self.deny_skipping:
+            return testcase.TestCase.EX_TESTCASE_FAILED
+        return super(TempestCommon, self).is_successful()

diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py
index 646f882..f5a2f41 100644 (file)
--- a/functest/tests/unit/openstack/tempest/test_tempest.py
+++ b/functest/tests/unit/openstack/tempest/test_tempest.py
@@ -250,6 +250,25 @@ class OSTempestTesting(unittest.TestCase):
             self._test_run(testcase.TestCase.EX_OK)
             args[0].assert_called_once_with()
 
+    def test_is_successful_false(self):
+        with mock.patch('six.moves.builtins.super') as mock_super:
+            self.tempestcommon.deny_skipping = True
+            self.tempestcommon.details = {"skipped_number": 2}
+            self.assertEqual(self.tempestcommon.is_successful(),
+                             testcase.TestCase.EX_TESTCASE_FAILED)
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.assert_not_called()
+
+    def test_is_successful_true(self):
+        with mock.patch('six.moves.builtins.super') as mock_super:
+            self.tempestcommon.deny_skipping = False
+            self.tempestcommon.details = {"skipped_number": 2}
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.return_value = 567
+            self.assertEqual(self.tempestcommon.is_successful(), 567)
+            mock_super(tempest.TempestCommon,
+                       self).is_successful.assert_called()
+
 
 if __name__ == "__main__":
     logging.disable(logging.CRITICAL)