Allow checking Tests count

It meets CNTT requirements (verifying conformance with a reference
implementation) and may protect vs synchronization issues (see gambia
refstack_defcore).

Change-Id: I842bc4ed91eec6bc545c489e3686d0d41dae7b44
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>
(cherry picked from commit 7c67624b8ee7f7a9a7fb36bb25e9881a63c982d4)

diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 4fa5aae..f5b7562 100644 (file)
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -43,6 +43,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 219
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Defcore testcases.

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index ef3c93e..ecc1739 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -206,6 +206,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 219
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Defcore testcases.

diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 05758ab..d30d594 100644 (file)
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -98,6 +98,7 @@ class TempestCommon(singlevm.VmReady2):
         except Exception:  # pylint: disable=broad-except
             pass
         self.deny_skipping = kwargs.get("deny_skipping", False)
+        self.tests_count = kwargs.get("tests_count", 0)
 
     def check_services(self):
         """Check the mandatory services."""
@@ -642,6 +643,9 @@ class TempestCommon(singlevm.VmReady2):
         skips = self.details.get("skipped_number", 0)
         if skips > 0 and self.deny_skipping:
             return testcase.TestCase.EX_TESTCASE_FAILED
+        if self.tests_count and (
+                self.details.get("tests_number", 0) != self.tests_count):
+            return testcase.TestCase.EX_TESTCASE_FAILED
         return super(TempestCommon, self).is_successful()