Allow checking Tests count

It meets CNTT requirements (verifying conformance with a reference
implementation) and may protect vs synchronization issues (see gambia
refstack_defcore).

Change-Id: I842bc4ed91eec6bc545c489e3686d0d41dae7b44
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml
index 00930a5..e600abd 100644 (file)
--- a/docker/smoke/testcases.yaml
+++ b/docker/smoke/testcases.yaml
@@ -44,6 +44,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 219
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Compute testcases.
@@ -60,6 +61,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 57
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Object testcases.
@@ -76,6 +78,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 275
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Platform testcases.

diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index 8b2a6c3..f7d8e9b 100644 (file)
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -207,6 +207,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 219
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Compute testcases.
@@ -223,6 +224,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 57
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Object testcases.
@@ -239,6 +241,7 @@ tiers:
                 criteria: 100
                 blocking: false
                 deny_skipping: true
+                tests_count: 275
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Platform testcases.

diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py
index 908d3bc..8711536 100644 (file)
--- a/functest/opnfv_tests/openstack/tempest/tempest.py
+++ b/functest/opnfv_tests/openstack/tempest/tempest.py
@@ -95,6 +95,7 @@ class TempestCommon(singlevm.VmReady2):
         except Exception:  # pylint: disable=broad-except
             pass
         self.deny_skipping = kwargs.get("deny_skipping", False)
+        self.tests_count = kwargs.get("tests_count", 0)
 
     def check_services(self):
         """Check the mandatory services."""
@@ -636,6 +637,9 @@ class TempestCommon(singlevm.VmReady2):
         skips = self.details.get("skipped_number", 0)
         if skips > 0 and self.deny_skipping:
             return testcase.TestCase.EX_TESTCASE_FAILED
+        if self.tests_count and (
+                self.details.get("tests_number", 0) != self.tests_count):
+            return testcase.TestCase.EX_TESTCASE_FAILED
         return super(TempestCommon, self).is_successful()