From: boucherv <valentin.boucher@orange.com>
Date: Wed, 23 Aug 2017 16:02:02 +0000 (+0200)
Subject: Put vnf_test pass if more than 80% of vnf test result are OK
X-Git-Tag: opnfv-5.0.RC1~137^2
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F17%2F40017%2F4;p=functest.git

Put vnf_test pass if more than 80% of vnf test result are OK

Change-Id: I1042176c587ef2bf1197c724cb5b1b6d67ffba3d
Signed-off-by: boucherv <valentin.boucher@orange.com>
---

diff --git a/docker/vnf/testcases.yaml b/docker/vnf/testcases.yaml
index 9f6533930..b6a338792 100644
--- a/docker/vnf/testcases.yaml
+++ b/docker/vnf/testcases.yaml
@@ -9,7 +9,7 @@ tiers:
             -
                 case_name: cloudify_ims
                 project_name: functest
-                criteria: 100
+                criteria: 80
                 blocking: false
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml
index b0472b27c..adc7e09e0 100644
--- a/functest/ci/testcases.yaml
+++ b/functest/ci/testcases.yaml
@@ -467,7 +467,7 @@ tiers:
             -
                 case_name: cloudify_ims
                 project_name: functest
-                criteria: 100
+                criteria: 80
                 blocking: false
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
index b07eaee23..8ed7f0dda 100644
--- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py
+++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py
@@ -281,6 +281,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             external_network_name=ext_net_name,
             network_name=network_settings.name
         ))
+        self.result = 1/3 * 100
         return True
 
     def deploy_vnf(self):
@@ -330,6 +331,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         self.__logger.info(execution)
         if execution.status == 'terminated':
             self.details['vnf'].update(status='PASS', duration=duration)
+            self.result += 1/3 * 100
             result = True
         else:
             self.details['vnf'].update(status='FAIL', duration=duration)
@@ -355,12 +357,20 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             dns_ip=dns_ip,
             public_domain=self.vnf['inputs']["public_domain"])
         duration = time.time() - start_time
-        short_result = sig_test_format(vims_test_result)
+        short_result, nb_test = sig_test_format(vims_test_result)
         self.__logger.info(short_result)
-        self.details['test_vnf'].update(status='PASS',
-                                        result=short_result,
+        self.details['test_vnf'].update(result=short_result,
                                         full_result=vims_test_result,
                                         duration=duration)
+        try:
+            vnf_test_rate = short_result['passed'] / nb_test
+            # orchestrator + vnf + test_vnf
+            self.result += vnf_test_rate / 3 * 100
+        except ZeroDivisionError:
+            self.__logger.error("No test has been executed")
+            self.details['test_vnf'].update(status='FAIL')
+            return False
+
         return True
 
     def clean(self):
@@ -507,11 +517,12 @@ def sig_test_format(sig_test):
             nb_failures += 1
         elif data_test['result'] == "Skipped":
             nb_skipped += 1
-    total_sig_test_result = {}
-    total_sig_test_result['passed'] = nb_passed
-    total_sig_test_result['failures'] = nb_failures
-    total_sig_test_result['skipped'] = nb_skipped
-    return total_sig_test_result
+    short_sig_test_result = {}
+    short_sig_test_result['passed'] = nb_passed
+    short_sig_test_result['failures'] = nb_failures
+    short_sig_test_result['skipped'] = nb_skipped
+    nb_test = nb_passed + nb_skipped
+    return (short_sig_test_result, nb_test)
 
 
 def run_blocking_ssh_command(ssh, cmd, error_msg="Unable to run this command"):