Put vnf_test pass if more than 80% of vnf test result are OK
[functest.git] / functest / opnfv_tests / vnf / ims / cloudify_ims.py
index fafc77e..8ed7f0d 100644 (file)
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2016 Orange and others.
+# Copyright (c) 2017 Orange and others.
 #
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
@@ -25,16 +25,16 @@ from functest.utils.constants import CONST
 import functest.utils.openstack_utils as os_utils
 
 from snaps.openstack.os_credentials import OSCreds
-from snaps.openstack.create_network import NetworkSettings, SubnetSettings, \
-                                            OpenStackNetwork
-from snaps.openstack.create_security_group import SecurityGroupSettings, \
-                                                    SecurityGroupRuleSettings,\
-                                                    Direction, Protocol, \
-                                                    OpenStackSecurityGroup
+from snaps.openstack.create_network import (NetworkSettings, SubnetSettings,
+                                            OpenStackNetwork)
+from snaps.openstack.create_security_group import (SecurityGroupSettings,
+                                                   SecurityGroupRuleSettings,
+                                                   Direction, Protocol,
+                                                   OpenStackSecurityGroup)
 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
-from snaps.openstack.create_instance import VmInstanceSettings, \
-                                                FloatingIpSettings, \
-                                                OpenStackVmInstance
+from snaps.openstack.create_instance import (VmInstanceSettings,
+                                             FloatingIpSettings,
+                                             OpenStackVmInstance)
 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
 from snaps.openstack.create_image import ImageSettings, OpenStackImage
 from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
@@ -110,15 +110,15 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
 
         # needs some images
         self.__logger.info("Upload some OS images if it doesn't exist")
-        for image_name, image_url in self.images.iteritems():
-            self.__logger.info("image: %s, url: %s", image_name, image_url)
-            if image_url and image_name:
+        for image_name, image_file in self.images.iteritems():
+            self.__logger.info("image: %s, file: %s", image_name, image_file)
+            if image_file and image_name:
                 image_creator = OpenStackImage(
                     self.snaps_creds,
                     ImageSettings(name=image_name,
                                   image_user='cloud',
                                   img_format='qcow2',
-                                  url=image_url))
+                                  image_file=image_file))
                 image_creator.create()
                 # self.created_object.append(image_creator)
 
@@ -239,6 +239,8 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         while str(cfy_status) != 'running' and retry:
             try:
                 cfy_status = cfy_client.manager.get_status()['status']
+                self.__logger.debug("The current manager status is %s",
+                                    cfy_status)
             except Exception:  # pylint: disable=broad-except
                 self.__logger.warning("Cloudify Manager isn't " +
                                       "up and running. Retrying ...")
@@ -263,14 +265,15 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         self.__logger.info("Put private keypair in manager")
         if manager_creator.vm_ssh_active(block=True):
             ssh = manager_creator.ssh_client()
-            scp = SCPClient(ssh.get_transport())
+            scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
             scp.put(kp_file, '~/')
             cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
-            ssh.exec_command(cmd)
+            run_blocking_ssh_command(ssh, cmd)
             cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
-            ssh.exec_command(cmd)
+            run_blocking_ssh_command(ssh, cmd)
             cmd = "sudo yum install -y gcc python-devel"
-            ssh.exec_command(cmd)
+            run_blocking_ssh_command(ssh, cmd, "Unable to install packages \
+                                                on manager")
 
         self.details['orchestrator'].update(status='PASS', duration=duration)
 
@@ -278,6 +281,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             external_network_name=ext_net_name,
             network_name=network_settings.name
         ))
+        self.result = 1/3 * 100
         return True
 
     def deploy_vnf(self):
@@ -292,15 +296,17 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
                                               descriptor.get('file_name'))
 
         self.__logger.info("Get or create flavor for all clearwater vm")
-        self.exist_obj['flavor2'], flavor_id = os_utils.get_or_create_flavor(
-            self.vnf['requirements']['flavor']['name'],
-            self.vnf['requirements']['flavor']['ram_min'],
-            '30',
-            '1',
-            public=True)
+        flavor_settings = FlavorSettings(
+            name=self.vnf['requirements']['flavor']['name'],
+            ram=self.vnf['requirements']['flavor']['ram_min'],
+            disk=25,
+            vcpus=1)
+        flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
+        flavor_creator.create()
+        self.created_object.append(flavor_creator)
 
         self.vnf['inputs'].update(dict(
-            flavor_id=flavor_id,
+            flavor_id=self.vnf['requirements']['flavor']['name'],
         ))
 
         self.__logger.info("Create VNF Instance")
@@ -325,6 +331,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
         self.__logger.info(execution)
         if execution.status == 'terminated':
             self.details['vnf'].update(status='PASS', duration=duration)
+            self.result += 1/3 * 100
             result = True
         else:
             self.details['vnf'].update(status='FAIL', duration=duration)
@@ -350,12 +357,20 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             dns_ip=dns_ip,
             public_domain=self.vnf['inputs']["public_domain"])
         duration = time.time() - start_time
-        short_result = sig_test_format(vims_test_result)
+        short_result, nb_test = sig_test_format(vims_test_result)
         self.__logger.info(short_result)
-        self.details['test_vnf'].update(status='PASS',
-                                        result=short_result,
+        self.details['test_vnf'].update(result=short_result,
                                         full_result=vims_test_result,
                                         duration=duration)
+        try:
+            vnf_test_rate = short_result['passed'] / nb_test
+            # orchestrator + vnf + test_vnf
+            self.result += vnf_test_rate / 3 * 100
+        except ZeroDivisionError:
+            self.__logger.error("No test has been executed")
+            self.details['test_vnf'].update(status='FAIL')
+            return False
+
         return True
 
     def clean(self):
@@ -371,7 +386,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
                     try:
                         cfy_client.executions.cancel(execution['id'],
                                                      force=True)
-                    except:
+                    except:  # pylint: disable=broad-except
                         self.__logger.warn("Can't cancel the current exec")
 
             execution = cfy_client.executions.start(
@@ -383,7 +398,7 @@ class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
             wait_for_execution(cfy_client, execution, self.__logger)
             cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
             cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
-        except:
+        except:  # pylint: disable=broad-except
             self.__logger.warn("Some issue during the undeployment ..")
             self.__logger.warn("Tenant clean continue ..")
 
@@ -502,8 +517,16 @@ def sig_test_format(sig_test):
             nb_failures += 1
         elif data_test['result'] == "Skipped":
             nb_skipped += 1
-    total_sig_test_result = {}
-    total_sig_test_result['passed'] = nb_passed
-    total_sig_test_result['failures'] = nb_failures
-    total_sig_test_result['skipped'] = nb_skipped
-    return total_sig_test_result
+    short_sig_test_result = {}
+    short_sig_test_result['passed'] = nb_passed
+    short_sig_test_result['failures'] = nb_failures
+    short_sig_test_result['skipped'] = nb_skipped
+    nb_test = nb_passed + nb_skipped
+    return (short_sig_test_result, nb_test)
+
+
+def run_blocking_ssh_command(ssh, cmd, error_msg="Unable to run this command"):
+    """Command to run ssh command with the exit status."""
+    stdin, stdout, stderr = ssh.exec_command(cmd)
+    if stdout.channel.recv_exit_status() != 0:
+        raise Exception(error_msg)