functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 import logging
  11 import os
  12 import time
  13 import yaml
  14 from scp import SCPClient
  15
  16 from cloudify_rest_client import CloudifyClient
  17 from cloudify_rest_client.executions import Execution
  18
  19 import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base
  20 from functest.utils.constants import CONST
  21 import functest.utils.openstack_utils as os_utils
  22
  23 from snaps.openstack.os_credentials import OSCreds
  24 from snaps.openstack.create_network import NetworkSettings, SubnetSettings, \
  25                                             OpenStackNetwork
  26 from snaps.openstack.create_security_group import SecurityGroupSettings, \
  27                                                     SecurityGroupRuleSettings,\
  28                                                     Direction, Protocol, \
  29                                                     OpenStackSecurityGroup
  30 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
  31 from snaps.openstack.create_instance import VmInstanceSettings, \
  32                                                 FloatingIpSettings, \
  33                                                 OpenStackVmInstance
  34 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  35 from snaps.openstack.create_image import ImageSettings, OpenStackImage
  36 from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
  37 from snaps.openstack.create_network import PortSettings
  38
  39 from functest.opnfv_tests.openstack.snaps import snaps_utils
  40
  41
  42 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
  43
  44
  45 class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
  46     """Clearwater vIMS deployed with Cloudify Orchestrator Case"""
  47
  48     __logger = logging.getLogger(__name__)
  49
  50     def __init__(self, **kwargs):
  51         if "case_name" not in kwargs:
  52             kwargs["case_name"] = "cloudify_ims"
  53         super(CloudifyIms, self).__init__(**kwargs)
  54
  55         # Retrieve the configuration
  56         try:
  57             self.config = CONST.__getattribute__(
  58                 'vnf_{}_config'.format(self.case_name))
  59         except Exception:
  60             raise Exception("VNF config file not found")
  61
  62         self.snaps_creds = ''
  63         self.created_object = []
  64
  65         config_file = os.path.join(self.case_dir, self.config)
  66         self.orchestrator = dict(
  67             requirements=get_config("orchestrator.requirements", config_file),
  68         )
  69         self.details['orchestrator'] = dict(
  70             name=get_config("orchestrator.name", config_file),
  71             version=get_config("orchestrator.version", config_file),
  72             status='ERROR',
  73             result=''
  74         )
  75         self.__logger.debug("Orchestrator configuration %s", self.orchestrator)
  76         self.vnf = dict(
  77             descriptor=get_config("vnf.descriptor", config_file),
  78             inputs=get_config("vnf.inputs", config_file),
  79             requirements=get_config("vnf.requirements", config_file)
  80         )
  81         self.details['vnf'] = dict(
  82             descriptor_version=self.vnf['descriptor']['version'],
  83             name=get_config("vnf.name", config_file),
  84             version=get_config("vnf.version", config_file),
  85         )
  86         self.__logger.debug("VNF configuration: %s", self.vnf)
  87
  88         self.details['test_vnf'] = dict(
  89             name=get_config("vnf_test_suite.name", config_file),
  90             version=get_config("vnf_test_suite.version", config_file)
  91         )
  92         self.images = get_config("tenant_images", config_file)
  93         self.__logger.info("Images needed for vIMS: %s", self.images)
  94
  95     def prepare(self):
  96         super(CloudifyIms, self).prepare()
  97
  98         self.__logger.info("Additional pre-configuration steps")
  99
 100         self.snaps_creds = OSCreds(
 101             username=self.creds['username'],
 102             password=self.creds['password'],
 103             auth_url=self.creds['auth_url'],
 104             project_name=self.creds['tenant'],
 105             identity_api_version=int(os_utils.get_keystone_client_version()))
 106
 107         # needs some images
 108         self.__logger.info("Upload some OS images if it doesn't exist")
 109         for image_name, image_url in self.images.iteritems():
 110             self.__logger.info("image: %s, url: %s", image_name, image_url)
 111             if image_url and image_name:
 112                 image_creator = OpenStackImage(
 113                     self.snaps_creds,
 114                     ImageSettings(name=image_name,
 115                                   image_user='cloud',
 116                                   img_format='qcow2',
 117                                   url=image_url))
 118                 image_creator.create()
 119                 # self.created_object.append(image_creator)
 120
 121     def deploy_orchestrator(self):
 122         """
 123         Deploy Cloudify Manager
 124
 125         network, security group, fip, VM creation
 126         """
 127         # network creation
 128
 129         start_time = time.time()
 130         self.__logger.info("Creating keypair ...")
 131         kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
 132         keypair_settings = KeypairSettings(name='cloudify_ims_kp',
 133                                            private_filepath=kp_file)
 134         keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
 135         keypair_creator.create()
 136         self.created_object.append(keypair_creator)
 137
 138         self.__logger.info("Creating full network ...")
 139         subnet_settings = SubnetSettings(name='cloudify_ims_subnet',
 140                                          cidr='10.67.79.0/24')
 141         network_settings = NetworkSettings(name='cloudify_ims_network',
 142                                            subnet_settings=[subnet_settings])
 143         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 144         network_creator.create()
 145         self.created_object.append(network_creator)
 146         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 147         router_creator = OpenStackRouter(
 148             self.snaps_creds,
 149             RouterSettings(
 150                 name='cloudify_ims_router',
 151                 external_gateway=ext_net_name,
 152                 internal_subnets=[subnet_settings.name]))
 153         router_creator.create()
 154         self.created_object.append(router_creator)
 155
 156         # security group creation
 157         self.__logger.info("Creating security group for cloudify manager vm")
 158         sg_rules = list()
 159         sg_rules.append(
 160             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 161                                       direction=Direction.ingress,
 162                                       protocol=Protocol.tcp, port_range_min=1,
 163                                       port_range_max=65535))
 164         sg_rules.append(
 165             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 166                                       direction=Direction.ingress,
 167                                       protocol=Protocol.udp, port_range_min=1,
 168                                       port_range_max=65535))
 169
 170         securit_group_creator = OpenStackSecurityGroup(
 171             self.snaps_creds,
 172             SecurityGroupSettings(
 173                 name="sg-cloudify-manager",
 174                 rule_settings=sg_rules))
 175
 176         securit_group_creator.create()
 177         self.created_object.append(securit_group_creator)
 178
 179         # orchestrator VM flavor
 180         self.__logger.info("Get or create flavor for cloudify manager vm ...")
 181
 182         flavor_settings = FlavorSettings(
 183             name=self.orchestrator['requirements']['flavor']['name'],
 184             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 185             disk=50,
 186             vcpus=2)
 187         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 188         flavor_creator.create()
 189         self.created_object.append(flavor_creator)
 190         image_settings = ImageSettings(
 191             name=self.orchestrator['requirements']['os_image'],
 192             image_user='centos',
 193             exists=True)
 194
 195         port_settings = PortSettings(name='cloudify_manager_port',
 196                                      network_name=network_settings.name)
 197
 198         manager_settings = VmInstanceSettings(
 199             name='cloudify_manager',
 200             flavor=flavor_settings.name,
 201             port_settings=[port_settings],
 202             security_group_names=[securit_group_creator.sec_grp_settings.name],
 203             floating_ip_settings=[FloatingIpSettings(
 204                 name='cloudify_manager_fip',
 205                 port_name=port_settings.name,
 206                 router_name=router_creator.router_settings.name)])
 207
 208         manager_creator = OpenStackVmInstance(self.snaps_creds,
 209                                               manager_settings,
 210                                               image_settings,
 211                                               keypair_settings)
 212
 213         self.__logger.info("Creating cloudify manager VM")
 214         manager_creator.create()
 215         self.created_object.append(manager_creator)
 216
 217         public_auth_url = os_utils.get_endpoint('identity')
 218
 219         self.__logger.info("Set creds for cloudify manager")
 220         cfy_creds = dict(keystone_username=self.tenant_name,
 221                          keystone_password=self.tenant_name,
 222                          keystone_tenant_name=self.tenant_name,
 223                          keystone_url=public_auth_url)
 224
 225         cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
 226                                     username='admin',
 227                                     password='admin',
 228                                     tenant='default_tenant')
 229
 230         self.orchestrator['object'] = cfy_client
 231
 232         self.__logger.info("Attemps running status of the Manager")
 233         cfy_status = None
 234         retry = 10
 235         while str(cfy_status) != 'running' and retry:
 236             try:
 237                 cfy_status = cfy_client.manager.get_status()['status']
 238             except Exception:  # pylint: disable=broad-except
 239                 self.__logger.warning("Cloudify Manager isn't " +
 240                                       "up and running. Retrying ...")
 241             retry = retry - 1
 242             time.sleep(30)
 243
 244         if str(cfy_status) == 'running':
 245             self.__logger.info("Cloudify Manager is up and running")
 246         else:
 247             raise Exception("Cloudify Manager isn't up and running")
 248
 249         self.__logger.info("Put OpenStack creds in manager")
 250         secrets_list = cfy_client.secrets.list()
 251         for k, val in cfy_creds.iteritems():
 252             if not any(d.get('key', None) == k for d in secrets_list):
 253                 cfy_client.secrets.create(k, val)
 254             else:
 255                 cfy_client.secrets.update(k, val)
 256
 257         duration = time.time() - start_time
 258
 259         self.__logger.info("Put private keypair in manager")
 260         if manager_creator.vm_ssh_active(block=True):
 261             ssh = manager_creator.ssh_client()
 262             scp = SCPClient(ssh.get_transport())
 263             scp.put(kp_file, '~/')
 264             cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
 265             ssh.exec_command(cmd)
 266             cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
 267             ssh.exec_command(cmd)
 268             cmd = "sudo yum install -y gcc python-devel"
 269             ssh.exec_command(cmd)
 270
 271         self.details['orchestrator'].update(status='PASS', duration=duration)
 272
 273         self.vnf['inputs'].update(dict(
 274             external_network_name=ext_net_name,
 275             network_name=network_settings.name
 276         ))
 277         return True
 278
 279     def deploy_vnf(self):
 280         """
 281         Deploy Clearwater IMS
 282         """
 283         start_time = time.time()
 284
 285         self.__logger.info("Upload VNFD")
 286         cfy_client = self.orchestrator['object']
 287         descriptor = self.vnf['descriptor']
 288         cfy_client.blueprints.publish_archive(descriptor.get('url'),
 289                                               descriptor.get('name'),
 290                                               descriptor.get('file_name'))
 291
 292         self.__logger.info("Get or create flavor for all clearwater vm")
 293         self.exist_obj['flavor2'], flavor_id = os_utils.get_or_create_flavor(
 294             self.vnf['requirements']['flavor']['name'],
 295             self.vnf['requirements']['flavor']['ram_min'],
 296             '30',
 297             '1',
 298             public=True)
 299
 300         self.vnf['inputs'].update(dict(
 301             flavor_id=flavor_id,
 302         ))
 303
 304         self.__logger.info("Create VNF Instance")
 305         cfy_client.deployments.create(descriptor.get('name'),
 306                                       descriptor.get('name'),
 307                                       self.vnf.get('inputs'))
 308
 309         wait_for_execution(cfy_client,
 310                            _get_deployment_environment_creation_execution(
 311                                cfy_client, descriptor.get('name')),
 312                            self.__logger,
 313                            timeout=600)
 314
 315         self.__logger.info("Start the VNF Instance deployment")
 316         execution = cfy_client.executions.start(descriptor.get('name'),
 317                                                 'install')
 318         # Show execution log
 319         execution = wait_for_execution(cfy_client, execution, self.__logger)
 320
 321         duration = time.time() - start_time
 322
 323         self.__logger.info(execution)
 324         if execution.status == 'terminated':
 325             self.details['vnf'].update(status='PASS', duration=duration)
 326             return True
 327         else:
 328             self.details['vnf'].update(status='FAIL', duration=duration)
 329             return False
 330
 331     def test_vnf(self):
 332         """
 333         Run test on clearwater ims instance
 334         """
 335         start_time = time.time()
 336
 337         cfy_client = self.orchestrator['object']
 338
 339         outputs = cfy_client.deployments.outputs.get(
 340             self.vnf['descriptor'].get('name'))['outputs']
 341         dns_ip = outputs['dns_ip']
 342         ellis_ip = outputs['ellis_ip']
 343         self.config_ellis(ellis_ip)
 344
 345         if dns_ip != "":
 346             vims_test_result = self.run_clearwater_live_test(
 347                 dns_ip=dns_ip,
 348                 public_domain=self.vnf['inputs']["public_domain"])
 349             duration = time.time() - start_time
 350             short_result = sig_test_format(vims_test_result)
 351             self.__logger.info(short_result)
 352             self.details['test_vnf'].update(status='PASS',
 353                                             result=short_result,
 354                                             full_result=vims_test_result,
 355                                             duration=duration)
 356             return True
 357         else:
 358             return False
 359
 360     def clean(self):
 361         try:
 362             cfy_client = self.orchestrator['object']
 363             dep_name = self.vnf['descriptor'].get('name')
 364             # kill existing execution
 365             self.__logger.info('Deleting the current deployment')
 366             exec_list = cfy_client.executions.list(dep_name)
 367             for execution in exec_list:
 368                 if execution['status'] == "started":
 369                     try:
 370                         cfy_client.executions.cancel(execution['id'],
 371                                                      force=True)
 372                     except:
 373                         self.__logger.warn("Can't cancel the current exec")
 374
 375             execution = cfy_client.executions.start(
 376                 dep_name,
 377                 'uninstall',
 378                 parameters=dict(ignore_failure=True),
 379                 force=True)
 380
 381             wait_for_execution(cfy_client, execution, self.__logger)
 382             cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
 383             cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
 384         except:
 385             self.__logger.warn("Some issue during the undeployment ..")
 386             self.__logger.warn("Tenant clean continue ..")
 387
 388         self.__logger.info('Remove the cloudify manager OS object ..')
 389         for creator in reversed(self.created_object):
 390             try:
 391                 creator.clean()
 392             except Exception as e:
 393                 self.logger.error('Unexpected error cleaning - %s', e)
 394         super(CloudifyIms, self).clean()
 395
 396
 397 # ----------------------------------------------------------
 398 #
 399 #               YAML UTILS
 400 #
 401 # -----------------------------------------------------------
 402 def get_config(parameter, file_path):
 403     """
 404     Returns the value of a given parameter in file.yaml
 405     parameter must be given in string format with dots
 406     Example: general.openstack.image_name
 407     """
 408     with open(file_path) as config_file:
 409         file_yaml = yaml.safe_load(config_file)
 410     config_file.close()
 411     value = file_yaml
 412     for element in parameter.split("."):
 413         value = value.get(element)
 414         if value is None:
 415             raise ValueError("The parameter %s is not defined in"
 416                              " reporting.yaml" % parameter)
 417     return value
 418
 419
 420 def wait_for_execution(client, execution, logger, timeout=2400, ):
 421     """
 422     Wait for a workflow execution on Cloudify Manager
 423     """
 424     # if execution already ended - return without waiting
 425     if execution.status in Execution.END_STATES:
 426         return execution
 427
 428     if timeout is not None:
 429         deadline = time.time() + timeout
 430
 431     # Poll for execution status and execution logs, until execution ends
 432     # and we receive an event of type in WORKFLOW_END_TYPES
 433     offset = 0
 434     batch_size = 50
 435     event_list = []
 436     execution_ended = False
 437     while True:
 438         event_list = client.events.list(
 439             execution_id=execution.id,
 440             _offset=offset,
 441             _size=batch_size,
 442             include_logs=False,
 443             sort='@timestamp').items
 444
 445         offset = offset + len(event_list)
 446         for event in event_list:
 447             logger.debug(event.get('message'))
 448
 449         if timeout is not None:
 450             if time.time() > deadline:
 451                 raise RuntimeError(
 452                     'execution of operation {0} for deployment {1} '
 453                     'timed out'.format(execution.workflow_id,
 454                                        execution.deployment_id))
 455             else:
 456                 # update the remaining timeout
 457                 timeout = deadline - time.time()
 458
 459         if not execution_ended:
 460             execution = client.executions.get(execution.id)
 461             execution_ended = execution.status in Execution.END_STATES
 462
 463         if execution_ended:
 464             break
 465
 466         time.sleep(5)
 467
 468     return execution
 469
 470
 471 def _get_deployment_environment_creation_execution(client, deployment_id):
 472     """
 473     Get the execution id of a env preparation
 474
 475     network, security group, fip, VM creation
 476     """
 477     executions = client.executions.list(deployment_id=deployment_id)
 478     for execution in executions:
 479         if execution.workflow_id == 'create_deployment_environment':
 480             return execution
 481     raise RuntimeError('Failed to get create_deployment_environment '
 482                        'workflow execution.'
 483                        'Available executions: {0}'.format(executions))
 484
 485
 486 def sig_test_format(sig_test):
 487     """
 488     Process the signaling result to have a short result
 489     """
 490     nb_passed = 0
 491     nb_failures = 0
 492     nb_skipped = 0
 493     for data_test in sig_test:
 494         if data_test['result'] == "Passed":
 495             nb_passed += 1
 496         elif data_test['result'] == "Failed":
 497             nb_failures += 1
 498         elif data_test['result'] == "Skipped":
 499             nb_skipped += 1
 500     total_sig_test_result = {}
 501     total_sig_test_result['passed'] = nb_passed
 502     total_sig_test_result['failures'] = nb_failures
 503     total_sig_test_result['skipped'] = nb_skipped
 504     return total_sig_test_result