functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2017 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """CloudifyIms testcase implementation."""
  11
  12 import logging
  13 import os
  14 import time
  15
  16 from cloudify_rest_client import CloudifyClient
  17 from cloudify_rest_client.executions import Execution
  18 from scp import SCPClient
  19 import yaml
  20
  21 from functest.energy import energy
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base
  24 from functest.utils.constants import CONST
  25 import functest.utils.openstack_utils as os_utils
  26
  27 from snaps.openstack.create_network import (NetworkSettings, SubnetSettings,
  28                                             OpenStackNetwork)
  29 from snaps.openstack.create_security_group import (SecurityGroupSettings,
  30                                                    SecurityGroupRuleSettings,
  31                                                    Direction, Protocol,
  32                                                    OpenStackSecurityGroup)
  33 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
  34 from snaps.openstack.create_instance import (VmInstanceSettings,
  35                                              FloatingIpSettings,
  36                                              OpenStackVmInstance)
  37 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  38 from snaps.openstack.create_image import ImageSettings, OpenStackImage
  39 from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
  40 from snaps.openstack.create_network import PortSettings
  41
  42
  43 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
  44
  45
  46 class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
  47     """Clearwater vIMS deployed with Cloudify Orchestrator Case."""
  48
  49     __logger = logging.getLogger(__name__)
  50
  51     def __init__(self, **kwargs):
  52         """Initialize CloudifyIms testcase object."""
  53         if "case_name" not in kwargs:
  54             kwargs["case_name"] = "cloudify_ims"
  55         super(CloudifyIms, self).__init__(**kwargs)
  56
  57         # Retrieve the configuration
  58         try:
  59             self.config = CONST.__getattribute__(
  60                 'vnf_{}_config'.format(self.case_name))
  61         except Exception:
  62             raise Exception("VNF config file not found")
  63
  64         self.snaps_creds = ''
  65
  66         config_file = os.path.join(self.case_dir, self.config)
  67         self.orchestrator = dict(
  68             requirements=get_config("orchestrator.requirements", config_file),
  69         )
  70         self.details['orchestrator'] = dict(
  71             name=get_config("orchestrator.name", config_file),
  72             version=get_config("orchestrator.version", config_file),
  73             status='ERROR',
  74             result=''
  75         )
  76         self.__logger.debug("Orchestrator configuration %s", self.orchestrator)
  77         self.vnf = dict(
  78             descriptor=get_config("vnf.descriptor", config_file),
  79             inputs=get_config("vnf.inputs", config_file),
  80             requirements=get_config("vnf.requirements", config_file)
  81         )
  82         self.details['vnf'] = dict(
  83             descriptor_version=self.vnf['descriptor']['version'],
  84             name=get_config("vnf.name", config_file),
  85             version=get_config("vnf.version", config_file),
  86         )
  87         self.__logger.debug("VNF configuration: %s", self.vnf)
  88
  89         self.details['test_vnf'] = dict(
  90             name=get_config("vnf_test_suite.name", config_file),
  91             version=get_config("vnf_test_suite.version", config_file)
  92         )
  93         self.images = get_config("tenant_images", config_file)
  94         self.__logger.info("Images needed for vIMS: %s", self.images)
  95
  96     def prepare(self):
  97         """Prepare testscase (Additional pre-configuration steps)."""
  98         super(CloudifyIms, self).prepare()
  99
 100         self.__logger.info("Additional pre-configuration steps")
 101
 102         compute_quotas = self.os_project.get_compute_quotas()
 103         network_quotas = self.os_project.get_network_quotas()
 104
 105         for key, value in (
 106                 self.orchestrator['requirements']['compute_quotas'].items()):
 107             setattr(compute_quotas, key, value)
 108
 109         for key, value in (
 110                 self.orchestrator['requirements']['network_quotas'].items()):
 111             setattr(network_quotas, key, value)
 112
 113         compute_quotas = self.project_ims.update_compute_quotas(compute_quotas)
 114         network_quotas = self.project_ims.update_network_quotas(network_quotas)
 115
 116         # needs some images
 117         self.__logger.info("Upload some OS images if it doesn't exist")
 118         for image_name, image_url in self.images.iteritems():
 119             self.__logger.info("image: %s, url: %s", image_name, image_url)
 120             if image_url and image_name:
 121                 image_creator = OpenStackImage(
 122                     self.snaps_creds,
 123                     ImageSettings(name=image_name,
 124                                   image_user='cloud',
 125                                   img_format='qcow2',
 126                                   url=image_url))
 127                 image_creator.create()
 128                 # self.created_object.append(image_creator)
 129
 130     def deploy_orchestrator(self):
 131         """
 132         Deploy Cloudify Manager.
 133
 134         network, security group, fip, VM creation
 135         """
 136         # network creation
 137
 138         start_time = time.time()
 139         self.__logger.info("Creating keypair ...")
 140         kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
 141         keypair_settings = KeypairSettings(name='cloudify_ims_kp',
 142                                            private_filepath=kp_file)
 143         keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
 144         keypair_creator.create()
 145         self.created_object.append(keypair_creator)
 146
 147         self.__logger.info("Creating full network ...")
 148         subnet_settings = SubnetSettings(name='cloudify_ims_subnet',
 149                                          cidr='10.67.79.0/24')
 150         network_settings = NetworkSettings(name='cloudify_ims_network',
 151                                            subnet_settings=[subnet_settings])
 152         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 153         network_creator.create()
 154         self.created_object.append(network_creator)
 155         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 156         router_creator = OpenStackRouter(
 157             self.snaps_creds,
 158             RouterSettings(
 159                 name='cloudify_ims_router',
 160                 external_gateway=ext_net_name,
 161                 internal_subnets=[subnet_settings.name]))
 162         router_creator.create()
 163         self.created_object.append(router_creator)
 164
 165         # security group creation
 166         self.__logger.info("Creating security group for cloudify manager vm")
 167         sg_rules = list()
 168         sg_rules.append(
 169             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 170                                       direction=Direction.ingress,
 171                                       protocol=Protocol.tcp, port_range_min=1,
 172                                       port_range_max=65535))
 173         sg_rules.append(
 174             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 175                                       direction=Direction.ingress,
 176                                       protocol=Protocol.udp, port_range_min=1,
 177                                       port_range_max=65535))
 178
 179         securit_group_creator = OpenStackSecurityGroup(
 180             self.snaps_creds,
 181             SecurityGroupSettings(
 182                 name="sg-cloudify-manager",
 183                 rule_settings=sg_rules))
 184
 185         securit_group_creator.create()
 186         self.created_object.append(securit_group_creator)
 187
 188         # orchestrator VM flavor
 189         self.__logger.info("Get or create flavor for cloudify manager vm ...")
 190
 191         flavor_settings = FlavorSettings(
 192             name=self.orchestrator['requirements']['flavor']['name'],
 193             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 194             disk=50,
 195             vcpus=2)
 196         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 197         flavor_creator.create()
 198         self.created_object.append(flavor_creator)
 199         image_settings = ImageSettings(
 200             name=self.orchestrator['requirements']['os_image'],
 201             image_user='centos',
 202             exists=True)
 203
 204         port_settings = PortSettings(name='cloudify_manager_port',
 205                                      network_name=network_settings.name)
 206
 207         manager_settings = VmInstanceSettings(
 208             name='cloudify_manager',
 209             flavor=flavor_settings.name,
 210             port_settings=[port_settings],
 211             security_group_names=[securit_group_creator.sec_grp_settings.name],
 212             floating_ip_settings=[FloatingIpSettings(
 213                 name='cloudify_manager_fip',
 214                 port_name=port_settings.name,
 215                 router_name=router_creator.router_settings.name)])
 216
 217         manager_creator = OpenStackVmInstance(self.snaps_creds,
 218                                               manager_settings,
 219                                               image_settings,
 220                                               keypair_settings)
 221
 222         self.__logger.info("Creating cloudify manager VM")
 223         manager_creator.create()
 224         self.created_object.append(manager_creator)
 225
 226         public_auth_url = os_utils.get_endpoint('identity')
 227
 228         self.__logger.info("Set creds for cloudify manager")
 229         cfy_creds = dict(keystone_username=self.tenant_name,
 230                          keystone_password=self.tenant_name,
 231                          keystone_tenant_name=self.tenant_name,
 232                          keystone_url=public_auth_url)
 233
 234         cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
 235                                     username='admin',
 236                                     password='admin',
 237                                     tenant='default_tenant')
 238
 239         self.orchestrator['object'] = cfy_client
 240
 241         self.__logger.info("Attemps running status of the Manager")
 242         cfy_status = None
 243         retry = 10
 244         while str(cfy_status) != 'running' and retry:
 245             try:
 246                 cfy_status = cfy_client.manager.get_status()['status']
 247                 self.__logger.debug("The current manager status is %s",
 248                                     cfy_status)
 249             except Exception:  # pylint: disable=broad-except
 250                 self.__logger.warning("Cloudify Manager isn't " +
 251                                       "up and running. Retrying ...")
 252             retry = retry - 1
 253             time.sleep(30)
 254
 255         if str(cfy_status) == 'running':
 256             self.__logger.info("Cloudify Manager is up and running")
 257         else:
 258             raise Exception("Cloudify Manager isn't up and running")
 259
 260         self.__logger.info("Put OpenStack creds in manager")
 261         secrets_list = cfy_client.secrets.list()
 262         for k, val in cfy_creds.iteritems():
 263             if not any(d.get('key', None) == k for d in secrets_list):
 264                 cfy_client.secrets.create(k, val)
 265             else:
 266                 cfy_client.secrets.update(k, val)
 267
 268         duration = time.time() - start_time
 269
 270         self.__logger.info("Put private keypair in manager")
 271         if manager_creator.vm_ssh_active(block=True):
 272             ssh = manager_creator.ssh_client()
 273             scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
 274             scp.put(kp_file, '~/')
 275             cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
 276             run_blocking_ssh_command(ssh, cmd)
 277             cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
 278             run_blocking_ssh_command(ssh, cmd)
 279             cmd = "sudo yum install -y gcc python-devel"
 280             run_blocking_ssh_command(ssh, cmd, "Unable to install packages \
 281                                                 on manager")
 282
 283         self.details['orchestrator'].update(status='PASS', duration=duration)
 284
 285         self.vnf['inputs'].update(dict(
 286             external_network_name=ext_net_name,
 287             network_name=network_settings.name
 288         ))
 289         return True
 290
 291     def deploy_vnf(self):
 292         """Deploy Clearwater IMS."""
 293         start_time = time.time()
 294
 295         self.__logger.info("Upload VNFD")
 296         cfy_client = self.orchestrator['object']
 297         descriptor = self.vnf['descriptor']
 298         cfy_client.blueprints.publish_archive(descriptor.get('url'),
 299                                               descriptor.get('name'),
 300                                               descriptor.get('file_name'))
 301
 302         self.__logger.info("Get or create flavor for all clearwater vm")
 303         flavor_settings = FlavorSettings(
 304             name=self.vnf['requirements']['flavor']['name'],
 305             ram=self.vnf['requirements']['flavor']['ram_min'],
 306             disk=25,
 307             vcpus=1)
 308         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 309         flavor_creator.create()
 310         self.created_object.append(flavor_creator)
 311
 312         self.vnf['inputs'].update(dict(
 313             flavor_id=self.vnf['requirements']['flavor']['name'],
 314         ))
 315
 316         self.__logger.info("Create VNF Instance")
 317         cfy_client.deployments.create(descriptor.get('name'),
 318                                       descriptor.get('name'),
 319                                       self.vnf.get('inputs'))
 320
 321         wait_for_execution(cfy_client,
 322                            _get_deployment_environment_creation_execution(
 323                                cfy_client, descriptor.get('name')),
 324                            self.__logger,
 325                            timeout=600)
 326
 327         self.__logger.info("Start the VNF Instance deployment")
 328         execution = cfy_client.executions.start(descriptor.get('name'),
 329                                                 'install')
 330         # Show execution log
 331         execution = wait_for_execution(cfy_client, execution, self.__logger)
 332
 333         duration = time.time() - start_time
 334
 335         self.__logger.info(execution)
 336         if execution.status == 'terminated':
 337             self.details['vnf'].update(status='PASS', duration=duration)
 338             result = True
 339         else:
 340             self.details['vnf'].update(status='FAIL', duration=duration)
 341             result = False
 342         return result
 343
 344     def test_vnf(self):
 345         """Run test on clearwater ims instance."""
 346         start_time = time.time()
 347
 348         cfy_client = self.orchestrator['object']
 349
 350         outputs = cfy_client.deployments.outputs.get(
 351             self.vnf['descriptor'].get('name'))['outputs']
 352         dns_ip = outputs['dns_ip']
 353         ellis_ip = outputs['ellis_ip']
 354         self.config_ellis(ellis_ip)
 355
 356         if not dns_ip:
 357             return False
 358
 359         vims_test_result = self.run_clearwater_live_test(
 360             dns_ip=dns_ip,
 361             public_domain=self.vnf['inputs']["public_domain"])
 362         duration = time.time() - start_time
 363         short_result = sig_test_format(vims_test_result)
 364         self.__logger.info(short_result)
 365         self.details['test_vnf'].update(status='PASS',
 366                                         result=short_result,
 367                                         full_result=vims_test_result,
 368                                         duration=duration)
 369         return True
 370
 371     def clean(self):
 372         """Clean created objects/functions."""
 373         try:
 374             cfy_client = self.orchestrator['object']
 375             dep_name = self.vnf['descriptor'].get('name')
 376             # kill existing execution
 377             self.__logger.info('Deleting the current deployment')
 378             exec_list = cfy_client.executions.list(dep_name)
 379             for execution in exec_list:
 380                 if execution['status'] == "started":
 381                     try:
 382                         cfy_client.executions.cancel(execution['id'],
 383                                                      force=True)
 384                     except:  # pylint: disable=broad-except
 385                         self.__logger.warn("Can't cancel the current exec")
 386
 387             execution = cfy_client.executions.start(
 388                 dep_name,
 389                 'uninstall',
 390                 parameters=dict(ignore_failure=True),
 391                 force=True)
 392
 393             wait_for_execution(cfy_client, execution, self.__logger)
 394             cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
 395             cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
 396         except:  # pylint: disable=broad-except
 397             self.__logger.warn("Some issue during the undeployment ..")
 398             self.__logger.warn("Tenant clean continue ..")
 399
 400         super(CloudifyIms, self).clean()
 401
 402     @energy.enable_recording
 403     def run(self, **kwargs):
 404         """Execute CloudifyIms test case."""
 405         super(CloudifyIms, self).run(**kwargs)
 406
 407
 408 # ----------------------------------------------------------
 409 #
 410 #               YAML UTILS
 411 #
 412 # -----------------------------------------------------------
 413 def get_config(parameter, file_path):
 414     """
 415     Get config parameter.
 416
 417     Returns the value of a given parameter in file.yaml
 418     parameter must be given in string format with dots
 419     Example: general.openstack.image_name
 420     """
 421     with open(file_path) as config_file:
 422         file_yaml = yaml.safe_load(config_file)
 423     config_file.close()
 424     value = file_yaml
 425     for element in parameter.split("."):
 426         value = value.get(element)
 427         if value is None:
 428             raise ValueError("The parameter %s is not defined in"
 429                              " reporting.yaml" % parameter)
 430     return value
 431
 432
 433 def wait_for_execution(client, execution, logger, timeout=2400, ):
 434     """Wait for a workflow execution on Cloudify Manager."""
 435     # if execution already ended - return without waiting
 436     if execution.status in Execution.END_STATES:
 437         return execution
 438
 439     if timeout is not None:
 440         deadline = time.time() + timeout
 441
 442     # Poll for execution status and execution logs, until execution ends
 443     # and we receive an event of type in WORKFLOW_END_TYPES
 444     offset = 0
 445     batch_size = 50
 446     event_list = []
 447     execution_ended = False
 448     while True:
 449         event_list = client.events.list(
 450             execution_id=execution.id,
 451             _offset=offset,
 452             _size=batch_size,
 453             include_logs=False,
 454             sort='@timestamp').items
 455
 456         offset = offset + len(event_list)
 457         for event in event_list:
 458             logger.debug(event.get('message'))
 459
 460         if timeout is not None:
 461             if time.time() > deadline:
 462                 raise RuntimeError(
 463                     'execution of operation {0} for deployment {1} '
 464                     'timed out'.format(execution.workflow_id,
 465                                        execution.deployment_id))
 466             else:
 467                 # update the remaining timeout
 468                 timeout = deadline - time.time()
 469
 470         if not execution_ended:
 471             execution = client.executions.get(execution.id)
 472             execution_ended = execution.status in Execution.END_STATES
 473
 474         if execution_ended:
 475             break
 476
 477         time.sleep(5)
 478
 479     return execution
 480
 481
 482 def _get_deployment_environment_creation_execution(client, deployment_id):
 483     """
 484     Get the execution id of a env preparation.
 485
 486     network, security group, fip, VM creation
 487     """
 488     executions = client.executions.list(deployment_id=deployment_id)
 489     for execution in executions:
 490         if execution.workflow_id == 'create_deployment_environment':
 491             return execution
 492     raise RuntimeError('Failed to get create_deployment_environment '
 493                        'workflow execution.'
 494                        'Available executions: {0}'.format(executions))
 495
 496
 497 def sig_test_format(sig_test):
 498     """Process the signaling result to have a short result."""
 499     nb_passed = 0
 500     nb_failures = 0
 501     nb_skipped = 0
 502     for data_test in sig_test:
 503         if data_test['result'] == "Passed":
 504             nb_passed += 1
 505         elif data_test['result'] == "Failed":
 506             nb_failures += 1
 507         elif data_test['result'] == "Skipped":
 508             nb_skipped += 1
 509     total_sig_test_result = {}
 510     total_sig_test_result['passed'] = nb_passed
 511     total_sig_test_result['failures'] = nb_failures
 512     total_sig_test_result['skipped'] = nb_skipped
 513     return total_sig_test_result
 514
 515
 516 def run_blocking_ssh_command(ssh, cmd, error_msg="Unable to run this command"):
 517     """Command to run ssh command with the exit status."""
 518     stdin, stdout, stderr = ssh.exec_command(cmd)
 519     if stdout.channel.recv_exit_status() != 0:
 520         raise Exception(error_msg)