functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2017 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """CloudifyIms testcase implementation."""
  11
  12 import logging
  13 import os
  14 import time
  15
  16 from cloudify_rest_client import CloudifyClient
  17 from cloudify_rest_client.executions import Execution
  18 from scp import SCPClient
  19 import yaml
  20
  21 from functest.energy import energy
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base
  24 from functest.utils.constants import CONST
  25 import functest.utils.openstack_utils as os_utils
  26
  27 from snaps.openstack.os_credentials import OSCreds
  28 from snaps.openstack.create_network import (NetworkSettings, SubnetSettings,
  29                                             OpenStackNetwork)
  30 from snaps.openstack.create_security_group import (SecurityGroupSettings,
  31                                                    SecurityGroupRuleSettings,
  32                                                    Direction, Protocol,
  33                                                    OpenStackSecurityGroup)
  34 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
  35 from snaps.openstack.create_instance import (VmInstanceSettings,
  36                                              FloatingIpSettings,
  37                                              OpenStackVmInstance)
  38 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  39 from snaps.openstack.create_image import ImageSettings, OpenStackImage
  40 from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
  41 from snaps.openstack.create_network import PortSettings
  42
  43
  44 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
  45
  46
  47 class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
  48     """Clearwater vIMS deployed with Cloudify Orchestrator Case."""
  49
  50     __logger = logging.getLogger(__name__)
  51
  52     def __init__(self, **kwargs):
  53         """Initialize CloudifyIms testcase object."""
  54         if "case_name" not in kwargs:
  55             kwargs["case_name"] = "cloudify_ims"
  56         super(CloudifyIms, self).__init__(**kwargs)
  57
  58         # Retrieve the configuration
  59         try:
  60             self.config = CONST.__getattribute__(
  61                 'vnf_{}_config'.format(self.case_name))
  62         except Exception:
  63             raise Exception("VNF config file not found")
  64
  65         self.snaps_creds = ''
  66         self.created_object = []
  67
  68         config_file = os.path.join(self.case_dir, self.config)
  69         self.orchestrator = dict(
  70             requirements=get_config("orchestrator.requirements", config_file),
  71         )
  72         self.details['orchestrator'] = dict(
  73             name=get_config("orchestrator.name", config_file),
  74             version=get_config("orchestrator.version", config_file),
  75             status='ERROR',
  76             result=''
  77         )
  78         self.__logger.debug("Orchestrator configuration %s", self.orchestrator)
  79         self.vnf = dict(
  80             descriptor=get_config("vnf.descriptor", config_file),
  81             inputs=get_config("vnf.inputs", config_file),
  82             requirements=get_config("vnf.requirements", config_file)
  83         )
  84         self.details['vnf'] = dict(
  85             descriptor_version=self.vnf['descriptor']['version'],
  86             name=get_config("vnf.name", config_file),
  87             version=get_config("vnf.version", config_file),
  88         )
  89         self.__logger.debug("VNF configuration: %s", self.vnf)
  90
  91         self.details['test_vnf'] = dict(
  92             name=get_config("vnf_test_suite.name", config_file),
  93             version=get_config("vnf_test_suite.version", config_file)
  94         )
  95         self.images = get_config("tenant_images", config_file)
  96         self.__logger.info("Images needed for vIMS: %s", self.images)
  97
  98     def prepare(self):
  99         """Prepare testscase (Additional pre-configuration steps)."""
 100         super(CloudifyIms, self).prepare()
 101
 102         self.__logger.info("Additional pre-configuration steps")
 103
 104         self.snaps_creds = OSCreds(
 105             username=self.creds['username'],
 106             password=self.creds['password'],
 107             auth_url=self.creds['auth_url'],
 108             project_name=self.creds['tenant'],
 109             identity_api_version=int(os_utils.get_keystone_client_version()))
 110
 111         # needs some images
 112         self.__logger.info("Upload some OS images if it doesn't exist")
 113         for image_name, image_file in self.images.iteritems():
 114             self.__logger.info("image: %s, file: %s", image_name, image_file)
 115             if image_file and image_name:
 116                 image_creator = OpenStackImage(
 117                     self.snaps_creds,
 118                     ImageSettings(name=image_name,
 119                                   image_user='cloud',
 120                                   img_format='qcow2',
 121                                   image_file=image_file))
 122                 image_creator.create()
 123                 # self.created_object.append(image_creator)
 124
 125     def deploy_orchestrator(self):
 126         """
 127         Deploy Cloudify Manager.
 128
 129         network, security group, fip, VM creation
 130         """
 131         # network creation
 132
 133         start_time = time.time()
 134         self.__logger.info("Creating keypair ...")
 135         kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
 136         keypair_settings = KeypairSettings(name='cloudify_ims_kp',
 137                                            private_filepath=kp_file)
 138         keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
 139         keypair_creator.create()
 140         self.created_object.append(keypair_creator)
 141
 142         self.__logger.info("Creating full network ...")
 143         subnet_settings = SubnetSettings(name='cloudify_ims_subnet',
 144                                          cidr='10.67.79.0/24')
 145         network_settings = NetworkSettings(name='cloudify_ims_network',
 146                                            subnet_settings=[subnet_settings])
 147         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 148         network_creator.create()
 149         self.created_object.append(network_creator)
 150         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 151         router_creator = OpenStackRouter(
 152             self.snaps_creds,
 153             RouterSettings(
 154                 name='cloudify_ims_router',
 155                 external_gateway=ext_net_name,
 156                 internal_subnets=[subnet_settings.name]))
 157         router_creator.create()
 158         self.created_object.append(router_creator)
 159
 160         # security group creation
 161         self.__logger.info("Creating security group for cloudify manager vm")
 162         sg_rules = list()
 163         sg_rules.append(
 164             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 165                                       direction=Direction.ingress,
 166                                       protocol=Protocol.tcp, port_range_min=1,
 167                                       port_range_max=65535))
 168         sg_rules.append(
 169             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 170                                       direction=Direction.ingress,
 171                                       protocol=Protocol.udp, port_range_min=1,
 172                                       port_range_max=65535))
 173
 174         securit_group_creator = OpenStackSecurityGroup(
 175             self.snaps_creds,
 176             SecurityGroupSettings(
 177                 name="sg-cloudify-manager",
 178                 rule_settings=sg_rules))
 179
 180         securit_group_creator.create()
 181         self.created_object.append(securit_group_creator)
 182
 183         # orchestrator VM flavor
 184         self.__logger.info("Get or create flavor for cloudify manager vm ...")
 185
 186         flavor_settings = FlavorSettings(
 187             name=self.orchestrator['requirements']['flavor']['name'],
 188             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 189             disk=50,
 190             vcpus=2)
 191         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 192         flavor_creator.create()
 193         self.created_object.append(flavor_creator)
 194         image_settings = ImageSettings(
 195             name=self.orchestrator['requirements']['os_image'],
 196             image_user='centos',
 197             exists=True)
 198
 199         port_settings = PortSettings(name='cloudify_manager_port',
 200                                      network_name=network_settings.name)
 201
 202         manager_settings = VmInstanceSettings(
 203             name='cloudify_manager',
 204             flavor=flavor_settings.name,
 205             port_settings=[port_settings],
 206             security_group_names=[securit_group_creator.sec_grp_settings.name],
 207             floating_ip_settings=[FloatingIpSettings(
 208                 name='cloudify_manager_fip',
 209                 port_name=port_settings.name,
 210                 router_name=router_creator.router_settings.name)])
 211
 212         manager_creator = OpenStackVmInstance(self.snaps_creds,
 213                                               manager_settings,
 214                                               image_settings,
 215                                               keypair_settings)
 216
 217         self.__logger.info("Creating cloudify manager VM")
 218         manager_creator.create()
 219         self.created_object.append(manager_creator)
 220
 221         public_auth_url = os_utils.get_endpoint('identity')
 222
 223         self.__logger.info("Set creds for cloudify manager")
 224         cfy_creds = dict(keystone_username=self.tenant_name,
 225                          keystone_password=self.tenant_name,
 226                          keystone_tenant_name=self.tenant_name,
 227                          keystone_url=public_auth_url)
 228
 229         cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
 230                                     username='admin',
 231                                     password='admin',
 232                                     tenant='default_tenant')
 233
 234         self.orchestrator['object'] = cfy_client
 235
 236         self.__logger.info("Attemps running status of the Manager")
 237         cfy_status = None
 238         retry = 10
 239         while str(cfy_status) != 'running' and retry:
 240             try:
 241                 cfy_status = cfy_client.manager.get_status()['status']
 242                 self.__logger.debug("The current manager status is %s",
 243                                     cfy_status)
 244             except Exception:  # pylint: disable=broad-except
 245                 self.__logger.warning("Cloudify Manager isn't " +
 246                                       "up and running. Retrying ...")
 247             retry = retry - 1
 248             time.sleep(30)
 249
 250         if str(cfy_status) == 'running':
 251             self.__logger.info("Cloudify Manager is up and running")
 252         else:
 253             raise Exception("Cloudify Manager isn't up and running")
 254
 255         self.__logger.info("Put OpenStack creds in manager")
 256         secrets_list = cfy_client.secrets.list()
 257         for k, val in cfy_creds.iteritems():
 258             if not any(d.get('key', None) == k for d in secrets_list):
 259                 cfy_client.secrets.create(k, val)
 260             else:
 261                 cfy_client.secrets.update(k, val)
 262
 263         duration = time.time() - start_time
 264
 265         self.__logger.info("Put private keypair in manager")
 266         if manager_creator.vm_ssh_active(block=True):
 267             ssh = manager_creator.ssh_client()
 268             scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
 269             scp.put(kp_file, '~/')
 270             cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
 271             run_blocking_ssh_command(ssh, cmd)
 272             cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
 273             run_blocking_ssh_command(ssh, cmd)
 274             cmd = "sudo yum install -y gcc python-devel"
 275             run_blocking_ssh_command(ssh, cmd, "Unable to install packages \
 276                                                 on manager")
 277
 278         self.details['orchestrator'].update(status='PASS', duration=duration)
 279
 280         self.vnf['inputs'].update(dict(
 281             external_network_name=ext_net_name,
 282             network_name=network_settings.name
 283         ))
 284         return True
 285
 286     def deploy_vnf(self):
 287         """Deploy Clearwater IMS."""
 288         start_time = time.time()
 289
 290         self.__logger.info("Upload VNFD")
 291         cfy_client = self.orchestrator['object']
 292         descriptor = self.vnf['descriptor']
 293         cfy_client.blueprints.publish_archive(descriptor.get('url'),
 294                                               descriptor.get('name'),
 295                                               descriptor.get('file_name'))
 296
 297         self.__logger.info("Get or create flavor for all clearwater vm")
 298         flavor_settings = FlavorSettings(
 299             name=self.vnf['requirements']['flavor']['name'],
 300             ram=self.vnf['requirements']['flavor']['ram_min'],
 301             disk=25,
 302             vcpus=1)
 303         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 304         flavor_creator.create()
 305         self.created_object.append(flavor_creator)
 306
 307         self.vnf['inputs'].update(dict(
 308             flavor_id=self.vnf['requirements']['flavor']['name'],
 309         ))
 310
 311         self.__logger.info("Create VNF Instance")
 312         cfy_client.deployments.create(descriptor.get('name'),
 313                                       descriptor.get('name'),
 314                                       self.vnf.get('inputs'))
 315
 316         wait_for_execution(cfy_client,
 317                            _get_deployment_environment_creation_execution(
 318                                cfy_client, descriptor.get('name')),
 319                            self.__logger,
 320                            timeout=600)
 321
 322         self.__logger.info("Start the VNF Instance deployment")
 323         execution = cfy_client.executions.start(descriptor.get('name'),
 324                                                 'install')
 325         # Show execution log
 326         execution = wait_for_execution(cfy_client, execution, self.__logger)
 327
 328         duration = time.time() - start_time
 329
 330         self.__logger.info(execution)
 331         if execution.status == 'terminated':
 332             self.details['vnf'].update(status='PASS', duration=duration)
 333             result = True
 334         else:
 335             self.details['vnf'].update(status='FAIL', duration=duration)
 336             result = False
 337         return result
 338
 339     def test_vnf(self):
 340         """Run test on clearwater ims instance."""
 341         start_time = time.time()
 342
 343         cfy_client = self.orchestrator['object']
 344
 345         outputs = cfy_client.deployments.outputs.get(
 346             self.vnf['descriptor'].get('name'))['outputs']
 347         dns_ip = outputs['dns_ip']
 348         ellis_ip = outputs['ellis_ip']
 349         self.config_ellis(ellis_ip)
 350
 351         if not dns_ip:
 352             return False
 353
 354         vims_test_result = self.run_clearwater_live_test(
 355             dns_ip=dns_ip,
 356             public_domain=self.vnf['inputs']["public_domain"])
 357         duration = time.time() - start_time
 358         short_result = sig_test_format(vims_test_result)
 359         self.__logger.info(short_result)
 360         self.details['test_vnf'].update(status='PASS',
 361                                         result=short_result,
 362                                         full_result=vims_test_result,
 363                                         duration=duration)
 364         return True
 365
 366     def clean(self):
 367         """Clean created objects/functions."""
 368         try:
 369             cfy_client = self.orchestrator['object']
 370             dep_name = self.vnf['descriptor'].get('name')
 371             # kill existing execution
 372             self.__logger.info('Deleting the current deployment')
 373             exec_list = cfy_client.executions.list(dep_name)
 374             for execution in exec_list:
 375                 if execution['status'] == "started":
 376                     try:
 377                         cfy_client.executions.cancel(execution['id'],
 378                                                      force=True)
 379                     except:  # pylint: disable=broad-except
 380                         self.__logger.warn("Can't cancel the current exec")
 381
 382             execution = cfy_client.executions.start(
 383                 dep_name,
 384                 'uninstall',
 385                 parameters=dict(ignore_failure=True),
 386                 force=True)
 387
 388             wait_for_execution(cfy_client, execution, self.__logger)
 389             cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
 390             cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
 391         except:  # pylint: disable=broad-except
 392             self.__logger.warn("Some issue during the undeployment ..")
 393             self.__logger.warn("Tenant clean continue ..")
 394
 395         self.__logger.info('Remove the cloudify manager OS object ..')
 396         for creator in reversed(self.created_object):
 397             try:
 398                 creator.clean()
 399             except Exception as exc:
 400                 self.logger.error('Unexpected error cleaning - %s', exc)
 401         super(CloudifyIms, self).clean()
 402
 403     @energy.enable_recording
 404     def run(self, **kwargs):
 405         """Execute CloudifyIms test case."""
 406         super(CloudifyIms, self).run(**kwargs)
 407
 408
 409 # ----------------------------------------------------------
 410 #
 411 #               YAML UTILS
 412 #
 413 # -----------------------------------------------------------
 414 def get_config(parameter, file_path):
 415     """
 416     Get config parameter.
 417
 418     Returns the value of a given parameter in file.yaml
 419     parameter must be given in string format with dots
 420     Example: general.openstack.image_name
 421     """
 422     with open(file_path) as config_file:
 423         file_yaml = yaml.safe_load(config_file)
 424     config_file.close()
 425     value = file_yaml
 426     for element in parameter.split("."):
 427         value = value.get(element)
 428         if value is None:
 429             raise ValueError("The parameter %s is not defined in"
 430                              " reporting.yaml" % parameter)
 431     return value
 432
 433
 434 def wait_for_execution(client, execution, logger, timeout=2400, ):
 435     """Wait for a workflow execution on Cloudify Manager."""
 436     # if execution already ended - return without waiting
 437     if execution.status in Execution.END_STATES:
 438         return execution
 439
 440     if timeout is not None:
 441         deadline = time.time() + timeout
 442
 443     # Poll for execution status and execution logs, until execution ends
 444     # and we receive an event of type in WORKFLOW_END_TYPES
 445     offset = 0
 446     batch_size = 50
 447     event_list = []
 448     execution_ended = False
 449     while True:
 450         event_list = client.events.list(
 451             execution_id=execution.id,
 452             _offset=offset,
 453             _size=batch_size,
 454             include_logs=False,
 455             sort='@timestamp').items
 456
 457         offset = offset + len(event_list)
 458         for event in event_list:
 459             logger.debug(event.get('message'))
 460
 461         if timeout is not None:
 462             if time.time() > deadline:
 463                 raise RuntimeError(
 464                     'execution of operation {0} for deployment {1} '
 465                     'timed out'.format(execution.workflow_id,
 466                                        execution.deployment_id))
 467             else:
 468                 # update the remaining timeout
 469                 timeout = deadline - time.time()
 470
 471         if not execution_ended:
 472             execution = client.executions.get(execution.id)
 473             execution_ended = execution.status in Execution.END_STATES
 474
 475         if execution_ended:
 476             break
 477
 478         time.sleep(5)
 479
 480     return execution
 481
 482
 483 def _get_deployment_environment_creation_execution(client, deployment_id):
 484     """
 485     Get the execution id of a env preparation.
 486
 487     network, security group, fip, VM creation
 488     """
 489     executions = client.executions.list(deployment_id=deployment_id)
 490     for execution in executions:
 491         if execution.workflow_id == 'create_deployment_environment':
 492             return execution
 493     raise RuntimeError('Failed to get create_deployment_environment '
 494                        'workflow execution.'
 495                        'Available executions: {0}'.format(executions))
 496
 497
 498 def sig_test_format(sig_test):
 499     """Process the signaling result to have a short result."""
 500     nb_passed = 0
 501     nb_failures = 0
 502     nb_skipped = 0
 503     for data_test in sig_test:
 504         if data_test['result'] == "Passed":
 505             nb_passed += 1
 506         elif data_test['result'] == "Failed":
 507             nb_failures += 1
 508         elif data_test['result'] == "Skipped":
 509             nb_skipped += 1
 510     total_sig_test_result = {}
 511     total_sig_test_result['passed'] = nb_passed
 512     total_sig_test_result['failures'] = nb_failures
 513     total_sig_test_result['skipped'] = nb_skipped
 514     return total_sig_test_result
 515
 516
 517 def run_blocking_ssh_command(ssh, cmd, error_msg="Unable to run this command"):
 518     """Command to run ssh command with the exit status."""
 519     stdin, stdout, stderr = ssh.exec_command(cmd)
 520     if stdout.channel.recv_exit_status() != 0:
 521         raise Exception(error_msg)