functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """CloudifyIms testcase implementation."""
  11
  12 import logging
  13 import os
  14 import time
  15
  16 from cloudify_rest_client import CloudifyClient
  17 from cloudify_rest_client.executions import Execution
  18 from scp import SCPClient
  19 import yaml
  20
  21 from functest.energy import energy
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base
  24 from functest.utils.constants import CONST
  25 import functest.utils.openstack_utils as os_utils
  26
  27 from snaps.openstack.os_credentials import OSCreds
  28 from snaps.openstack.create_network import NetworkSettings, SubnetSettings, \
  29                                             OpenStackNetwork
  30 from snaps.openstack.create_security_group import SecurityGroupSettings, \
  31                                                     SecurityGroupRuleSettings,\
  32                                                     Direction, Protocol, \
  33                                                     OpenStackSecurityGroup
  34 from snaps.openstack.create_router import RouterSettings, OpenStackRouter
  35 from snaps.openstack.create_instance import VmInstanceSettings, \
  36                                                 FloatingIpSettings, \
  37                                                 OpenStackVmInstance
  38 from snaps.openstack.create_flavor import FlavorSettings, OpenStackFlavor
  39 from snaps.openstack.create_image import ImageSettings, OpenStackImage
  40 from snaps.openstack.create_keypairs import KeypairSettings, OpenStackKeypair
  41 from snaps.openstack.create_network import PortSettings
  42
  43
  44 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
  45
  46
  47 class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase):
  48     """Clearwater vIMS deployed with Cloudify Orchestrator Case."""
  49
  50     __logger = logging.getLogger(__name__)
  51
  52     def __init__(self, **kwargs):
  53         """Initialize CloudifyIms testcase object."""
  54         if "case_name" not in kwargs:
  55             kwargs["case_name"] = "cloudify_ims"
  56         super(CloudifyIms, self).__init__(**kwargs)
  57
  58         # Retrieve the configuration
  59         try:
  60             self.config = CONST.__getattribute__(
  61                 'vnf_{}_config'.format(self.case_name))
  62         except Exception:
  63             raise Exception("VNF config file not found")
  64
  65         self.snaps_creds = ''
  66         self.created_object = []
  67
  68         config_file = os.path.join(self.case_dir, self.config)
  69         self.orchestrator = dict(
  70             requirements=get_config("orchestrator.requirements", config_file),
  71         )
  72         self.details['orchestrator'] = dict(
  73             name=get_config("orchestrator.name", config_file),
  74             version=get_config("orchestrator.version", config_file),
  75             status='ERROR',
  76             result=''
  77         )
  78         self.__logger.debug("Orchestrator configuration %s", self.orchestrator)
  79         self.vnf = dict(
  80             descriptor=get_config("vnf.descriptor", config_file),
  81             inputs=get_config("vnf.inputs", config_file),
  82             requirements=get_config("vnf.requirements", config_file)
  83         )
  84         self.details['vnf'] = dict(
  85             descriptor_version=self.vnf['descriptor']['version'],
  86             name=get_config("vnf.name", config_file),
  87             version=get_config("vnf.version", config_file),
  88         )
  89         self.__logger.debug("VNF configuration: %s", self.vnf)
  90
  91         self.details['test_vnf'] = dict(
  92             name=get_config("vnf_test_suite.name", config_file),
  93             version=get_config("vnf_test_suite.version", config_file)
  94         )
  95         self.images = get_config("tenant_images", config_file)
  96         self.__logger.info("Images needed for vIMS: %s", self.images)
  97
  98     def prepare(self):
  99         """Prepare testscase (Additional pre-configuration steps)."""
 100         super(CloudifyIms, self).prepare()
 101
 102         self.__logger.info("Additional pre-configuration steps")
 103
 104         self.snaps_creds = OSCreds(
 105             username=self.creds['username'],
 106             password=self.creds['password'],
 107             auth_url=self.creds['auth_url'],
 108             project_name=self.creds['tenant'],
 109             identity_api_version=int(os_utils.get_keystone_client_version()))
 110
 111         # needs some images
 112         self.__logger.info("Upload some OS images if it doesn't exist")
 113         for image_name, image_url in self.images.iteritems():
 114             self.__logger.info("image: %s, url: %s", image_name, image_url)
 115             if image_url and image_name:
 116                 image_creator = OpenStackImage(
 117                     self.snaps_creds,
 118                     ImageSettings(name=image_name,
 119                                   image_user='cloud',
 120                                   img_format='qcow2',
 121                                   url=image_url))
 122                 image_creator.create()
 123                 # self.created_object.append(image_creator)
 124
 125     def deploy_orchestrator(self):
 126         """
 127         Deploy Cloudify Manager.
 128
 129         network, security group, fip, VM creation
 130         """
 131         # network creation
 132
 133         start_time = time.time()
 134         self.__logger.info("Creating keypair ...")
 135         kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
 136         keypair_settings = KeypairSettings(name='cloudify_ims_kp',
 137                                            private_filepath=kp_file)
 138         keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
 139         keypair_creator.create()
 140         self.created_object.append(keypair_creator)
 141
 142         self.__logger.info("Creating full network ...")
 143         subnet_settings = SubnetSettings(name='cloudify_ims_subnet',
 144                                          cidr='10.67.79.0/24')
 145         network_settings = NetworkSettings(name='cloudify_ims_network',
 146                                            subnet_settings=[subnet_settings])
 147         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 148         network_creator.create()
 149         self.created_object.append(network_creator)
 150         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 151         router_creator = OpenStackRouter(
 152             self.snaps_creds,
 153             RouterSettings(
 154                 name='cloudify_ims_router',
 155                 external_gateway=ext_net_name,
 156                 internal_subnets=[subnet_settings.name]))
 157         router_creator.create()
 158         self.created_object.append(router_creator)
 159
 160         # security group creation
 161         self.__logger.info("Creating security group for cloudify manager vm")
 162         sg_rules = list()
 163         sg_rules.append(
 164             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 165                                       direction=Direction.ingress,
 166                                       protocol=Protocol.tcp, port_range_min=1,
 167                                       port_range_max=65535))
 168         sg_rules.append(
 169             SecurityGroupRuleSettings(sec_grp_name="sg-cloudify-manager",
 170                                       direction=Direction.ingress,
 171                                       protocol=Protocol.udp, port_range_min=1,
 172                                       port_range_max=65535))
 173
 174         securit_group_creator = OpenStackSecurityGroup(
 175             self.snaps_creds,
 176             SecurityGroupSettings(
 177                 name="sg-cloudify-manager",
 178                 rule_settings=sg_rules))
 179
 180         securit_group_creator.create()
 181         self.created_object.append(securit_group_creator)
 182
 183         # orchestrator VM flavor
 184         self.__logger.info("Get or create flavor for cloudify manager vm ...")
 185
 186         flavor_settings = FlavorSettings(
 187             name=self.orchestrator['requirements']['flavor']['name'],
 188             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 189             disk=50,
 190             vcpus=2)
 191         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 192         flavor_creator.create()
 193         self.created_object.append(flavor_creator)
 194         image_settings = ImageSettings(
 195             name=self.orchestrator['requirements']['os_image'],
 196             image_user='centos',
 197             exists=True)
 198
 199         port_settings = PortSettings(name='cloudify_manager_port',
 200                                      network_name=network_settings.name)
 201
 202         manager_settings = VmInstanceSettings(
 203             name='cloudify_manager',
 204             flavor=flavor_settings.name,
 205             port_settings=[port_settings],
 206             security_group_names=[securit_group_creator.sec_grp_settings.name],
 207             floating_ip_settings=[FloatingIpSettings(
 208                 name='cloudify_manager_fip',
 209                 port_name=port_settings.name,
 210                 router_name=router_creator.router_settings.name)])
 211
 212         manager_creator = OpenStackVmInstance(self.snaps_creds,
 213                                               manager_settings,
 214                                               image_settings,
 215                                               keypair_settings)
 216
 217         self.__logger.info("Creating cloudify manager VM")
 218         manager_creator.create()
 219         self.created_object.append(manager_creator)
 220
 221         public_auth_url = os_utils.get_endpoint('identity')
 222
 223         self.__logger.info("Set creds for cloudify manager")
 224         cfy_creds = dict(keystone_username=self.tenant_name,
 225                          keystone_password=self.tenant_name,
 226                          keystone_tenant_name=self.tenant_name,
 227                          keystone_url=public_auth_url)
 228
 229         cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
 230                                     username='admin',
 231                                     password='admin',
 232                                     tenant='default_tenant')
 233
 234         self.orchestrator['object'] = cfy_client
 235
 236         self.__logger.info("Attemps running status of the Manager")
 237         cfy_status = None
 238         retry = 10
 239         while str(cfy_status) != 'running' and retry:
 240             try:
 241                 cfy_status = cfy_client.manager.get_status()['status']
 242             except Exception:  # pylint: disable=broad-except
 243                 self.__logger.warning("Cloudify Manager isn't " +
 244                                       "up and running. Retrying ...")
 245             retry = retry - 1
 246             time.sleep(30)
 247
 248         if str(cfy_status) == 'running':
 249             self.__logger.info("Cloudify Manager is up and running")
 250         else:
 251             raise Exception("Cloudify Manager isn't up and running")
 252
 253         self.__logger.info("Put OpenStack creds in manager")
 254         secrets_list = cfy_client.secrets.list()
 255         for k, val in cfy_creds.iteritems():
 256             if not any(d.get('key', None) == k for d in secrets_list):
 257                 cfy_client.secrets.create(k, val)
 258             else:
 259                 cfy_client.secrets.update(k, val)
 260
 261         duration = time.time() - start_time
 262
 263         self.__logger.info("Put private keypair in manager")
 264         if manager_creator.vm_ssh_active(block=True):
 265             ssh = manager_creator.ssh_client()
 266             scp = SCPClient(ssh.get_transport())
 267             scp.put(kp_file, '~/')
 268             cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
 269             ssh.exec_command(cmd)
 270             cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
 271             ssh.exec_command(cmd)
 272             cmd = "sudo yum install -y gcc python-devel"
 273             ssh.exec_command(cmd)
 274
 275         self.details['orchestrator'].update(status='PASS', duration=duration)
 276
 277         self.vnf['inputs'].update(dict(
 278             external_network_name=ext_net_name,
 279             network_name=network_settings.name
 280         ))
 281         return True
 282
 283     def deploy_vnf(self):
 284         """Deploy Clearwater IMS."""
 285         start_time = time.time()
 286
 287         self.__logger.info("Upload VNFD")
 288         cfy_client = self.orchestrator['object']
 289         descriptor = self.vnf['descriptor']
 290         cfy_client.blueprints.publish_archive(descriptor.get('url'),
 291                                               descriptor.get('name'),
 292                                               descriptor.get('file_name'))
 293
 294         self.__logger.info("Get or create flavor for all clearwater vm")
 295         self.exist_obj['flavor2'], flavor_id = os_utils.get_or_create_flavor(
 296             self.vnf['requirements']['flavor']['name'],
 297             self.vnf['requirements']['flavor']['ram_min'],
 298             '30',
 299             '1',
 300             public=True)
 301
 302         self.vnf['inputs'].update(dict(
 303             flavor_id=flavor_id,
 304         ))
 305
 306         self.__logger.info("Create VNF Instance")
 307         cfy_client.deployments.create(descriptor.get('name'),
 308                                       descriptor.get('name'),
 309                                       self.vnf.get('inputs'))
 310
 311         wait_for_execution(cfy_client,
 312                            _get_deployment_environment_creation_execution(
 313                                cfy_client, descriptor.get('name')),
 314                            self.__logger,
 315                            timeout=600)
 316
 317         self.__logger.info("Start the VNF Instance deployment")
 318         execution = cfy_client.executions.start(descriptor.get('name'),
 319                                                 'install')
 320         # Show execution log
 321         execution = wait_for_execution(cfy_client, execution, self.__logger)
 322
 323         duration = time.time() - start_time
 324
 325         self.__logger.info(execution)
 326         if execution.status == 'terminated':
 327             self.details['vnf'].update(status='PASS', duration=duration)
 328             result = True
 329         else:
 330             self.details['vnf'].update(status='FAIL', duration=duration)
 331             result = False
 332         return result
 333
 334     def test_vnf(self):
 335         """Run test on clearwater ims instance."""
 336         start_time = time.time()
 337
 338         cfy_client = self.orchestrator['object']
 339
 340         outputs = cfy_client.deployments.outputs.get(
 341             self.vnf['descriptor'].get('name'))['outputs']
 342         dns_ip = outputs['dns_ip']
 343         ellis_ip = outputs['ellis_ip']
 344         self.config_ellis(ellis_ip)
 345
 346         if not dns_ip:
 347             return False
 348
 349         vims_test_result = self.run_clearwater_live_test(
 350             dns_ip=dns_ip,
 351             public_domain=self.vnf['inputs']["public_domain"])
 352         duration = time.time() - start_time
 353         short_result = sig_test_format(vims_test_result)
 354         self.__logger.info(short_result)
 355         self.details['test_vnf'].update(status='PASS',
 356                                         result=short_result,
 357                                         full_result=vims_test_result,
 358                                         duration=duration)
 359         return True
 360
 361     def clean(self):
 362         """Clean created objects/functions."""
 363         try:
 364             cfy_client = self.orchestrator['object']
 365             dep_name = self.vnf['descriptor'].get('name')
 366             # kill existing execution
 367             self.__logger.info('Deleting the current deployment')
 368             exec_list = cfy_client.executions.list(dep_name)
 369             for execution in exec_list:
 370                 if execution['status'] == "started":
 371                     try:
 372                         cfy_client.executions.cancel(execution['id'],
 373                                                      force=True)
 374                     except:
 375                         self.__logger.warn("Can't cancel the current exec")
 376
 377             execution = cfy_client.executions.start(
 378                 dep_name,
 379                 'uninstall',
 380                 parameters=dict(ignore_failure=True),
 381                 force=True)
 382
 383             wait_for_execution(cfy_client, execution, self.__logger)
 384             cfy_client.deployments.delete(self.vnf['descriptor'].get('name'))
 385             cfy_client.blueprints.delete(self.vnf['descriptor'].get('name'))
 386         except:
 387             self.__logger.warn("Some issue during the undeployment ..")
 388             self.__logger.warn("Tenant clean continue ..")
 389
 390         self.__logger.info('Remove the cloudify manager OS object ..')
 391         for creator in reversed(self.created_object):
 392             try:
 393                 creator.clean()
 394             except Exception as exc:
 395                 self.logger.error('Unexpected error cleaning - %s', exc)
 396         super(CloudifyIms, self).clean()
 397
 398     @energy.enable_recording
 399     def run(self, **kwargs):
 400         """Execute CloudifyIms test case."""
 401         super(CloudifyIms, self).run(**kwargs)
 402
 403
 404 # ----------------------------------------------------------
 405 #
 406 #               YAML UTILS
 407 #
 408 # -----------------------------------------------------------
 409 def get_config(parameter, file_path):
 410     """
 411     Get config parameter.
 412
 413     Returns the value of a given parameter in file.yaml
 414     parameter must be given in string format with dots
 415     Example: general.openstack.image_name
 416     """
 417     with open(file_path) as config_file:
 418         file_yaml = yaml.safe_load(config_file)
 419     config_file.close()
 420     value = file_yaml
 421     for element in parameter.split("."):
 422         value = value.get(element)
 423         if value is None:
 424             raise ValueError("The parameter %s is not defined in"
 425                              " reporting.yaml" % parameter)
 426     return value
 427
 428
 429 def wait_for_execution(client, execution, logger, timeout=2400, ):
 430     """Wait for a workflow execution on Cloudify Manager."""
 431     # if execution already ended - return without waiting
 432     if execution.status in Execution.END_STATES:
 433         return execution
 434
 435     if timeout is not None:
 436         deadline = time.time() + timeout
 437
 438     # Poll for execution status and execution logs, until execution ends
 439     # and we receive an event of type in WORKFLOW_END_TYPES
 440     offset = 0
 441     batch_size = 50
 442     event_list = []
 443     execution_ended = False
 444     while True:
 445         event_list = client.events.list(
 446             execution_id=execution.id,
 447             _offset=offset,
 448             _size=batch_size,
 449             include_logs=False,
 450             sort='@timestamp').items
 451
 452         offset = offset + len(event_list)
 453         for event in event_list:
 454             logger.debug(event.get('message'))
 455
 456         if timeout is not None:
 457             if time.time() > deadline:
 458                 raise RuntimeError(
 459                     'execution of operation {0} for deployment {1} '
 460                     'timed out'.format(execution.workflow_id,
 461                                        execution.deployment_id))
 462             else:
 463                 # update the remaining timeout
 464                 timeout = deadline - time.time()
 465
 466         if not execution_ended:
 467             execution = client.executions.get(execution.id)
 468             execution_ended = execution.status in Execution.END_STATES
 469
 470         if execution_ended:
 471             break
 472
 473         time.sleep(5)
 474
 475     return execution
 476
 477
 478 def _get_deployment_environment_creation_execution(client, deployment_id):
 479     """
 480     Get the execution id of a env preparation.
 481
 482     network, security group, fip, VM creation
 483     """
 484     executions = client.executions.list(deployment_id=deployment_id)
 485     for execution in executions:
 486         if execution.workflow_id == 'create_deployment_environment':
 487             return execution
 488     raise RuntimeError('Failed to get create_deployment_environment '
 489                        'workflow execution.'
 490                        'Available executions: {0}'.format(executions))
 491
 492
 493 def sig_test_format(sig_test):
 494     """Process the signaling result to have a short result."""
 495     nb_passed = 0
 496     nb_failures = 0
 497     nb_skipped = 0
 498     for data_test in sig_test:
 499         if data_test['result'] == "Passed":
 500             nb_passed += 1
 501         elif data_test['result'] == "Failed":
 502             nb_failures += 1
 503         elif data_test['result'] == "Skipped":
 504             nb_skipped += 1
 505     total_sig_test_result = {}
 506     total_sig_test_result['passed'] = nb_passed
 507     total_sig_test_result['failures'] = nb_failures
 508     total_sig_test_result['skipped'] = nb_skipped
 509     return total_sig_test_result