functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2017 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 """CloudifyIms testcase implementation."""
  11
  12 from __future__ import division
  13
  14 import logging
  15 import os
  16 import time
  17 import yaml
  18
  19 from cloudify_rest_client.executions import Execution
  20 import pkg_resources
  21 import scp
  22 import six
  23
  24 from functest.core import cloudify
  25 from functest.opnfv_tests.vnf.ims import clearwater
  26 from functest.utils import config
  27 from functest.utils import env
  28
  29 __author__ = "Valentin Boucher <valentin.boucher@orange.com>"
  30
  31
  32 class CloudifyIms(cloudify.Cloudify):
  33     """Clearwater vIMS deployed with Cloudify Orchestrator Case."""
  34
  35     __logger = logging.getLogger(__name__)
  36
  37     filename_alt = ('/home/opnfv/functest/images/'
  38                     'ubuntu-14.04-server-cloudimg-amd64-disk1.img')
  39
  40     flavor_alt_ram = 2048
  41     flavor_alt_vcpus = 2
  42     flavor_alt_disk = 25
  43
  44     quota_security_group = 20
  45     quota_security_group_rule = 100
  46     quota_port = 50
  47
  48     def __init__(self, **kwargs):
  49         """Initialize CloudifyIms testcase object."""
  50         if "case_name" not in kwargs:
  51             kwargs["case_name"] = "cloudify_ims"
  52         super(CloudifyIms, self).__init__(**kwargs)
  53
  54         # Retrieve the configuration
  55         try:
  56             self.config = getattr(
  57                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  58         except Exception:
  59             raise Exception("VNF config file not found")
  60
  61         self.case_dir = pkg_resources.resource_filename(
  62             'functest', 'opnfv_tests/vnf/ims')
  63         config_file = os.path.join(self.case_dir, self.config)
  64
  65         self.details['orchestrator'] = dict(
  66             name=get_config("orchestrator.name", config_file),
  67             version=get_config("orchestrator.version", config_file),
  68             status='ERROR',
  69             result=''
  70         )
  71
  72         self.vnf = dict(
  73             descriptor=get_config("vnf.descriptor", config_file),
  74             inputs=get_config("vnf.inputs", config_file)
  75         )
  76         self.details['vnf'] = dict(
  77             descriptor_version=self.vnf['descriptor']['version'],
  78             name=get_config("vnf.name", config_file),
  79             version=get_config("vnf.version", config_file),
  80         )
  81         self.__logger.debug("VNF configuration: %s", self.vnf)
  82
  83         self.details['test_vnf'] = dict(
  84             name=get_config("vnf_test_suite.name", config_file),
  85             version=get_config("vnf_test_suite.version", config_file)
  86         )
  87
  88         self.image_alt = None
  89         self.flavor_alt = None
  90         self.clearwater = None
  91
  92     def check_requirements(self):
  93         if env.get('NEW_USER_ROLE').lower() == "admin":
  94             self.__logger.warn(
  95                 "Defining NEW_USER_ROLE=admin will easily break the testcase "
  96                 "because Cloudify doesn't manage tenancy (e.g. subnet  "
  97                 "overlapping)")
  98
  99     def execute(self):
 100         """
 101         Deploy Cloudify Manager.
 102
 103         network, security group, fip, VM creation
 104         """
 105         assert super(CloudifyIms, self).execute() == 0
 106         start_time = time.time()
 107         self.orig_cloud.set_network_quotas(
 108             self.project.project.name,
 109             security_group=self.quota_security_group,
 110             security_group_rule=self.quota_security_group_rule,
 111             port=self.quota_port)
 112         self.__logger.info("Put OpenStack creds in manager")
 113         cfy_creds = dict(
 114             keystone_username=self.project.user.name,
 115             keystone_password=self.project.password,
 116             keystone_tenant_name=self.project.project.name,
 117             keystone_url=self.get_public_auth_url(self.orig_cloud),
 118             region=os.environ.get('OS_REGION_NAME', 'RegionOne'),
 119             user_domain_name=os.environ.get(
 120                 'OS_USER_DOMAIN_NAME', 'Default'),
 121             project_domain_name=os.environ.get(
 122                 'OS_PROJECT_DOMAIN_NAME', 'Default'))
 123         self.__logger.info("Set creds for cloudify manager %s", cfy_creds)
 124
 125         for loop in range(10):
 126             try:
 127                 secrets_list = self.cfy_client.secrets.list()
 128                 for k, val in six.iteritems(cfy_creds):
 129                     if not any(d.get('key', None) == k for d in secrets_list):
 130                         self.cfy_client.secrets.create(k, val)
 131                     else:
 132                         self.cfy_client.secrets.update(k, val)
 133                 break
 134             except Exception:  # pylint: disable=broad-except
 135                 self.__logger.info(
 136                     "try %s: Cannot create secrets", loop + 1)
 137                 time.sleep(30)
 138         else:
 139             self.__logger.error("Cannot create secrets")
 140             return 1
 141
 142         duration = time.time() - start_time
 143
 144         self.__logger.info("Put private keypair in manager")
 145         scpc = scp.SCPClient(self.ssh.get_transport())
 146         scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem')
 147         (_, stdout, stderr) = self.ssh.exec_command(
 148             "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && "
 149             "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && "
 150             "sudo yum install -y gcc python-devel python-cmd2 && "
 151             "cfy status")
 152         self.__logger.info("output:\n%s", stdout.read())
 153         self.__logger.info("error:\n%s", stderr.read())
 154
 155         self.details['orchestrator'].update(status='PASS', duration=duration)
 156
 157         self.vnf['inputs'].update(dict(
 158             external_network_name=self.ext_net.name,
 159             network_name=self.network.name,
 160             key_pair_name=self.keypair.name
 161         ))
 162         if (self.deploy_vnf() and self.test_vnf()):
 163             self.result = 100
 164             return 0
 165         self.result = 1/3 * 100
 166         return 1
 167
 168     def deploy_vnf(self):
 169         """Deploy Clearwater IMS."""
 170         start_time = time.time()
 171
 172         self.cloud.create_security_group_rule(
 173             'default', port_range_min=22, port_range_max=22,
 174             protocol='tcp', direction='ingress')
 175
 176         self.__logger.info("Upload VNFD")
 177         descriptor = self.vnf['descriptor']
 178         self.cfy_client.blueprints.upload(
 179             descriptor.get('file_name'), descriptor.get('name'))
 180
 181         self.image_alt = self.publish_image_alt()
 182         self.flavor_alt = self.create_flavor_alt()
 183         self.vnf['inputs'].update(dict(
 184             image_id=self.image_alt.id,
 185             flavor_id=self.flavor_alt.id,
 186         ))
 187
 188         self.__logger.info("Create VNF Instance")
 189         self.cfy_client.deployments.create(
 190             descriptor.get('name'), descriptor.get('name'),
 191             self.vnf.get('inputs'))
 192
 193         wait_for_execution(
 194             self.cfy_client,
 195             get_execution_id(self.cfy_client, descriptor.get('name')),
 196             self.__logger, timeout=300)
 197
 198         self.__logger.info("Start the VNF Instance deployment")
 199         execution = self.cfy_client.executions.start(
 200             descriptor.get('name'), 'install')
 201         # Show execution log
 202         execution = wait_for_execution(
 203             self.cfy_client, execution, self.__logger, timeout=3600)
 204
 205         self.__logger.info(execution)
 206         if execution.status != 'terminated':
 207             self.details['vnf'].update(status='FAIL',
 208                                        duration=time.time() - start_time)
 209             return False
 210
 211         ellis_ip = self.cfy_client.deployments.outputs.get(
 212             self.vnf['descriptor'].get('name'))['outputs']['ellis_ip']
 213         self.clearwater = clearwater.ClearwaterTesting(self.case_name,
 214                                                        ellis_ip)
 215         self.clearwater.availability_check_by_creating_numbers()
 216
 217         self.details['vnf'].update(status='PASS',
 218                                    duration=time.time() - start_time)
 219         self.result += 1/3 * 100
 220         return True
 221
 222     def test_vnf(self):
 223         """Run test on clearwater ims instance."""
 224         start_time = time.time()
 225
 226         dns_ip = self.cfy_client.deployments.outputs.get(
 227             self.vnf['descriptor'].get('name'))['outputs']['dns_ip']
 228
 229         if not dns_ip:
 230             return False
 231
 232         short_result = self.clearwater.run_clearwater_live_test(
 233             dns_ip=dns_ip,
 234             public_domain=self.vnf['inputs']["public_domain"])
 235         duration = time.time() - start_time
 236         self.__logger.info(short_result)
 237         self.details['test_vnf'].update(result=short_result,
 238                                         duration=duration)
 239         try:
 240             vnf_test_rate = short_result['passed'] / (
 241                 short_result['total'] - short_result['skipped'])
 242             # orchestrator + vnf + test_vnf
 243             self.result += vnf_test_rate / 3 * 100
 244         except ZeroDivisionError:
 245             self.__logger.error("No test has been executed")
 246             self.details['test_vnf'].update(status='FAIL')
 247             return False
 248         except Exception:  # pylint: disable=broad-except
 249             self.__logger.exception("Cannot calculate results")
 250             self.details['test_vnf'].update(status='FAIL')
 251             return False
 252         return True if vnf_test_rate > 0 else False
 253
 254     def clean(self):
 255         """Clean created objects/functions."""
 256         try:
 257             dep_name = self.vnf['descriptor'].get('name')
 258             # kill existing execution
 259             self.__logger.info('Deleting the current deployment')
 260             exec_list = self.cfy_client.executions.list()
 261             for execution in exec_list:
 262                 if execution['status'] == "started":
 263                     try:
 264                         self.cfy_client.executions.cancel(
 265                             execution['id'], force=True)
 266                     except Exception:  # pylint: disable=broad-except
 267                         self.__logger.warn("Can't cancel the current exec")
 268
 269             execution = self.cfy_client.executions.start(
 270                 dep_name,
 271                 'uninstall',
 272                 parameters=dict(ignore_failure=True),
 273                 force=True)
 274
 275             wait_for_execution(self.cfy_client, execution, self.__logger)
 276             self.cfy_client.deployments.delete(
 277                 self.vnf['descriptor'].get('name'))
 278             self.cfy_client.blueprints.delete(
 279                 self.vnf['descriptor'].get('name'))
 280         except Exception:  # pylint: disable=broad-except
 281             self.__logger.exception("Some issue during the undeployment ..")
 282
 283         super(CloudifyIms, self).clean()
 284         if self.image_alt:
 285             self.cloud.delete_image(self.image_alt)
 286         if self.flavor_alt:
 287             self.orig_cloud.delete_flavor(self.flavor_alt.id)
 288
 289
 290 # ----------------------------------------------------------
 291 #
 292 #               YAML UTILS
 293 #
 294 # -----------------------------------------------------------
 295 def get_config(parameter, file_path):
 296     """
 297     Get config parameter.
 298
 299     Returns the value of a given parameter in file.yaml
 300     parameter must be given in string format with dots
 301     Example: general.openstack.image_name
 302     """
 303     with open(file_path) as config_file:
 304         file_yaml = yaml.safe_load(config_file)
 305     config_file.close()
 306     value = file_yaml
 307     for element in parameter.split("."):
 308         value = value.get(element)
 309         if value is None:
 310             raise ValueError("The parameter %s is not defined in"
 311                              " reporting.yaml" % parameter)
 312     return value
 313
 314
 315 def wait_for_execution(client, execution, logger, timeout=3600, ):
 316     """Wait for a workflow execution on Cloudify Manager."""
 317     # if execution already ended - return without waiting
 318     if execution.status in Execution.END_STATES:
 319         return execution
 320
 321     if timeout is not None:
 322         deadline = time.time() + timeout
 323
 324     # Poll for execution status and execution logs, until execution ends
 325     # and we receive an event of type in WORKFLOW_END_TYPES
 326     offset = 0
 327     batch_size = 50
 328     event_list = []
 329     execution_ended = False
 330     while True:
 331         event_list = client.events.list(
 332             execution_id=execution.id,
 333             _offset=offset,
 334             _size=batch_size,
 335             include_logs=True,
 336             sort='@timestamp').items
 337
 338         offset = offset + len(event_list)
 339         for event in event_list:
 340             logger.debug(event.get('message'))
 341
 342         if timeout is not None:
 343             if time.time() > deadline:
 344                 raise RuntimeError(
 345                     'execution of operation {0} for deployment {1} '
 346                     'timed out'.format(execution.workflow_id,
 347                                        execution.deployment_id))
 348             else:
 349                 # update the remaining timeout
 350                 timeout = deadline - time.time()
 351
 352         if not execution_ended:
 353             execution = client.executions.get(execution.id)
 354             execution_ended = execution.status in Execution.END_STATES
 355
 356         if execution_ended:
 357             break
 358
 359         time.sleep(5)
 360
 361     return execution
 362
 363
 364 def get_execution_id(client, deployment_id):
 365     """
 366     Get the execution id of a env preparation.
 367
 368     network, security group, fip, VM creation
 369     """
 370     executions = client.executions.list(deployment_id=deployment_id)
 371     for execution in executions:
 372         if execution.workflow_id == 'create_deployment_environment':
 373             return execution
 374     raise RuntimeError('Failed to get create_deployment_environment '
 375                        'workflow execution.'
 376                        'Available executions: {0}'.format(executions))