functest/opnfv_tests/vnf/ims/cloudify_ims.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Orange and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9
  10 import json
  11 import os
  12 import requests
  13 import subprocess
  14 import sys
  15 import time
  16 import yaml
  17
  18 import functest.core.vnf_base as vnf_base
  19 import functest.utils.functest_logger as ft_logger
  20 import functest.utils.functest_utils as ft_utils
  21 import functest.utils.openstack_utils as os_utils
  22
  23 from clearwater import Clearwater
  24 from functest.utils.constants import CONST
  25 from orchestrator_cloudify import Orchestrator
  26
  27
  28 class ImsVnf(vnf_base.VnfOnBoardingBase):
  29
  30     def __init__(self, project='functest', case='cloudify_ims',
  31                  repo='', cmd=''):
  32         super(ImsVnf, self).__init__(project, case, repo, cmd)
  33         self.logger = ft_logger.Logger("vIMS").getLogger()
  34         self.case_dir = os.path.join(CONST.dir_functest_test, 'vnf/ims/')
  35         self.data_dir = CONST.dir_ims_data
  36         self.test_dir = CONST.dir_repo_vims_test
  37
  38         # Retrieve the configuration
  39         try:
  40             self.config = CONST.__getattribute__(
  41                 'vnf_{}_config'.format(self.case_name))
  42         except:
  43             raise Exception("VNF config file not found")
  44
  45         config_file = self.case_dir + self.config
  46         self.orchestrator = dict(
  47             requirements=get_config("cloudify.requirements", config_file),
  48             blueprint=get_config("cloudify.blueprint", config_file),
  49             inputs=get_config("cloudify.inputs", config_file)
  50         )
  51         self.logger.debug("Orchestrator configuration: %s" % self.orchestrator)
  52         self.vnf = dict(
  53             blueprint=get_config("clearwater.blueprint", config_file),
  54             deployment_name=get_config("clearwater.deployment_name",
  55                                        config_file),
  56             inputs=get_config("clearwater.inputs", config_file),
  57             requirements=get_config("clearwater.requirements", config_file)
  58         )
  59         self.logger.debug("VNF configuration: %s" % self.vnf)
  60
  61         self.images = get_config("tenant_images", config_file)
  62         self.logger.info("Images needed for vIMS: %s" % self.images)
  63
  64         # vIMS Data directory creation
  65         if not os.path.exists(self.data_dir):
  66             os.makedirs(self.data_dir)
  67
  68     def deploy_orchestrator(self, **kwargs):
  69
  70         self.logger.info("Additional pre-configuration steps")
  71         self.neutron_client = os_utils.get_neutron_client(self.admin_creds)
  72         self.glance_client = os_utils.get_glance_client(self.admin_creds)
  73         self.keystone_client = os_utils.get_keystone_client(self.admin_creds)
  74         self.nova_client = os_utils.get_nova_client(self.admin_creds)
  75
  76         # needs some images
  77         self.logger.info("Upload some OS images if it doesn't exist")
  78         temp_dir = os.path.join(self.data_dir, "tmp/")
  79         for image_name, image_url in self.images.iteritems():
  80             self.logger.info("image: %s, url: %s" % (image_name, image_url))
  81             try:
  82                 image_id = os_utils.get_image_id(self.glance_client,
  83                                                  image_name)
  84                 self.logger.debug("image_id: %s" % image_id)
  85             except:
  86                 self.logger.error("Unexpected error: %s" % sys.exc_info()[0])
  87
  88             if image_id == '':
  89                 self.logger.info("""%s image doesn't exist on glance repository. Try
  90                 downloading this image and upload on glance !""" % image_name)
  91                 image_id = download_and_add_image_on_glance(self.glance_client,
  92                                                             image_name,
  93                                                             image_url,
  94                                                             temp_dir)
  95             if image_id == '':
  96                 self.step_failure(
  97                     "Failed to find or upload required OS "
  98                     "image for this deployment")
  99         # Need to extend quota
 100         self.logger.info("Update security group quota for this tenant")
 101         tenant_id = os_utils.get_tenant_id(self.keystone_client,
 102                                            self.tenant_name)
 103         self.logger.debug("Tenant id found %s" % tenant_id)
 104         if not os_utils.update_sg_quota(self.neutron_client,
 105                                         tenant_id, 50, 100):
 106             self.step_failure("Failed to update security group quota" +
 107                               " for tenant " + self.tenant_name)
 108         self.logger.debug("group quota extended")
 109
 110         # start the deployment of cloudify
 111         public_auth_url = os_utils.get_endpoint('identity')
 112
 113         self.logger.debug("CFY inputs: %s" % self.orchestrator['inputs'])
 114         cfy = Orchestrator(self.data_dir, self.orchestrator['inputs'])
 115         self.orchestrator['object'] = cfy
 116         self.logger.debug("Orchestrator object created")
 117
 118         self.logger.debug("Tenant name: %s" % self.tenant_name)
 119
 120         cfy.set_credentials(username=self.tenant_name,
 121                             password=self.tenant_name,
 122                             tenant_name=self.tenant_name,
 123                             auth_url=public_auth_url)
 124         self.logger.info("Credentials set in CFY")
 125
 126         # orchestrator VM flavor
 127         self.logger.info("Check Flavor is available, if not create one")
 128         self.logger.debug("Flavor details %s " %
 129                           self.orchestrator['requirements']['ram_min'])
 130         flavor_exist, flavor_id = os_utils.get_or_create_flavor(
 131             "m1.large",
 132             self.orchestrator['requirements']['ram_min'],
 133             '50',
 134             '2',
 135             public=True)
 136         self.logger.debug("Flavor id: %s" % flavor_id)
 137
 138         if not flavor_id:
 139             self.logger.info("Available flavors are: ")
 140             self.logger.info(self.nova_client.flavor.list())
 141             self.step_failure("Failed to find required flavor"
 142                               "for this deployment")
 143         cfy.set_flavor_id(flavor_id)
 144         self.logger.debug("Flavor OK")
 145
 146         # orchestrator VM image
 147         self.logger.debug("Orchestrator image")
 148         if 'os_image' in self.orchestrator['requirements'].keys():
 149             image_id = os_utils.get_image_id(
 150                 self.glance_client,
 151                 self.orchestrator['requirements']['os_image'])
 152             self.logger.debug("Orchestrator image id: %s" % image_id)
 153             if image_id == '':
 154                 self.logger.error("CFY image not found")
 155                 self.step_failure("Failed to find required OS image"
 156                                   " for cloudify manager")
 157         else:
 158             self.step_failure("Failed to find required OS image"
 159                               " for cloudify manager")
 160
 161         cfy.set_image_id(image_id)
 162         self.logger.debug("Orchestrator image set")
 163
 164         self.logger.debug("Get External network")
 165         ext_net = os_utils.get_external_net(self.neutron_client)
 166         self.logger.debug("External network: %s" % ext_net)
 167         if not ext_net:
 168             self.step_failure("Failed to get external network")
 169
 170         cfy.set_external_network_name(ext_net)
 171         self.logger.debug("CFY External network set")
 172
 173         self.logger.debug("get resolvconf")
 174         ns = ft_utils.get_resolvconf_ns()
 175         if ns:
 176             cfy.set_nameservers(ns)
 177             self.logger.debug("Resolvconf set")
 178
 179         self.logger.info("Prepare virtualenv for cloudify-cli")
 180         cmd = "chmod +x " + self.case_dir + "create_venv.sh"
 181         ft_utils.execute_command(cmd)
 182         time.sleep(3)
 183         cmd = self.case_dir + "create_venv.sh " + self.data_dir
 184         ft_utils.execute_command(cmd)
 185
 186         cfy.download_manager_blueprint(
 187             self.orchestrator['blueprint']['url'],
 188             self.orchestrator['blueprint']['branch'])
 189
 190         error = cfy.deploy_manager()
 191         if error:
 192             self.logger.error(error)
 193             return {'status': 'FAIL', 'result': error}
 194         else:
 195             return {'status': 'PASS', 'result': ''}
 196
 197     def deploy_vnf(self):
 198         cw = Clearwater(self.vnf['inputs'], self.orchestrator['object'],
 199                         self.logger)
 200         self.vnf['object'] = cw
 201
 202         self.logger.info("Collect flavor id for all clearwater vm")
 203         flavor_exist, flavor_id = os_utils.get_or_create_flavor(
 204             "m1.small",
 205             self.vnf['requirements']['ram_min'],
 206             '20',
 207             '1',
 208             public=True)
 209         self.logger.debug("Flavor id: %s" % flavor_id)
 210         if not flavor_id:
 211             self.logger.info("Available flavors are: ")
 212             self.logger.info(self.nova_client.flavor.list())
 213             self.step_failure("Failed to find required flavor"
 214                               " for this deployment")
 215
 216         cw.set_flavor_id(flavor_id)
 217
 218         # VMs image
 219         if 'os_image' in self.vnf['requirements'].keys():
 220             image_id = os_utils.get_image_id(
 221                 self.glance_client, self.vnf['requirements']['os_image'])
 222             if image_id == '':
 223                 self.step_failure("Failed to find required OS image"
 224                                   " for clearwater VMs")
 225         else:
 226             self.step_failure("Failed to find required OS image"
 227                               " for clearwater VMs")
 228
 229         cw.set_image_id(image_id)
 230
 231         ext_net = os_utils.get_external_net(self.neutron_client)
 232         if not ext_net:
 233             self.step_failure("Failed to get external network")
 234
 235         cw.set_external_network_name(ext_net)
 236
 237         error = cw.deploy_vnf(self.vnf['blueprint'])
 238         if error:
 239             self.logger.error(error)
 240             return {'status': 'FAIL', 'result': error}
 241         else:
 242             return {'status': 'PASS', 'result': ''}
 243
 244     def test_vnf(self):
 245         script = "source {0}venv_cloudify/bin/activate; "
 246         script += "cd {0}; "
 247         script += "cfy status | grep -Eo \"([0-9]{{1,3}}\.){{3}}[0-9]{{1,3}}\""
 248         cmd = "/bin/bash -c '" + script.format(self.data_dir) + "'"
 249
 250         try:
 251             self.logger.debug("Trying to get clearwater manager IP ... ")
 252             mgr_ip = os.popen(cmd).read()
 253             mgr_ip = mgr_ip.splitlines()[0]
 254         except:
 255             self.step_failure("Unable to retrieve the IP of the "
 256                               "cloudify manager server !")
 257
 258         api_url = "http://" + mgr_ip + "/api/v2"
 259         dep_outputs = requests.get(api_url + "/deployments/" +
 260                                    self.vnf['deployment_name'] + "/outputs")
 261         dns_ip = dep_outputs.json()['outputs']['dns_ip']
 262         ellis_ip = dep_outputs.json()['outputs']['ellis_ip']
 263
 264         ellis_url = "http://" + ellis_ip + "/"
 265         url = ellis_url + "accounts"
 266
 267         params = {"password": "functest",
 268                   "full_name": "opnfv functest user",
 269                   "email": "functest@opnfv.fr",
 270                   "signup_code": "secret"}
 271
 272         rq = requests.post(url, data=params)
 273         i = 20
 274         while rq.status_code != 201 and i > 0:
 275             rq = requests.post(url, data=params)
 276             i = i - 1
 277             time.sleep(10)
 278
 279         if rq.status_code == 201:
 280             url = ellis_url + "session"
 281             rq = requests.post(url, data=params)
 282             cookies = rq.cookies
 283         else:
 284             self.step_failure("Unable to create an account for number" +
 285                               " provision: %s" % rq.json()['reason'])
 286
 287         url = ellis_url + "accounts/" + params['email'] + "/numbers"
 288         if cookies != "":
 289             rq = requests.post(url, cookies=cookies)
 290             i = 24
 291             while rq.status_code != 200 and i > 0:
 292                 rq = requests.post(url, cookies=cookies)
 293                 i = i - 1
 294                 time.sleep(25)
 295
 296         if rq.status_code != 200:
 297             self.step_failure("Unable to create a number: %s"
 298                               % rq.json()['reason'])
 299
 300         nameservers = ft_utils.get_resolvconf_ns()
 301         resolvconf = ""
 302         for ns in nameservers:
 303             resolvconf += "\nnameserver " + ns
 304
 305         if dns_ip != "":
 306             script = ('echo -e "nameserver ' + dns_ip + resolvconf +
 307                       '" > /etc/resolv.conf; ')
 308             script += 'source /etc/profile.d/rvm.sh; '
 309             script += 'cd {0}; '
 310             script += ('rake test[{1}] SIGNUP_CODE="secret"')
 311
 312             cmd = ("/bin/bash -c '" +
 313                    script.format(self.data_dir, self.inputs["public_domain"]) +
 314                    "'")
 315             output_file = "output.txt"
 316             f = open(output_file, 'w+')
 317             subprocess.call(cmd, shell=True, stdout=f,
 318                             stderr=subprocess.STDOUT)
 319             f.close()
 320
 321             f = open(output_file, 'r')
 322             result = f.read()
 323             if result != "":
 324                 self.logger.debug(result)
 325
 326             vims_test_result = ""
 327             tempFile = os.path.join(self.test_dir, "temp.json")
 328             try:
 329                 self.logger.debug("Trying to load test results")
 330                 with open(tempFile) as f:
 331                     vims_test_result = json.load(f)
 332                 f.close()
 333             except:
 334                 self.logger.error("Unable to retrieve test results")
 335
 336             try:
 337                 os.remove(tempFile)
 338             except:
 339                 self.logger.error("Deleting file failed")
 340
 341             if vims_test_result != '':
 342                 return {'status': 'PASS', 'result': vims_test_result}
 343             else:
 344                 return {'status': 'FAIL', 'result': ''}
 345
 346     def clean(self):
 347         self.vnf['object'].undeploy_vnf()
 348         self.orchestrator['object'].undeploy_manager()
 349         super(ImsVnf, self).clean()
 350
 351     def main(self, **kwargs):
 352         self.logger.info("Cloudify IMS VNF onboarding test starting")
 353         self.execute()
 354         self.logger.info("Cloudify IMS VNF onboarding test executed")
 355         if self.criteria is "PASS":
 356             return self.EX_OK
 357         else:
 358             return self.EX_RUN_ERROR
 359
 360     def run(self):
 361         kwargs = {}
 362         return self.main(**kwargs)
 363
 364
 365 # ----------------------------------------------------------
 366 #
 367 #               YAML UTILS
 368 #
 369 # -----------------------------------------------------------
 370 def get_config(parameter, file):
 371     """
 372     Returns the value of a given parameter in file.yaml
 373     parameter must be given in string format with dots
 374     Example: general.openstack.image_name
 375     """
 376     with open(file) as f:
 377         file_yaml = yaml.safe_load(f)
 378     f.close()
 379     value = file_yaml
 380     for element in parameter.split("."):
 381         value = value.get(element)
 382         if value is None:
 383             raise ValueError("The parameter %s is not defined in"
 384                              " reporting.yaml" % parameter)
 385     return value
 386
 387
 388 def download_and_add_image_on_glance(glance, image_name, image_url, data_dir):
 389     dest_path = data_dir
 390     if not os.path.exists(dest_path):
 391         os.makedirs(dest_path)
 392     file_name = image_url.rsplit('/')[-1]
 393     if not ft_utils.download_url(image_url, dest_path):
 394         return False
 395
 396     image = os_utils.create_glance_image(
 397         glance, image_name, dest_path + file_name)
 398     if not image:
 399         return False
 400
 401     return image