functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 import yaml
  19 import functest.utils.openstack_utils as os_utils
  20 import functest.core.vnf as vnf
  21 import pkg_resources
  22
  23 from functest.opnfv_tests.openstack.snaps import snaps_utils
  24 from functest.utils.constants import CONST
  25
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.tests import openstack_tests
  33 from snaps.openstack.utils import keystone_utils
  34
  35 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  36 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  37
  38
  39 class JujuEpc(vnf.VnfOnBoarding):
  40     """Abot EPC deployed with JUJU Orchestrator Case"""
  41
  42     __logger = logging.getLogger(__name__)
  43
  44     def __init__(self, **kwargs):
  45         if "case_name" not in kwargs:
  46             kwargs["case_name"] = "juju_epc"
  47         super(JujuEpc, self).__init__(**kwargs)
  48
  49         # Retrieve the configuration
  50         self.case_dir = pkg_resources.resource_filename(
  51             'functest', 'opnfv_tests/vnf/epc')
  52         try:
  53             self.config = CONST.__getattribute__(
  54                 'vnf_{}_config'.format(self.case_name))
  55         except Exception:
  56             raise Exception("VNF config file not found")
  57         config_file = os.path.join(self.case_dir, self.config)
  58         self.orchestrator = dict(
  59             requirements=get_config("orchestrator.requirements", config_file),
  60         )
  61
  62         self.created_object = []
  63         self.snaps_creds = ''
  64
  65         self.os_creds = openstack_tests.get_credentials(
  66             os_env_file=CONST.__getattribute__('openstack_creds'))
  67
  68         self.details['orchestrator'] = dict(
  69             name=get_config("orchestrator.name", config_file),
  70             version=get_config("orchestrator.version", config_file),
  71             status='ERROR',
  72             result=''
  73         )
  74
  75         self.vnf = dict(
  76             descriptor=get_config("vnf.descriptor", config_file),
  77             requirements=get_config("vnf.requirements", config_file)
  78         )
  79         self.details['vnf'] = dict(
  80             descriptor_version=self.vnf['descriptor']['version'],
  81             name=get_config("vnf.name", config_file),
  82             version=get_config("vnf.version", config_file),
  83         )
  84         self.__logger.debug("VNF configuration: %s", self.vnf)
  85
  86         self.details['test_vnf'] = dict(
  87             name=get_config("vnf_test_suite.name", config_file),
  88             version=get_config("vnf_test_suite.version", config_file),
  89             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  90         )
  91         self.images = get_config("tenant_images", config_file)
  92         self.__logger.info("Images needed for vEPC: %s", self.images)
  93         self.keystone_client = os_utils.get_keystone_client()
  94         self.glance_client = os_utils.get_glance_client()
  95         self.neutron_client = os_utils.get_neutron_client()
  96         self.nova_client = os_utils.get_nova_client()
  97
  98     def prepare(self):
  99         """Prepare testcase (Additional pre-configuration steps)."""
 100         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 101
 102         super(JujuEpc, self).prepare()
 103
 104         self.__logger.info("Additional pre-configuration steps")
 105         self.public_auth_url = keystone_utils.get_endpoint(
 106             self.snaps_creds, 'identity')
 107
 108         self.creds = {
 109             "tenant": self.tenant_name,
 110             "username": self.tenant_name,
 111             "password": self.tenant_name,
 112             "auth_url": os_utils.get_credentials()['auth_url']
 113             }
 114
 115         self.snaps_creds = OSCreds(
 116             username=self.creds['username'],
 117             password=self.creds['password'],
 118             auth_url=self.creds['auth_url'],
 119             project_name=self.creds['tenant'],
 120             identity_api_version=int(os_utils.get_keystone_client_version()))
 121
 122         cloud_data = {
 123             'url': self.public_auth_url,
 124             'pass': self.tenant_name,
 125             'tenant_n': self.tenant_name,
 126             'user_n': self.tenant_name
 127         }
 128         self.__logger.info("Cloud DATA:  %s", cloud_data)
 129         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 130         self.__logger.info("Cretae  %s to add cloud info", self.filename)
 131         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 132
 133         if self.snaps_creds.identity_api_version == 3:
 134             append_config(self.filename, '{}'.format(
 135                 os_utils.get_credentials()['project_domain_name']),
 136                           '{}'.format(os_utils.get_credentials()
 137                                       ['user_domain_name']))
 138
 139         self.__logger.info("Upload some OS images if it doesn't exist")
 140         for image_name, image_file in self.images.iteritems():
 141             self.__logger.info("image: %s, file: %s", image_name, image_file)
 142             if image_file and image_name:
 143                 image_creator = OpenStackImage(
 144                     self.snaps_creds,
 145                     ImageSettings(name=image_name,
 146                                   image_user='cloud',
 147                                   img_format='qcow2',
 148                                   image_file=image_file))
 149                 image_creator.create()
 150                 self.created_object.append(image_creator)
 151
 152     def deploy_orchestrator(self):
 153         self.__logger.info("Deployed Orchestrator")
 154         private_net_name = CONST.__getattribute__(
 155             'vnf_{}_private_net_name'.format(self.case_name))
 156         private_subnet_name = CONST.__getattribute__(
 157             'vnf_{}_private_subnet_name'.format(self.case_name))
 158         private_subnet_cidr = CONST.__getattribute__(
 159             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 160         abot_router = CONST.__getattribute__(
 161             'vnf_{}_external_router'.format(self.case_name))
 162         dns_nameserver = CONST.__getattribute__(
 163             'vnf_{}_dns_nameserver'.format(self.case_name))
 164         ext_net_name = CONST.__getattribute__(
 165             'vnf_{}_external_network_name'.format(self.case_name))
 166
 167         self.__logger.info("Creating full network ...")
 168         subnet_settings = SubnetSettings(name=private_subnet_name,
 169                                          cidr=private_subnet_cidr,
 170                                          dns_nameservers=dns_nameserver)
 171         network_settings = NetworkSettings(name=private_net_name,
 172                                            subnet_settings=[subnet_settings])
 173         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 174         network_creator.create()
 175         self.created_object.append(network_creator)
 176
 177         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 178         self.__logger.info("Creating network Router ....")
 179         router_creator = OpenStackRouter(
 180             self.snaps_creds,
 181             RouterSettings(
 182                 name=abot_router,
 183                 external_gateway=ext_net_name,
 184                 internal_subnets=[subnet_settings.name]))
 185         router_creator.create()
 186         self.created_object.append(router_creator)
 187         self.__logger.info("Creating Flavor ....")
 188         flavor_settings = FlavorSettings(
 189             name=self.orchestrator['requirements']['flavor']['name'],
 190             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 191             disk=10,
 192             vcpus=1)
 193         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 194         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 195         flavor_creator.create()
 196         self.created_object.append(flavor_creator)
 197         self.__logger.info("Installing Dependency Packages .......")
 198         source_dir = "/src/epc-requirements/juju_bin_build"
 199         if os.path.exists(source_dir):
 200             shutil.rmtree(source_dir)
 201         os.makedirs(source_dir)
 202         os.environ['GOPATH'] = str(source_dir)
 203         os.environ['GOBIN'] = str(source_dir) + "/bin"
 204         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 205                               (os.path.expandvars('$GOBIN')) + ":" +
 206                               (os.path.expandvars('$PATH')))
 207         os.system('go get -d -v github.com/juju/juju/...')
 208         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 209         os.system('git checkout tags/juju-2.2.5')
 210         os.system('go get github.com/rogpeppe/godeps')
 211         os.system('godeps -u dependencies.tsv')
 212         os.system('go install -v github.com/juju/juju/...')
 213         self.__logger.info("Creating Cloud for Abot-epc .....")
 214         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 215         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 216         for image_name in self.images.keys():
 217             self.__logger.info("Generating Metadata for %s", image_name)
 218             image_id = os_utils.get_image_id(self.glance_client, image_name)
 219             os.system('juju metadata generate-image -d ~ -i {} -s {} -r '
 220                       'RegionOne -u {}'.format(image_id,
 221                                                image_name,
 222                                                self.public_auth_url))
 223         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 224         self.__logger.info("Credential information  : %s", net_id)
 225         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 226                                   '--config network={} --metadata-source ~  '
 227                                   '--constraints mem=2G --bootstrap-series '
 228                                   'trusty '
 229                                   '--config use-floating-ip=true --debug'.
 230                                   format(net_id))
 231         os.system(juju_bootstrap_command)
 232         return True
 233
 234     def deploy_vnf(self):
 235         """Deploy ABOT-OAI-EPC."""
 236         self.__logger.info("Upload VNFD")
 237         descriptor = self.vnf['descriptor']
 238         self.__logger.info("Get or create flavor for all Abot-EPC")
 239         flavor_settings = FlavorSettings(
 240             name=self.vnf['requirements']['flavor']['name'],
 241             ram=self.vnf['requirements']['flavor']['ram_min'],
 242             disk=10,
 243             vcpus=1)
 244         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 245         flavor_creator.create()
 246         self.created_object.append(flavor_creator)
 247         self.__logger.info("Deploying Abot-epc bundle file ...")
 248         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 249         self.__logger.info("Waiting for instances .....")
 250         status = os.system('juju-wait')
 251         self.__logger.info("juju wait completed: %s", status)
 252         self.__logger.info("Deployed Abot-epc on Openstack")
 253         if status == 0:
 254             instances = os_utils.get_instances(self.nova_client)
 255             for items in instances:
 256                 metadata = get_instance_metadata(self.nova_client, items)
 257                 if 'juju-units-deployed' in metadata:
 258                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 259                                  '-' + metadata['juju-model-uuid'])
 260                     self.sec_group_id = os_utils.get_security_group_id(
 261                         self.neutron_client, sec_group)
 262                     break
 263             self.__logger.info("Adding Security group rule....")
 264             os_utils.create_secgroup_rule(self.neutron_client,
 265                                           self.sec_group_id, 'ingress', 132)
 266             self.__logger.info("Copying the feature files to Abot_node ")
 267             os.system('juju scp -- -r {}/featureFiles abot-'
 268                       'epc-basic/0:~/'.format(self.case_dir))
 269             self.__logger.info("Copying the feature files in Abot_node ")
 270             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 271                       "~/featureFiles /etc/rebaca-test-suite"
 272                       "/featureFiles'")
 273             count = 0
 274             while count < 10:
 275                 epcstatus = os.system('juju status oai-epc | '
 276                                       'grep {} | grep {} | grep {}'
 277                                       .format('EPC', 'is', 'running'))
 278                 if epcstatus == 0:
 279                     break
 280                 else:
 281                     time.sleep(60)
 282                     count = count + 1
 283             os.system('juju-wait')
 284             return True
 285         else:
 286             return False
 287
 288     def test_vnf(self):
 289         start_time = time.time()
 290         self.__logger.info("Running VNF Test cases....")
 291         os.system('juju run-action abot-epc-basic/0 run '
 292                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 293         os.system('juju-wait')
 294         duration = time.time() - start_time
 295         self.__logger.info("Getting results from Abot node....")
 296         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 297                   'epc-basic/artifacts/TestResults.json {}/.'
 298                   .format(self.case_dir))
 299         self.__logger.info("Parsing the Test results...")
 300         res = (process_abot_test_result('{}/TestResults.'
 301                                         'json'.format(self.case_dir)))
 302         short_result = sig_test_format(res)
 303         self.__logger.info(short_result)
 304         self.details['test_vnf'].update(status='PASS',
 305                                         result=short_result,
 306                                         full_result=res,
 307                                         duration=duration)
 308
 309         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 310                            "%d, Skipped: %d", short_result['passed'],
 311                            short_result['failures'], short_result['skipped'])
 312         return True
 313
 314     def clean(self):
 315         try:
 316             if not self.orchestrator['requirements']['preserve_setup']:
 317                 self.__logger.info("Removing deployment files...")
 318                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 319                 if os.path.exists(testresult):
 320                     os.remove(testresult)
 321                 self.__logger.info("Removing %s file ", self.filename)
 322                 if os.path.exists(self.filename):
 323                     os.remove(self.filename)
 324                 self.__logger.info("Destroying Orchestrator...")
 325                 os.system('juju destroy-controller -y abot-controller '
 326                           '--destroy-all-models')
 327         except:
 328             self.__logger.warn("Some issue during the undeployment ..")
 329             self.__logger.warn("Tenant clean continue ..")
 330
 331         if not self.orchestrator['requirements']['preserve_setup']:
 332             self.__logger.info('Remove the Abot_epc OS object ..')
 333             for creator in reversed(self.created_object):
 334                 try:
 335                     creator.clean()
 336                 except Exception as exc:
 337                     self.__logger.error('Unexpected error cleaning - %s', exc)
 338
 339             self.__logger.info("Releasing all the floating IPs")
 340             # user_id = os_utils.get_user_id(self.keystone_client,
 341             #                               self.tenant_name)
 342             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 343             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 344                                                self.tenant_name)
 345             self.__logger.info("TENANT ID : %s", tenant_id)
 346             for item in floating_ips:
 347                 if item['tenant_id'] == tenant_id:
 348                     os_utils.delete_floating_ip(self.neutron_client,
 349                                                 item['id'])
 350             self.__logger.info("Cleaning Projects and Users")
 351             for creator in reversed(self.created_object):
 352                 try:
 353                     creator.clean()
 354                 except Exception as exc:  # pylint: disable=broad-except
 355                     self.__logger.error('Unexpected error cleaning - %s', exc)
 356         return True
 357
 358
 359 # ----------------------------------------------------------
 360 #
 361 #               YAML UTILS
 362 #
 363 # -----------------------------------------------------------
 364 def get_config(parameter, file_path):
 365     """
 366     Returns the value of a given parameter in file.yaml
 367     parameter must be given in string format with dots
 368     Example: general.openstack.image_name
 369     """
 370     with open(file_path) as config_file:
 371         file_yaml = yaml.safe_load(config_file)
 372     config_file.close()
 373     value = file_yaml
 374     for element in parameter.split("."):
 375         value = value.get(element)
 376         if value is None:
 377             raise ValueError("The parameter %s is not defined in"
 378                              " reporting.yaml" % parameter)
 379     return value
 380
 381
 382 def sig_test_format(sig_test):
 383     """
 384     Process the signaling result to have a short result
 385     """
 386     nb_passed = 0
 387     nb_failures = 0
 388     nb_skipped = 0
 389     for data_test in sig_test:
 390         if data_test['result'] == "passed":
 391             nb_passed += 1
 392         elif data_test['result'] == "failed":
 393             nb_failures += 1
 394         elif data_test['result'] == "skipped":
 395             nb_skipped += 1
 396     total_sig_test_result = {}
 397     total_sig_test_result['passed'] = nb_passed
 398     total_sig_test_result['failures'] = nb_failures
 399     total_sig_test_result['skipped'] = nb_skipped
 400     return total_sig_test_result
 401
 402
 403 def process_abot_test_result(file_path):
 404     """ Process ABoT Result """
 405     with open(file_path) as test_result:
 406         data = json.load(test_result)
 407         res = []
 408         for tests in data:
 409             tests = update_data(tests)
 410             try:
 411                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 412                 for steps in flatten_steps:
 413                     steps['result'] = steps['step_status']
 414                     res.append(steps)
 415             except:
 416                 logging.error("Could not post data to ElasticSearch host")
 417                 raise
 418         return res
 419
 420
 421 def update_data(obj):
 422     """ Update Result data"""
 423     try:
 424         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 425
 426         for element in obj['elements']:
 427             element['final_result'] = "passed"
 428             element['flatten_steps'] = []
 429
 430             for step in element['steps']:
 431                 temp_dict = {}
 432                 step['result'][step['result']['status']] = 1
 433                 if step['result']['status'].lower() in ['fail', 'failed']:
 434                     element['final_result'] = "failed"
 435
 436                 temp_dict['feature_file'] = obj['feature_file']
 437                 temp_dict['step_name'] = step['name']
 438                 temp_dict['step_status'] = step['result']['status']
 439                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 440                 temp_dict['step_' + step['result']['status']] = 1
 441                 element['flatten_steps'].append(deepcopy(temp_dict))
 442
 443             # Need to put the tag in OBJ and not ELEMENT
 444             if 'tags' in obj:
 445                 element['tags'] = deepcopy(obj['tags'])
 446                 for tag in obj['tags']:
 447                     element[tag['name']] = 1
 448             else:
 449                 for tag in element['tags']:
 450                     element[tag['name']] = 1
 451
 452     except:
 453         logging.error("Error in updating data, %s" % (sys.exc_info()[0]))
 454         raise
 455
 456     return obj
 457
 458
 459 def get_instance_metadata(nova_client, instance):
 460     """ Get instance Metadata - Instance ID """
 461     try:
 462         instance = nova_client.servers.get(instance.id)
 463         return instance.metadata
 464     except Exception as e:
 465         logging.error("Error [get_instance_status(nova_client)]: %s" % e)
 466         return None
 467
 468
 469 CLOUD_TEMPLATE = """clouds:
 470     abot-epc:
 471       type: openstack
 472       auth-types: [userpass]
 473       endpoint: {url}
 474       regions:
 475         RegionOne:
 476           endpoint: {url}
 477 credentials:
 478   abot-epc:
 479     abot-epc:
 480       auth-type: userpass
 481       password: {pass}
 482       tenant-name: {tenant_n}
 483       username: {user_n}"""
 484
 485
 486 def write_config(fname, template, **kwargs):
 487     """ Generate yaml from template for addinh cloud in juju """
 488     with open(fname, 'w') as yfile:
 489         yfile.write(template.format(**kwargs))
 490
 491
 492 def append_config(file_name, p_domain, u_domain):
 493     """ Append values into a yaml file  """
 494     with open(file_name) as yfile:
 495         doc = yaml.load(yfile)
 496     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 497         p_domain)
 498     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 499         u_domain)
 500
 501     with open(file_name, 'w') as yfile:
 502         yaml.safe_dump(doc, yfile, default_flow_style=False)