functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 from urlparse import urljoin
  19
  20 import functest.core.vnf as vnf
  21 from functest.opnfv_tests.openstack.snaps import snaps_utils
  22 from functest.utils.constants import CONST
  23 import functest.utils.openstack_utils as os_utils
  24
  25 import pkg_resources
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.tests import openstack_tests
  33 from snaps.openstack.utils import keystone_utils
  34 import yaml
  35
  36 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  37 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  38
  39
  40 class JujuEpc(vnf.VnfOnBoarding):
  41     # pylint:disable=too-many-instance-attributes
  42     """Abot EPC deployed with JUJU Orchestrator Case"""
  43
  44     __logger = logging.getLogger(__name__)
  45
  46     def __init__(self, **kwargs):
  47         if "case_name" not in kwargs:
  48             kwargs["case_name"] = "juju_epc"
  49         super(JujuEpc, self).__init__(**kwargs)
  50
  51         # Retrieve the configuration
  52         self.case_dir = pkg_resources.resource_filename(
  53             'functest', 'opnfv_tests/vnf/epc')
  54         try:
  55             self.config = CONST.__getattribute__(
  56                 'vnf_{}_config'.format(self.case_name))
  57         except Exception:
  58             raise Exception("VNF config file not found")
  59         config_file = os.path.join(self.case_dir, self.config)
  60         self.orchestrator = dict(
  61             requirements=get_config("orchestrator.requirements", config_file),
  62         )
  63
  64         self.created_object = []
  65         self.snaps_creds = ''
  66
  67         self.os_creds = openstack_tests.get_credentials(
  68             os_env_file=CONST.__getattribute__('openstack_creds'))
  69
  70         self.details['orchestrator'] = dict(
  71             name=get_config("orchestrator.name", config_file),
  72             version=get_config("orchestrator.version", config_file),
  73             status='ERROR',
  74             result=''
  75         )
  76
  77         self.vnf = dict(
  78             descriptor=get_config("vnf.descriptor", config_file),
  79             requirements=get_config("vnf.requirements", config_file)
  80         )
  81         self.details['vnf'] = dict(
  82             descriptor_version=self.vnf['descriptor']['version'],
  83             name=get_config("vnf.name", config_file),
  84             version=get_config("vnf.version", config_file),
  85         )
  86         self.__logger.debug("VNF configuration: %s", self.vnf)
  87
  88         self.details['test_vnf'] = dict(
  89             name=get_config("vnf_test_suite.name", config_file),
  90             version=get_config("vnf_test_suite.version", config_file),
  91             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  92         )
  93         self.images = get_config("tenant_images", config_file)
  94         self.__logger.info("Images needed for vEPC: %s", self.images)
  95         self.keystone_client = os_utils.get_keystone_client()
  96         self.glance_client = os_utils.get_glance_client()
  97         self.neutron_client = os_utils.get_neutron_client()
  98         self.nova_client = os_utils.get_nova_client()
  99         self.sec_group_id = None
 100         self.public_auth_url = None
 101         self.creds = None
 102         self.filename = None
 103
 104     def prepare(self):
 105         """Prepare testcase (Additional pre-configuration steps)."""
 106         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 107
 108         super(JujuEpc, self).prepare()
 109
 110         self.__logger.info("Additional pre-configuration steps")
 111         self.public_auth_url = keystone_utils.get_endpoint(
 112             self.snaps_creds, 'identity')
 113         # it enforces a versioned public identity endpoint as juju simply
 114         # adds /auth/tokens wich fails vs an unversioned endpoint.
 115         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 116             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 117
 118         self.creds = {
 119             "tenant": self.tenant_name,
 120             "username": self.tenant_name,
 121             "password": self.tenant_name,
 122             "auth_url": os_utils.get_credentials()['auth_url']
 123             }
 124
 125         self.snaps_creds = OSCreds(
 126             username=self.creds['username'],
 127             password=self.creds['password'],
 128             auth_url=self.creds['auth_url'],
 129             project_name=self.creds['tenant'],
 130             identity_api_version=int(os_utils.get_keystone_client_version()))
 131
 132         cloud_data = {
 133             'url': self.public_auth_url,
 134             'pass': self.tenant_name,
 135             'tenant_n': self.tenant_name,
 136             'user_n': self.tenant_name
 137         }
 138         self.__logger.info("Cloud DATA:  %s", cloud_data)
 139         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 140         self.__logger.info("Create  %s to add cloud info", self.filename)
 141         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 142
 143         if self.snaps_creds.identity_api_version == 3:
 144             append_config(self.filename, '{}'.format(
 145                 os_utils.get_credentials()['project_domain_name']),
 146                           '{}'.format(os_utils.get_credentials()
 147                                       ['user_domain_name']))
 148
 149         self.__logger.info("Upload some OS images if it doesn't exist")
 150         for image_name, image_file in self.images.iteritems():
 151             self.__logger.info("image: %s, file: %s", image_name, image_file)
 152             if image_file and image_name:
 153                 image_creator = OpenStackImage(
 154                     self.snaps_creds,
 155                     ImageSettings(name=image_name,
 156                                   image_user='cloud',
 157                                   img_format='qcow2',
 158                                   image_file=image_file))
 159                 image_creator.create()
 160                 self.created_object.append(image_creator)
 161
 162     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 163         """
 164         Create network, subnet, router
 165
 166         Bootstrap juju
 167         """
 168         self.__logger.info("Deployed Orchestrator")
 169         private_net_name = CONST.__getattribute__(
 170             'vnf_{}_private_net_name'.format(self.case_name))
 171         private_subnet_name = CONST.__getattribute__(
 172             'vnf_{}_private_subnet_name'.format(self.case_name))
 173         private_subnet_cidr = CONST.__getattribute__(
 174             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 175         abot_router = CONST.__getattribute__(
 176             'vnf_{}_external_router'.format(self.case_name))
 177         dns_nameserver = CONST.__getattribute__(
 178             'vnf_{}_dns_nameserver'.format(self.case_name))
 179         ext_net_name = CONST.__getattribute__(
 180             'vnf_{}_external_network_name'.format(self.case_name))
 181
 182         self.__logger.info("Creating full network ...")
 183         subnet_settings = SubnetSettings(name=private_subnet_name,
 184                                          cidr=private_subnet_cidr,
 185                                          dns_nameservers=dns_nameserver)
 186         network_settings = NetworkSettings(name=private_net_name,
 187                                            subnet_settings=[subnet_settings])
 188         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 189         network_creator.create()
 190         self.created_object.append(network_creator)
 191
 192         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 193         self.__logger.info("Creating network Router ....")
 194         router_creator = OpenStackRouter(
 195             self.snaps_creds,
 196             RouterSettings(
 197                 name=abot_router,
 198                 external_gateway=ext_net_name,
 199                 internal_subnets=[subnet_settings.name]))
 200         router_creator.create()
 201         self.created_object.append(router_creator)
 202         self.__logger.info("Creating Flavor ....")
 203         flavor_settings = FlavorSettings(
 204             name=self.orchestrator['requirements']['flavor']['name'],
 205             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 206             disk=10,
 207             vcpus=1)
 208         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 209         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 210         flavor_creator.create()
 211         self.created_object.append(flavor_creator)
 212         self.__logger.info("Installing Dependency Packages .......")
 213         source_dir = "/src/epc-requirements/juju_bin_build"
 214         if os.path.exists(source_dir):
 215             shutil.rmtree(source_dir)
 216         os.makedirs(source_dir)
 217         os.environ['GOPATH'] = str(source_dir)
 218         os.environ['GOBIN'] = str(source_dir) + "/bin"
 219         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 220                               (os.path.expandvars('$GOBIN')) + ":" +
 221                               (os.path.expandvars('$PATH')))
 222         os.system('go get -d -v github.com/juju/juju/...')
 223         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 224         os.system('git checkout tags/juju-2.2.5')
 225         os.system('go get github.com/rogpeppe/godeps')
 226         os.system('godeps -u dependencies.tsv')
 227         os.system('go install -v github.com/juju/juju/...')
 228         self.__logger.info("Creating Cloud for Abot-epc .....")
 229         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 230         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 231         for image_name in self.images.keys():
 232             self.__logger.info("Generating Metadata for %s", image_name)
 233             image_id = os_utils.get_image_id(self.glance_client, image_name)
 234             os.system('juju metadata generate-image -d ~ -i {} -s {} -r '
 235                       'RegionOne -u {}'.format(image_id,
 236                                                image_name,
 237                                                self.public_auth_url))
 238         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 239         self.__logger.info("Credential information  : %s", net_id)
 240         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 241                                   '--config network={} --metadata-source ~  '
 242                                   '--config ssl-hostname-verification=false '
 243                                   '--constraints mem=2G --bootstrap-series '
 244                                   'trusty '
 245                                   '--config use-floating-ip=true --debug'.
 246                                   format(net_id))
 247         os.system(juju_bootstrap_command)
 248         return True
 249
 250     def deploy_vnf(self):
 251         """Deploy ABOT-OAI-EPC."""
 252         self.__logger.info("Upload VNFD")
 253         descriptor = self.vnf['descriptor']
 254         self.__logger.info("Get or create flavor for all Abot-EPC")
 255         flavor_settings = FlavorSettings(
 256             name=self.vnf['requirements']['flavor']['name'],
 257             ram=self.vnf['requirements']['flavor']['ram_min'],
 258             disk=10,
 259             vcpus=1)
 260         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 261         flavor_creator.create()
 262         self.created_object.append(flavor_creator)
 263         self.__logger.info("Deploying Abot-epc bundle file ...")
 264         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 265         self.__logger.info("Waiting for instances .....")
 266         status = os.system('juju-wait')
 267         self.__logger.info("juju wait completed: %s", status)
 268         self.__logger.info("Deployed Abot-epc on Openstack")
 269         if status == 0:
 270             instances = os_utils.get_instances(self.nova_client)
 271             for items in instances:
 272                 metadata = get_instance_metadata(self.nova_client, items)
 273                 if 'juju-units-deployed' in metadata:
 274                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 275                                  '-' + metadata['juju-model-uuid'])
 276                     self.sec_group_id = os_utils.get_security_group_id(
 277                         self.neutron_client, sec_group)
 278                     break
 279             self.__logger.info("Adding Security group rule....")
 280             os_utils.create_secgroup_rule(self.neutron_client,
 281                                           self.sec_group_id, 'ingress', 132)
 282             self.__logger.info("Copying the feature files to Abot_node ")
 283             os.system('juju scp -- -r {}/featureFiles abot-'
 284                       'epc-basic/0:~/'.format(self.case_dir))
 285             self.__logger.info("Copying the feature files in Abot_node ")
 286             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 287                       "~/featureFiles /etc/rebaca-test-suite"
 288                       "/featureFiles'")
 289             count = 0
 290             while count < 10:
 291                 epcstatus = os.system('juju status oai-epc | '
 292                                       'grep {} | grep {} | grep {}'
 293                                       .format('EPC', 'is', 'running'))
 294                 if epcstatus == 0:
 295                     break
 296                 else:
 297                     time.sleep(60)
 298                     count = count + 1
 299             os.system('juju-wait')
 300             return True
 301         return False
 302
 303     def test_vnf(self):
 304         """Run test on ABoT."""
 305         start_time = time.time()
 306         self.__logger.info("Running VNF Test cases....")
 307         os.system('juju run-action abot-epc-basic/0 run '
 308                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 309         os.system('juju-wait')
 310         duration = time.time() - start_time
 311         self.__logger.info("Getting results from Abot node....")
 312         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 313                   'epc-basic/artifacts/TestResults.json {}/.'
 314                   .format(self.case_dir))
 315         self.__logger.info("Parsing the Test results...")
 316         res = (process_abot_test_result('{}/TestResults.'
 317                                         'json'.format(self.case_dir)))
 318         short_result = sig_test_format(res)
 319         self.__logger.info(short_result)
 320         self.details['test_vnf'].update(status='PASS',
 321                                         result=short_result,
 322                                         full_result=res,
 323                                         duration=duration)
 324
 325         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 326                            "%d, Skipped: %d", short_result['passed'],
 327                            short_result['failures'], short_result['skipped'])
 328         return True
 329
 330     def clean(self):
 331         """Clean created objects/functions."""
 332         try:
 333             if not self.orchestrator['requirements']['preserve_setup']:
 334                 self.__logger.info("Removing deployment files...")
 335                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 336                 if os.path.exists(testresult):
 337                     os.remove(testresult)
 338                 self.__logger.info("Removing %s file ", self.filename)
 339                 if os.path.exists(self.filename):
 340                     os.remove(self.filename)
 341                 self.__logger.info("Destroying Orchestrator...")
 342                 os.system('juju destroy-controller -y abot-controller '
 343                           '--destroy-all-models')
 344         except Exception:  # pylint: disable=broad-except
 345             self.__logger.warn("Some issue during the undeployment ..")
 346             self.__logger.warn("Tenant clean continue ..")
 347
 348         if not self.orchestrator['requirements']['preserve_setup']:
 349             self.__logger.info('Remove the Abot_epc OS object ..')
 350             for creator in reversed(self.created_object):
 351                 try:
 352                     creator.clean()
 353                 except Exception as exc:  # pylint: disable=broad-except
 354                     self.__logger.error('Unexpected error cleaning - %s', exc)
 355
 356             self.__logger.info("Releasing all the floating IPs")
 357             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 358             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 359                                                self.tenant_name)
 360             self.__logger.info("TENANT ID : %s", tenant_id)
 361             for item in floating_ips:
 362                 if item['tenant_id'] == tenant_id:
 363                     os_utils.delete_floating_ip(self.neutron_client,
 364                                                 item['id'])
 365             self.__logger.info("Cleaning Projects and Users")
 366             for creator in reversed(self.created_object):
 367                 try:
 368                     creator.clean()
 369                 except Exception as exc:  # pylint: disable=broad-except
 370                     self.__logger.error('Unexpected error cleaning - %s', exc)
 371         return True
 372
 373
 374 # ----------------------------------------------------------
 375 #
 376 #               YAML UTILS
 377 #
 378 # -----------------------------------------------------------
 379 def get_config(parameter, file_path):
 380     """
 381     Returns the value of a given parameter in file.yaml
 382     parameter must be given in string format with dots
 383     Example: general.openstack.image_name
 384     """
 385     with open(file_path) as config_file:
 386         file_yaml = yaml.safe_load(config_file)
 387     config_file.close()
 388     value = file_yaml
 389     for element in parameter.split("."):
 390         value = value.get(element)
 391         if value is None:
 392             raise ValueError("The parameter %s is not defined in"
 393                              " reporting.yaml" % parameter)
 394     return value
 395
 396
 397 def sig_test_format(sig_test):
 398     """
 399     Process the signaling result to have a short result
 400     """
 401     nb_passed = 0
 402     nb_failures = 0
 403     nb_skipped = 0
 404     for data_test in sig_test:
 405         if data_test['result'] == "passed":
 406             nb_passed += 1
 407         elif data_test['result'] == "failed":
 408             nb_failures += 1
 409         elif data_test['result'] == "skipped":
 410             nb_skipped += 1
 411     total_sig_test_result = {}
 412     total_sig_test_result['passed'] = nb_passed
 413     total_sig_test_result['failures'] = nb_failures
 414     total_sig_test_result['skipped'] = nb_skipped
 415     return total_sig_test_result
 416
 417
 418 def process_abot_test_result(file_path):
 419     """ Process ABoT Result """
 420     with open(file_path) as test_result:
 421         data = json.load(test_result)
 422         res = []
 423         for tests in data:
 424             tests = update_data(tests)
 425             try:
 426                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 427                 for steps in flatten_steps:
 428                     steps['result'] = steps['step_status']
 429                     res.append(steps)
 430             except:
 431                 logging.error("Could not post data to ElasticSearch host")
 432                 raise
 433         return res
 434
 435
 436 def update_data(obj):
 437     """ Update Result data"""
 438     try:
 439         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 440
 441         for element in obj['elements']:
 442             element['final_result'] = "passed"
 443             element['flatten_steps'] = []
 444
 445             for step in element['steps']:
 446                 temp_dict = {}
 447                 step['result'][step['result']['status']] = 1
 448                 if step['result']['status'].lower() in ['fail', 'failed']:
 449                     element['final_result'] = "failed"
 450
 451                 temp_dict['feature_file'] = obj['feature_file']
 452                 temp_dict['step_name'] = step['name']
 453                 temp_dict['step_status'] = step['result']['status']
 454                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 455                 temp_dict['step_' + step['result']['status']] = 1
 456                 element['flatten_steps'].append(deepcopy(temp_dict))
 457
 458             # Need to put the tag in OBJ and not ELEMENT
 459             if 'tags' in obj:
 460                 element['tags'] = deepcopy(obj['tags'])
 461                 for tag in obj['tags']:
 462                     element[tag['name']] = 1
 463             else:
 464                 for tag in element['tags']:
 465                     element[tag['name']] = 1
 466
 467     except Exception:  # pylint: disable=broad-except
 468         logging.error("Error in updating data, %s", sys.exc_info()[0])
 469         raise
 470
 471     return obj
 472
 473
 474 def get_instance_metadata(nova_client, instance):
 475     """ Get instance Metadata - Instance ID """
 476     try:
 477         instance = nova_client.servers.get(instance.id)
 478         return instance.metadata
 479     except Exception as exc:  # pylint: disable=broad-except
 480         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 481         return None
 482
 483
 484 CLOUD_TEMPLATE = """clouds:
 485     abot-epc:
 486       type: openstack
 487       auth-types: [userpass]
 488       endpoint: {url}
 489       regions:
 490         RegionOne:
 491           endpoint: {url}
 492 credentials:
 493   abot-epc:
 494     abot-epc:
 495       auth-type: userpass
 496       password: {pass}
 497       tenant-name: {tenant_n}
 498       username: {user_n}"""
 499
 500
 501 def write_config(fname, template, **kwargs):
 502     """ Generate yaml from template for addinh cloud in juju """
 503     with open(fname, 'w') as yfile:
 504         yfile.write(template.format(**kwargs))
 505
 506
 507 def append_config(file_name, p_domain, u_domain):
 508     """ Append values into a yaml file  """
 509     with open(file_name) as yfile:
 510         doc = yaml.load(yfile)
 511     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 512         p_domain)
 513     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 514         u_domain)
 515
 516     with open(file_name, 'w') as yfile:
 517         yaml.safe_dump(doc, yfile, default_flow_style=False)