functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 from urlparse import urljoin
  19
  20 import functest.core.vnf as vnf
  21 from functest.opnfv_tests.openstack.snaps import snaps_utils
  22 from functest.utils.constants import CONST
  23 import functest.utils.openstack_utils as os_utils
  24
  25 import pkg_resources
  26 from snaps.config.flavor import FlavorConfig
  27 from snaps.config.image import ImageConfig
  28 from snaps.config.network import NetworkConfig, SubnetConfig
  29 from snaps.config.router import RouterConfig
  30 from snaps.openstack.create_flavor import OpenStackFlavor
  31 from snaps.openstack.create_image import OpenStackImage
  32 from snaps.openstack.create_network import OpenStackNetwork
  33 from snaps.openstack.create_router import OpenStackRouter
  34 from snaps.openstack.utils import keystone_utils
  35 import yaml
  36
  37 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  38 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  39
  40
  41 class JujuEpc(vnf.VnfOnBoarding):
  42     # pylint:disable=too-many-instance-attributes
  43     """Abot EPC deployed with JUJU Orchestrator Case"""
  44
  45     __logger = logging.getLogger(__name__)
  46
  47     default_region_name = "RegionOne"
  48
  49     def __init__(self, **kwargs):
  50         if "case_name" not in kwargs:
  51             kwargs["case_name"] = "juju_epc"
  52         super(JujuEpc, self).__init__(**kwargs)
  53
  54         # Retrieve the configuration
  55         self.case_dir = pkg_resources.resource_filename(
  56             'functest', 'opnfv_tests/vnf/epc')
  57         try:
  58             self.config = CONST.__getattribute__(
  59                 'vnf_{}_config'.format(self.case_name))
  60         except Exception:
  61             raise Exception("VNF config file not found")
  62         config_file = os.path.join(self.case_dir, self.config)
  63         self.orchestrator = dict(
  64             requirements=get_config("orchestrator.requirements", config_file),
  65         )
  66
  67         self.created_object = []
  68         self.snaps_creds = snaps_utils.get_credentials()
  69         self.details['orchestrator'] = dict(
  70             name=get_config("orchestrator.name", config_file),
  71             version=get_config("orchestrator.version", config_file),
  72             status='ERROR',
  73             result=''
  74         )
  75
  76         self.vnf = dict(
  77             descriptor=get_config("vnf.descriptor", config_file),
  78             requirements=get_config("vnf.requirements", config_file)
  79         )
  80         self.details['vnf'] = dict(
  81             descriptor_version=self.vnf['descriptor']['version'],
  82             name=get_config("vnf.name", config_file),
  83             version=get_config("vnf.version", config_file),
  84         )
  85         self.__logger.debug("VNF configuration: %s", self.vnf)
  86
  87         self.details['test_vnf'] = dict(
  88             name=get_config("vnf_test_suite.name", config_file),
  89             version=get_config("vnf_test_suite.version", config_file),
  90             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  91         )
  92         self.images = get_config("tenant_images", config_file)
  93         self.__logger.info("Images needed for vEPC: %s", self.images)
  94         self.keystone_client = os_utils.get_keystone_client()
  95         self.glance_client = os_utils.get_glance_client()
  96         self.neutron_client = os_utils.get_neutron_client()
  97         self.nova_client = os_utils.get_nova_client()
  98         self.sec_group_id = None
  99         self.public_auth_url = None
 100         self.creds = None
 101         self.filename = None
 102
 103     def prepare(self):
 104         """Prepare testcase (Additional pre-configuration steps)."""
 105         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 106
 107         super(JujuEpc, self).prepare()
 108
 109         self.__logger.info("Additional pre-configuration steps")
 110         self.public_auth_url = keystone_utils.get_endpoint(
 111             self.snaps_creds, 'identity')
 112         # it enforces a versioned public identity endpoint as juju simply
 113         # adds /auth/tokens wich fails vs an unversioned endpoint.
 114         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 115             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 116
 117         self.creds = {
 118             "tenant": self.tenant_name,
 119             "username": self.tenant_name,
 120             "password": self.tenant_name,
 121             "auth_url": os_utils.get_credentials()['auth_url']
 122             }
 123
 124         cloud_data = {
 125             'url': self.public_auth_url,
 126             'pass': self.tenant_name,
 127             'tenant_n': self.tenant_name,
 128             'user_n': self.tenant_name,
 129             'region': os.environ.get(
 130                 "OS_REGION_NAME", self.default_region_name)
 131         }
 132         self.__logger.info("Cloud DATA:  %s", cloud_data)
 133         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 134         self.__logger.info("Create  %s to add cloud info", self.filename)
 135         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 136
 137         if self.snaps_creds.identity_api_version == 3:
 138             append_config(
 139                 self.filename, '{}'.format(
 140                     os_utils.get_credentials()['project_domain_name']),
 141                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 142
 143         self.__logger.info("Upload some OS images if it doesn't exist")
 144         for image_name, image_file in self.images.iteritems():
 145             self.__logger.info("image: %s, file: %s", image_name, image_file)
 146             if image_file and image_name:
 147                 image_creator = OpenStackImage(
 148                     self.snaps_creds,
 149                     ImageConfig(name=image_name,
 150                                 image_user='cloud',
 151                                 img_format='qcow2',
 152                                 image_file=image_file))
 153                 image_creator.create()
 154                 self.created_object.append(image_creator)
 155
 156     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 157         """
 158         Create network, subnet, router
 159
 160         Bootstrap juju
 161         """
 162         self.__logger.info("Deployed Orchestrator")
 163         private_net_name = CONST.__getattribute__(
 164             'vnf_{}_private_net_name'.format(self.case_name))
 165         private_subnet_name = CONST.__getattribute__(
 166             'vnf_{}_private_subnet_name'.format(self.case_name))
 167         private_subnet_cidr = CONST.__getattribute__(
 168             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 169         abot_router = CONST.__getattribute__(
 170             'vnf_{}_external_router'.format(self.case_name))
 171         dns_nameserver = CONST.__getattribute__(
 172             'vnf_{}_dns_nameserver'.format(self.case_name))
 173         ext_net_name = CONST.__getattribute__(
 174             'vnf_{}_external_network_name'.format(self.case_name))
 175
 176         self.__logger.info("Creating full network ...")
 177         subnet_settings = SubnetConfig(name=private_subnet_name,
 178                                        cidr=private_subnet_cidr,
 179                                        dns_nameservers=dns_nameserver)
 180         network_settings = NetworkConfig(name=private_net_name,
 181                                          subnet_settings=[subnet_settings])
 182         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 183         network_creator.create()
 184         self.created_object.append(network_creator)
 185
 186         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 187         self.__logger.info("Creating network Router ....")
 188         router_creator = OpenStackRouter(
 189             self.snaps_creds,
 190             RouterConfig(
 191                 name=abot_router,
 192                 external_gateway=ext_net_name,
 193                 internal_subnets=[subnet_settings.name]))
 194         router_creator.create()
 195         self.created_object.append(router_creator)
 196         self.__logger.info("Creating Flavor ....")
 197         flavor_settings = FlavorConfig(
 198             name=self.orchestrator['requirements']['flavor']['name'],
 199             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 200             disk=10,
 201             vcpus=1)
 202         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 203         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 204         flavor_creator.create()
 205         self.created_object.append(flavor_creator)
 206         self.__logger.info("Installing Dependency Packages .......")
 207         source_dir = "/src/epc-requirements/juju_bin_build"
 208         if os.path.exists(source_dir):
 209             shutil.rmtree(source_dir)
 210         os.makedirs(source_dir)
 211         os.environ['GOPATH'] = str(source_dir)
 212         os.environ['GOBIN'] = str(source_dir) + "/bin"
 213         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 214                               (os.path.expandvars('$GOBIN')) + ":" +
 215                               (os.path.expandvars('$PATH')))
 216         os.system('go get -d -v github.com/juju/juju/...')
 217         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 218         os.system('git checkout tags/juju-2.2.5')
 219         os.system('go get github.com/rogpeppe/godeps')
 220         os.system('godeps -u dependencies.tsv')
 221         os.system('go install -v github.com/juju/juju/...')
 222         self.__logger.info("Creating Cloud for Abot-epc .....")
 223         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 224         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 225         for image_name in self.images.keys():
 226             self.__logger.info("Generating Metadata for %s", image_name)
 227             image_id = os_utils.get_image_id(self.glance_client, image_name)
 228             os.system(
 229                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 230                 '{} -u {}'.format(
 231                     image_id, image_name,
 232                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 233                     self.public_auth_url))
 234         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 235         self.__logger.info("Credential information  : %s", net_id)
 236         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 237                                   '--config network={} --metadata-source ~  '
 238                                   '--config ssl-hostname-verification=false '
 239                                   '--constraints mem=2G --bootstrap-series '
 240                                   'xenial '
 241                                   '--config use-floating-ip=true --debug'.
 242                                   format(net_id))
 243         os.system(juju_bootstrap_command)
 244         return True
 245
 246     def deploy_vnf(self):
 247         """Deploy ABOT-OAI-EPC."""
 248         self.__logger.info("Upload VNFD")
 249         descriptor = self.vnf['descriptor']
 250         self.__logger.info("Get or create flavor for all Abot-EPC")
 251         flavor_settings = FlavorConfig(
 252             name=self.vnf['requirements']['flavor']['name'],
 253             ram=self.vnf['requirements']['flavor']['ram_min'],
 254             disk=10,
 255             vcpus=1)
 256         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 257         flavor_creator.create()
 258         self.created_object.append(flavor_creator)
 259         self.__logger.info("Deploying Abot-epc bundle file ...")
 260         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 261         self.__logger.info("Waiting for instances .....")
 262         status = os.system('juju-wait')
 263         self.__logger.info("juju wait completed: %s", status)
 264         self.__logger.info("Deployed Abot-epc on Openstack")
 265         if status == 0:
 266             instances = os_utils.get_instances(self.nova_client)
 267             for items in instances:
 268                 metadata = get_instance_metadata(self.nova_client, items)
 269                 if 'juju-units-deployed' in metadata:
 270                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 271                                  '-' + metadata['juju-model-uuid'])
 272                     self.sec_group_id = os_utils.get_security_group_id(
 273                         self.neutron_client, sec_group)
 274                     break
 275             self.__logger.info("Adding Security group rule....")
 276             os_utils.create_secgroup_rule(self.neutron_client,
 277                                           self.sec_group_id, 'ingress', 132)
 278             self.__logger.info("Copying the feature files to Abot_node ")
 279             os.system('juju scp -- -r {}/featureFiles abot-'
 280                       'epc-basic/0:~/'.format(self.case_dir))
 281             self.__logger.info("Copying the feature files in Abot_node ")
 282             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 283                       "~/featureFiles /etc/rebaca-test-suite"
 284                       "/featureFiles'")
 285             count = 0
 286             while count < 10:
 287                 epcstatus = os.system('juju status oai-epc | '
 288                                       'grep {} | grep {} | grep {}'
 289                                       .format('EPC', 'is', 'running'))
 290                 if epcstatus == 0:
 291                     break
 292                 else:
 293                     time.sleep(60)
 294                     count = count + 1
 295             os.system('juju-wait')
 296             return True
 297         return False
 298
 299     def test_vnf(self):
 300         """Run test on ABoT."""
 301         start_time = time.time()
 302         self.__logger.info("Running VNF Test cases....")
 303         os.system('juju run-action abot-epc-basic/0 run '
 304                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 305         os.system('juju-wait')
 306         duration = time.time() - start_time
 307         self.__logger.info("Getting results from Abot node....")
 308         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 309                   'epc-basic/artifacts/TestResults.json {}/.'
 310                   .format(self.case_dir))
 311         self.__logger.info("Parsing the Test results...")
 312         res = (process_abot_test_result('{}/TestResults.'
 313                                         'json'.format(self.case_dir)))
 314         short_result = sig_test_format(res)
 315         self.__logger.info(short_result)
 316         self.details['test_vnf'].update(status='PASS',
 317                                         result=short_result,
 318                                         full_result=res,
 319                                         duration=duration)
 320
 321         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 322                            "%d, Skipped: %d", short_result['passed'],
 323                            short_result['failures'], short_result['skipped'])
 324         return True
 325
 326     def clean(self):
 327         """Clean created objects/functions."""
 328         try:
 329             if not self.orchestrator['requirements']['preserve_setup']:
 330                 self.__logger.info("Removing deployment files...")
 331                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 332                 if os.path.exists(testresult):
 333                     os.remove(testresult)
 334                 self.__logger.info("Removing %s file ", self.filename)
 335                 if os.path.exists(self.filename):
 336                     os.remove(self.filename)
 337                 self.__logger.info("Destroying Orchestrator...")
 338                 os.system('juju destroy-controller -y abot-controller '
 339                           '--destroy-all-models')
 340         except Exception:  # pylint: disable=broad-except
 341             self.__logger.warn("Some issue during the undeployment ..")
 342             self.__logger.warn("Tenant clean continue ..")
 343
 344         if not self.orchestrator['requirements']['preserve_setup']:
 345             self.__logger.info('Remove the Abot_epc OS object ..')
 346             for creator in reversed(self.created_object):
 347                 try:
 348                     creator.clean()
 349                 except Exception as exc:  # pylint: disable=broad-except
 350                     self.__logger.error('Unexpected error cleaning - %s', exc)
 351
 352             self.__logger.info("Releasing all the floating IPs")
 353             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 354             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 355                                                self.tenant_name)
 356             self.__logger.info("TENANT ID : %s", tenant_id)
 357             for item in floating_ips:
 358                 if item['tenant_id'] == tenant_id:
 359                     os_utils.delete_floating_ip(self.neutron_client,
 360                                                 item['id'])
 361             self.__logger.info("Cleaning Projects and Users")
 362             for creator in reversed(self.created_object):
 363                 try:
 364                     creator.clean()
 365                 except Exception as exc:  # pylint: disable=broad-except
 366                     self.__logger.error('Unexpected error cleaning - %s', exc)
 367         return True
 368
 369
 370 # ----------------------------------------------------------
 371 #
 372 #               YAML UTILS
 373 #
 374 # -----------------------------------------------------------
 375 def get_config(parameter, file_path):
 376     """
 377     Returns the value of a given parameter in file.yaml
 378     parameter must be given in string format with dots
 379     Example: general.openstack.image_name
 380     """
 381     with open(file_path) as config_file:
 382         file_yaml = yaml.safe_load(config_file)
 383     config_file.close()
 384     value = file_yaml
 385     for element in parameter.split("."):
 386         value = value.get(element)
 387         if value is None:
 388             raise ValueError("The parameter %s is not defined in"
 389                              " reporting.yaml" % parameter)
 390     return value
 391
 392
 393 def sig_test_format(sig_test):
 394     """
 395     Process the signaling result to have a short result
 396     """
 397     nb_passed = 0
 398     nb_failures = 0
 399     nb_skipped = 0
 400     for data_test in sig_test:
 401         if data_test['result'] == "passed":
 402             nb_passed += 1
 403         elif data_test['result'] == "failed":
 404             nb_failures += 1
 405         elif data_test['result'] == "skipped":
 406             nb_skipped += 1
 407     total_sig_test_result = {}
 408     total_sig_test_result['passed'] = nb_passed
 409     total_sig_test_result['failures'] = nb_failures
 410     total_sig_test_result['skipped'] = nb_skipped
 411     return total_sig_test_result
 412
 413
 414 def process_abot_test_result(file_path):
 415     """ Process ABoT Result """
 416     with open(file_path) as test_result:
 417         data = json.load(test_result)
 418         res = []
 419         for tests in data:
 420             tests = update_data(tests)
 421             try:
 422                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 423                 for steps in flatten_steps:
 424                     steps['result'] = steps['step_status']
 425                     res.append(steps)
 426             except:
 427                 logging.error("Could not post data to ElasticSearch host")
 428                 raise
 429         return res
 430
 431
 432 def update_data(obj):
 433     """ Update Result data"""
 434     try:
 435         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 436
 437         for element in obj['elements']:
 438             element['final_result'] = "passed"
 439             element['flatten_steps'] = []
 440
 441             for step in element['steps']:
 442                 temp_dict = {}
 443                 step['result'][step['result']['status']] = 1
 444                 if step['result']['status'].lower() in ['fail', 'failed']:
 445                     element['final_result'] = "failed"
 446
 447                 temp_dict['feature_file'] = obj['feature_file']
 448                 temp_dict['step_name'] = step['name']
 449                 temp_dict['step_status'] = step['result']['status']
 450                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 451                 temp_dict['step_' + step['result']['status']] = 1
 452                 element['flatten_steps'].append(deepcopy(temp_dict))
 453
 454             # Need to put the tag in OBJ and not ELEMENT
 455             if 'tags' in obj:
 456                 element['tags'] = deepcopy(obj['tags'])
 457                 for tag in obj['tags']:
 458                     element[tag['name']] = 1
 459             else:
 460                 for tag in element['tags']:
 461                     element[tag['name']] = 1
 462
 463     except Exception:  # pylint: disable=broad-except
 464         logging.error("Error in updating data, %s", sys.exc_info()[0])
 465         raise
 466
 467     return obj
 468
 469
 470 def get_instance_metadata(nova_client, instance):
 471     """ Get instance Metadata - Instance ID """
 472     try:
 473         instance = nova_client.servers.get(instance.id)
 474         return instance.metadata
 475     except Exception as exc:  # pylint: disable=broad-except
 476         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 477         return None
 478
 479
 480 CLOUD_TEMPLATE = """clouds:
 481     abot-epc:
 482       type: openstack
 483       auth-types: [userpass]
 484       endpoint: {url}
 485       regions:
 486         {region}:
 487           endpoint: {url}
 488 credentials:
 489   abot-epc:
 490     abot-epc:
 491       auth-type: userpass
 492       password: {pass}
 493       tenant-name: {tenant_n}
 494       username: {user_n}"""
 495
 496
 497 def write_config(fname, template, **kwargs):
 498     """ Generate yaml from template for addinh cloud in juju """
 499     with open(fname, 'w') as yfile:
 500         yfile.write(template.format(**kwargs))
 501
 502
 503 def append_config(file_name, p_domain, u_domain):
 504     """ Append values into a yaml file  """
 505     with open(file_name) as yfile:
 506         doc = yaml.load(yfile)
 507     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 508         p_domain)
 509     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 510         u_domain)
 511
 512     with open(file_name, 'w') as yfile:
 513         yaml.safe_dump(doc, yfile, default_flow_style=False)