functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import os
  13 import time
  14 import json
  15 import sys
  16 from copy import deepcopy
  17 from urlparse import urljoin
  18
  19 import functest.core.vnf as vnf
  20 from functest.opnfv_tests.openstack.snaps import snaps_utils
  21 from functest.utils.constants import CONST
  22 import functest.utils.openstack_utils as os_utils
  23
  24 import pkg_resources
  25 from snaps.config.flavor import FlavorConfig
  26 from snaps.config.image import ImageConfig
  27 from snaps.config.network import NetworkConfig, SubnetConfig
  28 from snaps.config.router import RouterConfig
  29 from snaps.openstack.create_flavor import OpenStackFlavor
  30 from snaps.openstack.create_image import OpenStackImage
  31 from snaps.openstack.create_network import OpenStackNetwork
  32 from snaps.openstack.create_router import OpenStackRouter
  33 from snaps.openstack.utils import keystone_utils
  34 import yaml
  35
  36 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  37 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  38
  39
  40 class JujuEpc(vnf.VnfOnBoarding):
  41     # pylint:disable=too-many-instance-attributes
  42     """Abot EPC deployed with JUJU Orchestrator Case"""
  43
  44     __logger = logging.getLogger(__name__)
  45
  46     default_region_name = "RegionOne"
  47
  48     def __init__(self, **kwargs):
  49         if "case_name" not in kwargs:
  50             kwargs["case_name"] = "juju_epc"
  51         super(JujuEpc, self).__init__(**kwargs)
  52
  53         # Retrieve the configuration
  54         self.case_dir = pkg_resources.resource_filename(
  55             'functest', 'opnfv_tests/vnf/epc')
  56         try:
  57             self.config = CONST.__getattribute__(
  58                 'vnf_{}_config'.format(self.case_name))
  59         except Exception:
  60             raise Exception("VNF config file not found")
  61         config_file = os.path.join(self.case_dir, self.config)
  62         self.orchestrator = dict(
  63             requirements=get_config("orchestrator.requirements", config_file),
  64         )
  65
  66         self.created_object = []
  67         self.snaps_creds = snaps_utils.get_credentials()
  68         self.details['orchestrator'] = dict(
  69             name=get_config("orchestrator.name", config_file),
  70             version=get_config("orchestrator.version", config_file),
  71             status='ERROR',
  72             result=''
  73         )
  74
  75         self.vnf = dict(
  76             descriptor=get_config("vnf.descriptor", config_file),
  77             requirements=get_config("vnf.requirements", config_file)
  78         )
  79         self.details['vnf'] = dict(
  80             descriptor_version=self.vnf['descriptor']['version'],
  81             name=get_config("vnf.name", config_file),
  82             version=get_config("vnf.version", config_file),
  83         )
  84         self.__logger.debug("VNF configuration: %s", self.vnf)
  85
  86         self.details['test_vnf'] = dict(
  87             name=get_config("vnf_test_suite.name", config_file),
  88             version=get_config("vnf_test_suite.version", config_file),
  89             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  90         )
  91         self.images = get_config("tenant_images", config_file)
  92         self.__logger.info("Images needed for vEPC: %s", self.images)
  93         self.keystone_client = os_utils.get_keystone_client()
  94         self.glance_client = os_utils.get_glance_client()
  95         self.neutron_client = os_utils.get_neutron_client()
  96         self.nova_client = os_utils.get_nova_client()
  97         self.sec_group_id = None
  98         self.public_auth_url = None
  99         self.creds = None
 100         self.filename = None
 101
 102     def prepare(self):
 103         """Prepare testcase (Additional pre-configuration steps)."""
 104         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 105
 106         super(JujuEpc, self).prepare()
 107
 108         self.__logger.info("Additional pre-configuration steps")
 109         self.public_auth_url = keystone_utils.get_endpoint(
 110             self.snaps_creds, 'identity')
 111         # it enforces a versioned public identity endpoint as juju simply
 112         # adds /auth/tokens wich fails vs an unversioned endpoint.
 113         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 114             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 115
 116         self.creds = {
 117             "tenant": self.tenant_name,
 118             "username": self.tenant_name,
 119             "password": self.tenant_name,
 120             "auth_url": os_utils.get_credentials()['auth_url']
 121             }
 122
 123         cloud_data = {
 124             'url': self.public_auth_url,
 125             'pass': self.tenant_name,
 126             'tenant_n': self.tenant_name,
 127             'user_n': self.tenant_name,
 128             'region': os.environ.get(
 129                 "OS_REGION_NAME", self.default_region_name)
 130         }
 131         self.__logger.info("Cloud DATA:  %s", cloud_data)
 132         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 133         self.__logger.info("Create  %s to add cloud info", self.filename)
 134         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 135
 136         if self.snaps_creds.identity_api_version == 3:
 137             append_config(
 138                 self.filename, '{}'.format(
 139                     os_utils.get_credentials()['project_domain_name']),
 140                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 141
 142         self.__logger.info("Upload some OS images if it doesn't exist")
 143         for image_name, image_file in self.images.iteritems():
 144             self.__logger.info("image: %s, file: %s", image_name, image_file)
 145             if image_file and image_name:
 146                 image_creator = OpenStackImage(
 147                     self.snaps_creds,
 148                     ImageConfig(name=image_name,
 149                                 image_user='cloud',
 150                                 img_format='qcow2',
 151                                 image_file=image_file))
 152                 image_creator.create()
 153                 self.created_object.append(image_creator)
 154
 155     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 156         """
 157         Create network, subnet, router
 158
 159         Bootstrap juju
 160         """
 161         self.__logger.info("Deployed Orchestrator")
 162         private_net_name = CONST.__getattribute__(
 163             'vnf_{}_private_net_name'.format(self.case_name))
 164         private_subnet_name = CONST.__getattribute__(
 165             'vnf_{}_private_subnet_name'.format(self.case_name))
 166         private_subnet_cidr = CONST.__getattribute__(
 167             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 168         abot_router = CONST.__getattribute__(
 169             'vnf_{}_external_router'.format(self.case_name))
 170         dns_nameserver = CONST.__getattribute__(
 171             'vnf_{}_dns_nameserver'.format(self.case_name))
 172         ext_net_name = CONST.__getattribute__(
 173             'vnf_{}_external_network_name'.format(self.case_name))
 174
 175         self.__logger.info("Creating full network ...")
 176         subnet_settings = SubnetConfig(name=private_subnet_name,
 177                                        cidr=private_subnet_cidr,
 178                                        dns_nameservers=dns_nameserver)
 179         network_settings = NetworkConfig(name=private_net_name,
 180                                          subnet_settings=[subnet_settings])
 181         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 182         network_creator.create()
 183         self.created_object.append(network_creator)
 184
 185         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 186         self.__logger.info("Creating network Router ....")
 187         router_creator = OpenStackRouter(
 188             self.snaps_creds,
 189             RouterConfig(
 190                 name=abot_router,
 191                 external_gateway=ext_net_name,
 192                 internal_subnets=[subnet_settings.name]))
 193         router_creator.create()
 194         self.created_object.append(router_creator)
 195         self.__logger.info("Creating Flavor ....")
 196         flavor_settings = FlavorConfig(
 197             name=self.orchestrator['requirements']['flavor']['name'],
 198             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 199             disk=10,
 200             vcpus=1)
 201         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 202         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 203         flavor_creator.create()
 204         self.created_object.append(flavor_creator)
 205         self.__logger.info("Creating Cloud for Abot-epc .....")
 206         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 207         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 208         for image_name in self.images.keys():
 209             self.__logger.info("Generating Metadata for %s", image_name)
 210             image_id = os_utils.get_image_id(self.glance_client, image_name)
 211             os.system(
 212                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 213                 '{} -u {}'.format(
 214                     image_id, image_name,
 215                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 216                     self.public_auth_url))
 217         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 218         self.__logger.info("Credential information  : %s", net_id)
 219         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 220                                   '--config network={} --metadata-source ~  '
 221                                   '--config ssl-hostname-verification=false '
 222                                   '--constraints mem=2G --bootstrap-series '
 223                                   'xenial '
 224                                   '--config use-floating-ip=true --debug'.
 225                                   format(net_id))
 226         os.system(juju_bootstrap_command)
 227         return True
 228
 229     def deploy_vnf(self):
 230         """Deploy ABOT-OAI-EPC."""
 231         self.__logger.info("Upload VNFD")
 232         descriptor = self.vnf['descriptor']
 233         self.__logger.info("Get or create flavor for all Abot-EPC")
 234         flavor_settings = FlavorConfig(
 235             name=self.vnf['requirements']['flavor']['name'],
 236             ram=self.vnf['requirements']['flavor']['ram_min'],
 237             disk=10,
 238             vcpus=1)
 239         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 240         flavor_creator.create()
 241         self.created_object.append(flavor_creator)
 242         self.__logger.info("Deploying Abot-epc bundle file ...")
 243         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 244         self.__logger.info("Waiting for instances .....")
 245         status = os.system('juju-wait')
 246         self.__logger.info("juju wait completed: %s", status)
 247         self.__logger.info("Deployed Abot-epc on Openstack")
 248         if status == 0:
 249             instances = os_utils.get_instances(self.nova_client)
 250             for items in instances:
 251                 metadata = get_instance_metadata(self.nova_client, items)
 252                 if 'juju-units-deployed' in metadata:
 253                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 254                                  '-' + metadata['juju-model-uuid'])
 255                     self.sec_group_id = os_utils.get_security_group_id(
 256                         self.neutron_client, sec_group)
 257                     break
 258             self.__logger.info("Adding Security group rule....")
 259             os_utils.create_secgroup_rule(self.neutron_client,
 260                                           self.sec_group_id, 'ingress', 132)
 261             self.__logger.info("Copying the feature files to Abot_node ")
 262             os.system('juju scp -- -r {}/featureFiles abot-'
 263                       'epc-basic/0:~/'.format(self.case_dir))
 264             self.__logger.info("Copying the feature files in Abot_node ")
 265             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 266                       "~/featureFiles /etc/rebaca-test-suite"
 267                       "/featureFiles'")
 268             count = 0
 269             while count < 10:
 270                 epcstatus = os.system('juju status oai-epc | '
 271                                       'grep {} | grep {} | grep {}'
 272                                       .format('EPC', 'is', 'running'))
 273                 if epcstatus == 0:
 274                     break
 275                 else:
 276                     time.sleep(60)
 277                     count = count + 1
 278             os.system('juju-wait')
 279             return True
 280         return False
 281
 282     def test_vnf(self):
 283         """Run test on ABoT."""
 284         start_time = time.time()
 285         self.__logger.info("Running VNF Test cases....")
 286         os.system('juju run-action abot-epc-basic/0 run '
 287                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 288         os.system('juju-wait')
 289         duration = time.time() - start_time
 290         self.__logger.info("Getting results from Abot node....")
 291         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 292                   'epc-basic/artifacts/TestResults.json {}/.'
 293                   .format(self.case_dir))
 294         self.__logger.info("Parsing the Test results...")
 295         res = (process_abot_test_result('{}/TestResults.'
 296                                         'json'.format(self.case_dir)))
 297         short_result = sig_test_format(res)
 298         self.__logger.info(short_result)
 299         self.details['test_vnf'].update(status='PASS',
 300                                         result=short_result,
 301                                         full_result=res,
 302                                         duration=duration)
 303
 304         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 305                            "%d, Skipped: %d", short_result['passed'],
 306                            short_result['failures'], short_result['skipped'])
 307         return True
 308
 309     def clean(self):
 310         """Clean created objects/functions."""
 311         try:
 312             if not self.orchestrator['requirements']['preserve_setup']:
 313                 self.__logger.info("Removing deployment files...")
 314                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 315                 if os.path.exists(testresult):
 316                     os.remove(testresult)
 317                 self.__logger.info("Removing %s file ", self.filename)
 318                 if os.path.exists(self.filename):
 319                     os.remove(self.filename)
 320                 self.__logger.info("Destroying Orchestrator...")
 321                 os.system('juju destroy-controller -y abot-controller '
 322                           '--destroy-all-models')
 323         except Exception:  # pylint: disable=broad-except
 324             self.__logger.warn("Some issue during the undeployment ..")
 325             self.__logger.warn("Tenant clean continue ..")
 326
 327         if not self.orchestrator['requirements']['preserve_setup']:
 328             self.__logger.info('Remove the Abot_epc OS object ..')
 329             for creator in reversed(self.created_object):
 330                 try:
 331                     creator.clean()
 332                 except Exception as exc:  # pylint: disable=broad-except
 333                     self.__logger.error('Unexpected error cleaning - %s', exc)
 334
 335             self.__logger.info("Releasing all the floating IPs")
 336             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 337             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 338                                                self.tenant_name)
 339             self.__logger.info("TENANT ID : %s", tenant_id)
 340             for item in floating_ips:
 341                 if item['tenant_id'] == tenant_id:
 342                     os_utils.delete_floating_ip(self.neutron_client,
 343                                                 item['id'])
 344             self.__logger.info("Cleaning Projects and Users")
 345             for creator in reversed(self.created_object):
 346                 try:
 347                     creator.clean()
 348                 except Exception as exc:  # pylint: disable=broad-except
 349                     self.__logger.error('Unexpected error cleaning - %s', exc)
 350         return True
 351
 352
 353 # ----------------------------------------------------------
 354 #
 355 #               YAML UTILS
 356 #
 357 # -----------------------------------------------------------
 358 def get_config(parameter, file_path):
 359     """
 360     Returns the value of a given parameter in file.yaml
 361     parameter must be given in string format with dots
 362     Example: general.openstack.image_name
 363     """
 364     with open(file_path) as config_file:
 365         file_yaml = yaml.safe_load(config_file)
 366     config_file.close()
 367     value = file_yaml
 368     for element in parameter.split("."):
 369         value = value.get(element)
 370         if value is None:
 371             raise ValueError("The parameter %s is not defined in"
 372                              " reporting.yaml" % parameter)
 373     return value
 374
 375
 376 def sig_test_format(sig_test):
 377     """
 378     Process the signaling result to have a short result
 379     """
 380     nb_passed = 0
 381     nb_failures = 0
 382     nb_skipped = 0
 383     for data_test in sig_test:
 384         if data_test['result'] == "passed":
 385             nb_passed += 1
 386         elif data_test['result'] == "failed":
 387             nb_failures += 1
 388         elif data_test['result'] == "skipped":
 389             nb_skipped += 1
 390     total_sig_test_result = {}
 391     total_sig_test_result['passed'] = nb_passed
 392     total_sig_test_result['failures'] = nb_failures
 393     total_sig_test_result['skipped'] = nb_skipped
 394     return total_sig_test_result
 395
 396
 397 def process_abot_test_result(file_path):
 398     """ Process ABoT Result """
 399     with open(file_path) as test_result:
 400         data = json.load(test_result)
 401         res = []
 402         for tests in data:
 403             tests = update_data(tests)
 404             try:
 405                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 406                 for steps in flatten_steps:
 407                     steps['result'] = steps['step_status']
 408                     res.append(steps)
 409             except:
 410                 logging.error("Could not post data to ElasticSearch host")
 411                 raise
 412         return res
 413
 414
 415 def update_data(obj):
 416     """ Update Result data"""
 417     try:
 418         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 419
 420         for element in obj['elements']:
 421             element['final_result'] = "passed"
 422             element['flatten_steps'] = []
 423
 424             for step in element['steps']:
 425                 temp_dict = {}
 426                 step['result'][step['result']['status']] = 1
 427                 if step['result']['status'].lower() in ['fail', 'failed']:
 428                     element['final_result'] = "failed"
 429
 430                 temp_dict['feature_file'] = obj['feature_file']
 431                 temp_dict['step_name'] = step['name']
 432                 temp_dict['step_status'] = step['result']['status']
 433                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 434                 temp_dict['step_' + step['result']['status']] = 1
 435                 element['flatten_steps'].append(deepcopy(temp_dict))
 436
 437             # Need to put the tag in OBJ and not ELEMENT
 438             if 'tags' in obj:
 439                 element['tags'] = deepcopy(obj['tags'])
 440                 for tag in obj['tags']:
 441                     element[tag['name']] = 1
 442             else:
 443                 for tag in element['tags']:
 444                     element[tag['name']] = 1
 445
 446     except Exception:  # pylint: disable=broad-except
 447         logging.error("Error in updating data, %s", sys.exc_info()[0])
 448         raise
 449
 450     return obj
 451
 452
 453 def get_instance_metadata(nova_client, instance):
 454     """ Get instance Metadata - Instance ID """
 455     try:
 456         instance = nova_client.servers.get(instance.id)
 457         return instance.metadata
 458     except Exception as exc:  # pylint: disable=broad-except
 459         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 460         return None
 461
 462
 463 CLOUD_TEMPLATE = """clouds:
 464     abot-epc:
 465       type: openstack
 466       auth-types: [userpass]
 467       endpoint: {url}
 468       regions:
 469         {region}:
 470           endpoint: {url}
 471 credentials:
 472   abot-epc:
 473     abot-epc:
 474       auth-type: userpass
 475       password: {pass}
 476       tenant-name: {tenant_n}
 477       username: {user_n}"""
 478
 479
 480 def write_config(fname, template, **kwargs):
 481     """ Generate yaml from template for addinh cloud in juju """
 482     with open(fname, 'w') as yfile:
 483         yfile.write(template.format(**kwargs))
 484
 485
 486 def append_config(file_name, p_domain, u_domain):
 487     """ Append values into a yaml file  """
 488     with open(file_name) as yfile:
 489         doc = yaml.load(yfile)
 490     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 491         p_domain)
 492     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 493         u_domain)
 494
 495     with open(file_name, 'w') as yfile:
 496         yaml.safe_dump(doc, yfile, default_flow_style=False)