functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import os
  13 import time
  14 import json
  15 import sys
  16 from copy import deepcopy
  17 from urlparse import urljoin
  18
  19 from functest.core import vnf
  20 from functest.opnfv_tests.openstack.snaps import snaps_utils
  21 from functest.utils.constants import CONST
  22 import functest.utils.openstack_utils as os_utils
  23
  24 import pkg_resources
  25 from snaps.config.flavor import FlavorConfig
  26 from snaps.config.image import ImageConfig
  27 from snaps.config.network import NetworkConfig, SubnetConfig
  28 from snaps.config.router import RouterConfig
  29 from snaps.openstack.create_flavor import OpenStackFlavor
  30 from snaps.openstack.create_image import OpenStackImage
  31 from snaps.openstack.create_network import OpenStackNetwork
  32 from snaps.openstack.create_router import OpenStackRouter
  33 from snaps.openstack.utils import keystone_utils
  34 import yaml
  35
  36 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  37 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  38
  39
  40 class JujuEpc(vnf.VnfOnBoarding):
  41     # pylint:disable=too-many-instance-attributes
  42     """Abot EPC deployed with JUJU Orchestrator Case"""
  43
  44     __logger = logging.getLogger(__name__)
  45
  46     default_region_name = "RegionOne"
  47
  48     def __init__(self, **kwargs):
  49         if "case_name" not in kwargs:
  50             kwargs["case_name"] = "juju_epc"
  51         super(JujuEpc, self).__init__(**kwargs)
  52
  53         # Retrieve the configuration
  54         self.case_dir = pkg_resources.resource_filename(
  55             'functest', 'opnfv_tests/vnf/epc')
  56         try:
  57             self.config = CONST.__getattribute__(
  58                 'vnf_{}_config'.format(self.case_name))
  59         except Exception:
  60             raise Exception("VNF config file not found")
  61         config_file = os.path.join(self.case_dir, self.config)
  62         self.orchestrator = dict(
  63             requirements=get_config("orchestrator.requirements", config_file),
  64         )
  65
  66         self.created_object = []
  67         self.snaps_creds = snaps_utils.get_credentials()
  68         self.details['orchestrator'] = dict(
  69             name=get_config("orchestrator.name", config_file),
  70             version=get_config("orchestrator.version", config_file),
  71             status='ERROR',
  72             result=''
  73         )
  74
  75         self.vnf = dict(
  76             descriptor=get_config("vnf.descriptor", config_file),
  77             requirements=get_config("vnf.requirements", config_file)
  78         )
  79         self.details['vnf'] = dict(
  80             descriptor_version=self.vnf['descriptor']['version'],
  81             name=get_config("vnf.name", config_file),
  82             version=get_config("vnf.version", config_file),
  83         )
  84         self.__logger.debug("VNF configuration: %s", self.vnf)
  85
  86         self.details['test_vnf'] = dict(
  87             name=get_config("vnf_test_suite.name", config_file),
  88             version=get_config("vnf_test_suite.version", config_file),
  89             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  90         )
  91         self.images = get_config("tenant_images", config_file)
  92         self.__logger.info("Images needed for vEPC: %s", self.images)
  93         self.keystone_client = os_utils.get_keystone_client()
  94         self.glance_client = os_utils.get_glance_client()
  95         self.neutron_client = os_utils.get_neutron_client()
  96         self.nova_client = os_utils.get_nova_client()
  97         self.sec_group_id = None
  98         self.public_auth_url = None
  99         self.creds = None
 100         self.filename = None
 101
 102     def prepare(self):
 103         """Prepare testcase (Additional pre-configuration steps)."""
 104         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 105
 106         super(JujuEpc, self).prepare()
 107
 108         self.__logger.info("Additional pre-configuration steps")
 109         self.public_auth_url = keystone_utils.get_endpoint(
 110             self.snaps_creds, 'identity')
 111         # it enforces a versioned public identity endpoint as juju simply
 112         # adds /auth/tokens wich fails vs an unversioned endpoint.
 113         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 114             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 115
 116         self.creds = {
 117             "tenant": self.tenant_name,
 118             "username": self.tenant_name,
 119             "password": self.tenant_name,
 120             "auth_url": os_utils.get_credentials()['auth_url']
 121             }
 122
 123         cloud_data = {
 124             'url': self.public_auth_url,
 125             'pass': self.tenant_name,
 126             'tenant_n': self.tenant_name,
 127             'user_n': self.tenant_name,
 128             'region': os.environ.get(
 129                 "OS_REGION_NAME", self.default_region_name)
 130         }
 131         self.__logger.info("Cloud DATA:  %s", cloud_data)
 132         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 133         self.__logger.info("Create  %s to add cloud info", self.filename)
 134         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 135
 136         if self.snaps_creds.identity_api_version == 3:
 137             append_config(
 138                 self.filename, '{}'.format(
 139                     os_utils.get_credentials()['project_domain_name']),
 140                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 141
 142         self.__logger.info("Upload some OS images if it doesn't exist")
 143         for image_name, image_file in self.images.iteritems():
 144             self.__logger.info("image: %s, file: %s", image_name, image_file)
 145             if image_file and image_name:
 146                 image_creator = OpenStackImage(
 147                     self.snaps_creds,
 148                     ImageConfig(name=image_name,
 149                                 image_user='cloud',
 150                                 img_format='qcow2',
 151                                 image_file=image_file))
 152                 image_creator.create()
 153                 self.created_object.append(image_creator)
 154
 155     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 156         """
 157         Create network, subnet, router
 158
 159         Bootstrap juju
 160         """
 161         self.__logger.info("Deployed Orchestrator")
 162         private_net_name = CONST.__getattribute__(
 163             'vnf_{}_private_net_name'.format(self.case_name))
 164         private_subnet_name = CONST.__getattribute__(
 165             'vnf_{}_private_subnet_name'.format(self.case_name))
 166         private_subnet_cidr = CONST.__getattribute__(
 167             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 168         abot_router = CONST.__getattribute__(
 169             'vnf_{}_external_router'.format(self.case_name))
 170         dns_nameserver = CONST.__getattribute__(
 171             'vnf_{}_dns_nameserver'.format(self.case_name))
 172
 173         self.__logger.info("Creating full network ...")
 174         subnet_settings = SubnetConfig(name=private_subnet_name,
 175                                        cidr=private_subnet_cidr,
 176                                        dns_nameservers=dns_nameserver)
 177         network_settings = NetworkConfig(name=private_net_name,
 178                                          subnet_settings=[subnet_settings])
 179         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 180         network_creator.create()
 181         self.created_object.append(network_creator)
 182
 183         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 184         self.__logger.info("Creating network Router ....")
 185         router_creator = OpenStackRouter(
 186             self.snaps_creds,
 187             RouterConfig(
 188                 name=abot_router,
 189                 external_gateway=ext_net_name,
 190                 internal_subnets=[subnet_settings.name]))
 191         router_creator.create()
 192         self.created_object.append(router_creator)
 193         self.__logger.info("Creating Flavor ....")
 194         flavor_settings = FlavorConfig(
 195             name=self.orchestrator['requirements']['flavor']['name'],
 196             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 197             disk=10,
 198             vcpus=1)
 199         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 200         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 201         flavor_creator.create()
 202         self.created_object.append(flavor_creator)
 203         self.__logger.info("Creating Cloud for Abot-epc .....")
 204         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 205         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 206         for image_name in self.images.keys():
 207             self.__logger.info("Generating Metadata for %s", image_name)
 208             image_id = os_utils.get_image_id(self.glance_client, image_name)
 209             os.system(
 210                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 211                 '{} -u {}'.format(
 212                     image_id, image_name,
 213                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 214                     self.public_auth_url))
 215         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 216         self.__logger.info("Credential information  : %s", net_id)
 217         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 218                                   '--config network={} --metadata-source ~  '
 219                                   '--config ssl-hostname-verification=false '
 220                                   '--constraints mem=2G --bootstrap-series '
 221                                   'xenial '
 222                                   '--config use-floating-ip=true --debug'.
 223                                   format(net_id))
 224         os.system(juju_bootstrap_command)
 225         return True
 226
 227     def deploy_vnf(self):
 228         """Deploy ABOT-OAI-EPC."""
 229         self.__logger.info("Upload VNFD")
 230         descriptor = self.vnf['descriptor']
 231         self.__logger.info("Get or create flavor for all Abot-EPC")
 232         flavor_settings = FlavorConfig(
 233             name=self.vnf['requirements']['flavor']['name'],
 234             ram=self.vnf['requirements']['flavor']['ram_min'],
 235             disk=10,
 236             vcpus=1)
 237         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 238         flavor_creator.create()
 239         self.created_object.append(flavor_creator)
 240         self.__logger.info("Deploying Abot-epc bundle file ...")
 241         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 242         self.__logger.info("Waiting for instances .....")
 243         status = os.system('juju-wait')
 244         self.__logger.info("juju wait completed: %s", status)
 245         self.__logger.info("Deployed Abot-epc on Openstack")
 246         if status == 0:
 247             instances = os_utils.get_instances(self.nova_client)
 248             for items in instances:
 249                 metadata = get_instance_metadata(self.nova_client, items)
 250                 if 'juju-units-deployed' in metadata:
 251                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 252                                  '-' + metadata['juju-model-uuid'])
 253                     self.sec_group_id = os_utils.get_security_group_id(
 254                         self.neutron_client, sec_group)
 255                     break
 256             self.__logger.info("Adding Security group rule....")
 257             os_utils.create_secgroup_rule(self.neutron_client,
 258                                           self.sec_group_id, 'ingress', 132)
 259             self.__logger.info("Copying the feature files to Abot_node ")
 260             os.system('juju scp -- -r {}/featureFiles abot-'
 261                       'epc-basic/0:~/'.format(self.case_dir))
 262             self.__logger.info("Copying the feature files in Abot_node ")
 263             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 264                       "~/featureFiles /etc/rebaca-test-suite"
 265                       "/featureFiles'")
 266             count = 0
 267             while count < 10:
 268                 epcstatus = os.system('juju status oai-epc | '
 269                                       'grep {} | grep {} | grep {}'
 270                                       .format('EPC', 'is', 'running'))
 271                 if epcstatus == 0:
 272                     break
 273                 else:
 274                     time.sleep(60)
 275                     count = count + 1
 276             os.system('juju-wait')
 277             return True
 278         return False
 279
 280     def test_vnf(self):
 281         """Run test on ABoT."""
 282         start_time = time.time()
 283         self.__logger.info("Running VNF Test cases....")
 284         os.system('juju run-action abot-epc-basic/0 run '
 285                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 286         os.system('juju-wait')
 287         duration = time.time() - start_time
 288         self.__logger.info("Getting results from Abot node....")
 289         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 290                   'epc-basic/artifacts/TestResults.json {}/.'
 291                   .format(self.case_dir))
 292         self.__logger.info("Parsing the Test results...")
 293         res = (process_abot_test_result('{}/TestResults.'
 294                                         'json'.format(self.case_dir)))
 295         short_result = sig_test_format(res)
 296         self.__logger.info(short_result)
 297         self.details['test_vnf'].update(status='PASS',
 298                                         result=short_result,
 299                                         full_result=res,
 300                                         duration=duration)
 301
 302         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 303                            "%d, Skipped: %d", short_result['passed'],
 304                            short_result['failures'], short_result['skipped'])
 305         return True
 306
 307     def clean(self):
 308         """Clean created objects/functions."""
 309         try:
 310             if not self.orchestrator['requirements']['preserve_setup']:
 311                 self.__logger.info("Removing deployment files...")
 312                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 313                 if os.path.exists(testresult):
 314                     os.remove(testresult)
 315                 self.__logger.info("Removing %s file ", self.filename)
 316                 if os.path.exists(self.filename):
 317                     os.remove(self.filename)
 318                 self.__logger.info("Destroying Orchestrator...")
 319                 os.system('juju destroy-controller -y abot-controller '
 320                           '--destroy-all-models')
 321         except Exception:  # pylint: disable=broad-except
 322             self.__logger.warn("Some issue during the undeployment ..")
 323             self.__logger.warn("Tenant clean continue ..")
 324
 325         if not self.orchestrator['requirements']['preserve_setup']:
 326             self.__logger.info('Remove the Abot_epc OS object ..')
 327             for creator in reversed(self.created_object):
 328                 try:
 329                     creator.clean()
 330                 except Exception as exc:  # pylint: disable=broad-except
 331                     self.__logger.error('Unexpected error cleaning - %s', exc)
 332
 333             self.__logger.info("Releasing all the floating IPs")
 334             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 335             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 336                                                self.tenant_name)
 337             self.__logger.info("TENANT ID : %s", tenant_id)
 338             for item in floating_ips:
 339                 if item['tenant_id'] == tenant_id:
 340                     os_utils.delete_floating_ip(self.neutron_client,
 341                                                 item['id'])
 342             self.__logger.info("Cleaning Projects and Users")
 343             for creator in reversed(self.created_object):
 344                 try:
 345                     creator.clean()
 346                 except Exception as exc:  # pylint: disable=broad-except
 347                     self.__logger.error('Unexpected error cleaning - %s', exc)
 348         return True
 349
 350
 351 # ----------------------------------------------------------
 352 #
 353 #               YAML UTILS
 354 #
 355 # -----------------------------------------------------------
 356 def get_config(parameter, file_path):
 357     """
 358     Returns the value of a given parameter in file.yaml
 359     parameter must be given in string format with dots
 360     Example: general.openstack.image_name
 361     """
 362     with open(file_path) as config_file:
 363         file_yaml = yaml.safe_load(config_file)
 364     config_file.close()
 365     value = file_yaml
 366     for element in parameter.split("."):
 367         value = value.get(element)
 368         if value is None:
 369             raise ValueError("The parameter %s is not defined in"
 370                              " reporting.yaml" % parameter)
 371     return value
 372
 373
 374 def sig_test_format(sig_test):
 375     """
 376     Process the signaling result to have a short result
 377     """
 378     nb_passed = 0
 379     nb_failures = 0
 380     nb_skipped = 0
 381     for data_test in sig_test:
 382         if data_test['result'] == "passed":
 383             nb_passed += 1
 384         elif data_test['result'] == "failed":
 385             nb_failures += 1
 386         elif data_test['result'] == "skipped":
 387             nb_skipped += 1
 388     total_sig_test_result = {}
 389     total_sig_test_result['passed'] = nb_passed
 390     total_sig_test_result['failures'] = nb_failures
 391     total_sig_test_result['skipped'] = nb_skipped
 392     return total_sig_test_result
 393
 394
 395 def process_abot_test_result(file_path):
 396     """ Process ABoT Result """
 397     with open(file_path) as test_result:
 398         data = json.load(test_result)
 399         res = []
 400         for tests in data:
 401             tests = update_data(tests)
 402             try:
 403                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 404                 for steps in flatten_steps:
 405                     steps['result'] = steps['step_status']
 406                     res.append(steps)
 407             except:
 408                 logging.error("Could not post data to ElasticSearch host")
 409                 raise
 410         return res
 411
 412
 413 def update_data(obj):
 414     """ Update Result data"""
 415     try:
 416         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 417
 418         for element in obj['elements']:
 419             element['final_result'] = "passed"
 420             element['flatten_steps'] = []
 421
 422             for step in element['steps']:
 423                 temp_dict = {}
 424                 step['result'][step['result']['status']] = 1
 425                 if step['result']['status'].lower() in ['fail', 'failed']:
 426                     element['final_result'] = "failed"
 427
 428                 temp_dict['feature_file'] = obj['feature_file']
 429                 temp_dict['step_name'] = step['name']
 430                 temp_dict['step_status'] = step['result']['status']
 431                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 432                 temp_dict['step_' + step['result']['status']] = 1
 433                 element['flatten_steps'].append(deepcopy(temp_dict))
 434
 435             # Need to put the tag in OBJ and not ELEMENT
 436             if 'tags' in obj:
 437                 element['tags'] = deepcopy(obj['tags'])
 438                 for tag in obj['tags']:
 439                     element[tag['name']] = 1
 440             else:
 441                 for tag in element['tags']:
 442                     element[tag['name']] = 1
 443
 444     except Exception:  # pylint: disable=broad-except
 445         logging.error("Error in updating data, %s", sys.exc_info()[0])
 446         raise
 447
 448     return obj
 449
 450
 451 def get_instance_metadata(nova_client, instance):
 452     """ Get instance Metadata - Instance ID """
 453     try:
 454         instance = nova_client.servers.get(instance.id)
 455         return instance.metadata
 456     except Exception as exc:  # pylint: disable=broad-except
 457         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 458         return None
 459
 460
 461 CLOUD_TEMPLATE = """clouds:
 462     abot-epc:
 463       type: openstack
 464       auth-types: [userpass]
 465       endpoint: {url}
 466       regions:
 467         {region}:
 468           endpoint: {url}
 469 credentials:
 470   abot-epc:
 471     abot-epc:
 472       auth-type: userpass
 473       password: {pass}
 474       tenant-name: {tenant_n}
 475       username: {user_n}"""
 476
 477
 478 def write_config(fname, template, **kwargs):
 479     """ Generate yaml from template for addinh cloud in juju """
 480     with open(fname, 'w') as yfile:
 481         yfile.write(template.format(**kwargs))
 482
 483
 484 def append_config(file_name, p_domain, u_domain):
 485     """ Append values into a yaml file  """
 486     with open(file_name) as yfile:
 487         doc = yaml.load(yfile)
 488     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 489         p_domain)
 490     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 491         u_domain)
 492
 493     with open(file_name, 'w') as yfile:
 494         yaml.safe_dump(doc, yfile, default_flow_style=False)