functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 from urlparse import urljoin
  19
  20 import functest.core.vnf as vnf
  21 from functest.opnfv_tests.openstack.snaps import snaps_utils
  22 from functest.utils.constants import CONST
  23 import functest.utils.openstack_utils as os_utils
  24
  25 import pkg_resources
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.tests import openstack_tests
  33 from snaps.openstack.utils import keystone_utils
  34 import yaml
  35
  36 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  37 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  38
  39
  40 class JujuEpc(vnf.VnfOnBoarding):
  41     # pylint:disable=too-many-instance-attributes
  42     """Abot EPC deployed with JUJU Orchestrator Case"""
  43
  44     __logger = logging.getLogger(__name__)
  45
  46     default_region_name = "RegionOne"
  47
  48     def __init__(self, **kwargs):
  49         if "case_name" not in kwargs:
  50             kwargs["case_name"] = "juju_epc"
  51         super(JujuEpc, self).__init__(**kwargs)
  52
  53         # Retrieve the configuration
  54         self.case_dir = pkg_resources.resource_filename(
  55             'functest', 'opnfv_tests/vnf/epc')
  56         try:
  57             self.config = CONST.__getattribute__(
  58                 'vnf_{}_config'.format(self.case_name))
  59         except Exception:
  60             raise Exception("VNF config file not found")
  61         config_file = os.path.join(self.case_dir, self.config)
  62         self.orchestrator = dict(
  63             requirements=get_config("orchestrator.requirements", config_file),
  64         )
  65
  66         self.created_object = []
  67         self.snaps_creds = ''
  68
  69         self.os_creds = openstack_tests.get_credentials(
  70             os_env_file=CONST.__getattribute__('openstack_creds'))
  71
  72         self.details['orchestrator'] = dict(
  73             name=get_config("orchestrator.name", config_file),
  74             version=get_config("orchestrator.version", config_file),
  75             status='ERROR',
  76             result=''
  77         )
  78
  79         self.vnf = dict(
  80             descriptor=get_config("vnf.descriptor", config_file),
  81             requirements=get_config("vnf.requirements", config_file)
  82         )
  83         self.details['vnf'] = dict(
  84             descriptor_version=self.vnf['descriptor']['version'],
  85             name=get_config("vnf.name", config_file),
  86             version=get_config("vnf.version", config_file),
  87         )
  88         self.__logger.debug("VNF configuration: %s", self.vnf)
  89
  90         self.details['test_vnf'] = dict(
  91             name=get_config("vnf_test_suite.name", config_file),
  92             version=get_config("vnf_test_suite.version", config_file),
  93             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  94         )
  95         self.images = get_config("tenant_images", config_file)
  96         self.__logger.info("Images needed for vEPC: %s", self.images)
  97         self.keystone_client = os_utils.get_keystone_client()
  98         self.glance_client = os_utils.get_glance_client()
  99         self.neutron_client = os_utils.get_neutron_client()
 100         self.nova_client = os_utils.get_nova_client()
 101         self.sec_group_id = None
 102         self.public_auth_url = None
 103         self.creds = None
 104         self.filename = None
 105
 106     def prepare(self):
 107         """Prepare testcase (Additional pre-configuration steps)."""
 108         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 109
 110         super(JujuEpc, self).prepare()
 111
 112         self.__logger.info("Additional pre-configuration steps")
 113         self.public_auth_url = keystone_utils.get_endpoint(
 114             self.snaps_creds, 'identity')
 115         # it enforces a versioned public identity endpoint as juju simply
 116         # adds /auth/tokens wich fails vs an unversioned endpoint.
 117         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 118             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 119
 120         self.creds = {
 121             "tenant": self.tenant_name,
 122             "username": self.tenant_name,
 123             "password": self.tenant_name,
 124             "auth_url": os_utils.get_credentials()['auth_url']
 125             }
 126
 127         self.snaps_creds = OSCreds(
 128             username=self.creds['username'],
 129             password=self.creds['password'],
 130             auth_url=self.creds['auth_url'],
 131             project_name=self.creds['tenant'],
 132             identity_api_version=int(os_utils.get_keystone_client_version()))
 133
 134         cloud_data = {
 135             'url': self.public_auth_url,
 136             'pass': self.tenant_name,
 137             'tenant_n': self.tenant_name,
 138             'user_n': self.tenant_name,
 139             'region': os.environ.get(
 140                 "OS_REGION_NAME", self.default_region_name)
 141         }
 142         self.__logger.info("Cloud DATA:  %s", cloud_data)
 143         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 144         self.__logger.info("Create  %s to add cloud info", self.filename)
 145         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 146
 147         if self.snaps_creds.identity_api_version == 3:
 148             append_config(
 149                 self.filename, '{}'.format(
 150                     os_utils.get_credentials()['project_domain_name']),
 151                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 152
 153         self.__logger.info("Upload some OS images if it doesn't exist")
 154         for image_name, image_file in self.images.iteritems():
 155             self.__logger.info("image: %s, file: %s", image_name, image_file)
 156             if image_file and image_name:
 157                 image_creator = OpenStackImage(
 158                     self.snaps_creds,
 159                     ImageSettings(name=image_name,
 160                                   image_user='cloud',
 161                                   img_format='qcow2',
 162                                   image_file=image_file))
 163                 image_creator.create()
 164                 self.created_object.append(image_creator)
 165
 166     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 167         """
 168         Create network, subnet, router
 169
 170         Bootstrap juju
 171         """
 172         self.__logger.info("Deployed Orchestrator")
 173         private_net_name = CONST.__getattribute__(
 174             'vnf_{}_private_net_name'.format(self.case_name))
 175         private_subnet_name = CONST.__getattribute__(
 176             'vnf_{}_private_subnet_name'.format(self.case_name))
 177         private_subnet_cidr = CONST.__getattribute__(
 178             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 179         abot_router = CONST.__getattribute__(
 180             'vnf_{}_external_router'.format(self.case_name))
 181         dns_nameserver = CONST.__getattribute__(
 182             'vnf_{}_dns_nameserver'.format(self.case_name))
 183         ext_net_name = CONST.__getattribute__(
 184             'vnf_{}_external_network_name'.format(self.case_name))
 185
 186         self.__logger.info("Creating full network ...")
 187         subnet_settings = SubnetSettings(name=private_subnet_name,
 188                                          cidr=private_subnet_cidr,
 189                                          dns_nameservers=dns_nameserver)
 190         network_settings = NetworkSettings(name=private_net_name,
 191                                            subnet_settings=[subnet_settings])
 192         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 193         network_creator.create()
 194         self.created_object.append(network_creator)
 195
 196         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 197         self.__logger.info("Creating network Router ....")
 198         router_creator = OpenStackRouter(
 199             self.snaps_creds,
 200             RouterSettings(
 201                 name=abot_router,
 202                 external_gateway=ext_net_name,
 203                 internal_subnets=[subnet_settings.name]))
 204         router_creator.create()
 205         self.created_object.append(router_creator)
 206         self.__logger.info("Creating Flavor ....")
 207         flavor_settings = FlavorSettings(
 208             name=self.orchestrator['requirements']['flavor']['name'],
 209             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 210             disk=10,
 211             vcpus=1)
 212         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 213         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 214         flavor_creator.create()
 215         self.created_object.append(flavor_creator)
 216         self.__logger.info("Installing Dependency Packages .......")
 217         source_dir = "/src/epc-requirements/juju_bin_build"
 218         if os.path.exists(source_dir):
 219             shutil.rmtree(source_dir)
 220         os.makedirs(source_dir)
 221         os.environ['GOPATH'] = str(source_dir)
 222         os.environ['GOBIN'] = str(source_dir) + "/bin"
 223         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 224                               (os.path.expandvars('$GOBIN')) + ":" +
 225                               (os.path.expandvars('$PATH')))
 226         os.system('go get -d -v github.com/juju/juju/...')
 227         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 228         os.system('git checkout tags/juju-2.2.5')
 229         os.system('go get github.com/rogpeppe/godeps')
 230         os.system('godeps -u dependencies.tsv')
 231         os.system('go install -v github.com/juju/juju/...')
 232         self.__logger.info("Creating Cloud for Abot-epc .....")
 233         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 234         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 235         for image_name in self.images.keys():
 236             self.__logger.info("Generating Metadata for %s", image_name)
 237             image_id = os_utils.get_image_id(self.glance_client, image_name)
 238             os.system(
 239                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 240                 '{} -u {}'.format(
 241                     image_id, image_name,
 242                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 243                     self.public_auth_url))
 244         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 245         self.__logger.info("Credential information  : %s", net_id)
 246         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 247                                   '--config network={} --metadata-source ~  '
 248                                   '--config ssl-hostname-verification=false '
 249                                   '--constraints mem=2G --bootstrap-series '
 250                                   'xenial '
 251                                   '--config use-floating-ip=true --debug'.
 252                                   format(net_id))
 253         os.system(juju_bootstrap_command)
 254         return True
 255
 256     def deploy_vnf(self):
 257         """Deploy ABOT-OAI-EPC."""
 258         self.__logger.info("Upload VNFD")
 259         descriptor = self.vnf['descriptor']
 260         self.__logger.info("Get or create flavor for all Abot-EPC")
 261         flavor_settings = FlavorSettings(
 262             name=self.vnf['requirements']['flavor']['name'],
 263             ram=self.vnf['requirements']['flavor']['ram_min'],
 264             disk=10,
 265             vcpus=1)
 266         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 267         flavor_creator.create()
 268         self.created_object.append(flavor_creator)
 269         self.__logger.info("Deploying Abot-epc bundle file ...")
 270         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 271         self.__logger.info("Waiting for instances .....")
 272         status = os.system('juju-wait')
 273         self.__logger.info("juju wait completed: %s", status)
 274         self.__logger.info("Deployed Abot-epc on Openstack")
 275         if status == 0:
 276             instances = os_utils.get_instances(self.nova_client)
 277             for items in instances:
 278                 metadata = get_instance_metadata(self.nova_client, items)
 279                 if 'juju-units-deployed' in metadata:
 280                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 281                                  '-' + metadata['juju-model-uuid'])
 282                     self.sec_group_id = os_utils.get_security_group_id(
 283                         self.neutron_client, sec_group)
 284                     break
 285             self.__logger.info("Adding Security group rule....")
 286             os_utils.create_secgroup_rule(self.neutron_client,
 287                                           self.sec_group_id, 'ingress', 132)
 288             self.__logger.info("Copying the feature files to Abot_node ")
 289             os.system('juju scp -- -r {}/featureFiles abot-'
 290                       'epc-basic/0:~/'.format(self.case_dir))
 291             self.__logger.info("Copying the feature files in Abot_node ")
 292             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 293                       "~/featureFiles /etc/rebaca-test-suite"
 294                       "/featureFiles'")
 295             count = 0
 296             while count < 10:
 297                 epcstatus = os.system('juju status oai-epc | '
 298                                       'grep {} | grep {} | grep {}'
 299                                       .format('EPC', 'is', 'running'))
 300                 if epcstatus == 0:
 301                     break
 302                 else:
 303                     time.sleep(60)
 304                     count = count + 1
 305             os.system('juju-wait')
 306             return True
 307         return False
 308
 309     def test_vnf(self):
 310         """Run test on ABoT."""
 311         start_time = time.time()
 312         self.__logger.info("Running VNF Test cases....")
 313         os.system('juju run-action abot-epc-basic/0 run '
 314                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 315         os.system('juju-wait')
 316         duration = time.time() - start_time
 317         self.__logger.info("Getting results from Abot node....")
 318         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 319                   'epc-basic/artifacts/TestResults.json {}/.'
 320                   .format(self.case_dir))
 321         self.__logger.info("Parsing the Test results...")
 322         res = (process_abot_test_result('{}/TestResults.'
 323                                         'json'.format(self.case_dir)))
 324         short_result = sig_test_format(res)
 325         self.__logger.info(short_result)
 326         self.details['test_vnf'].update(status='PASS',
 327                                         result=short_result,
 328                                         full_result=res,
 329                                         duration=duration)
 330
 331         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 332                            "%d, Skipped: %d", short_result['passed'],
 333                            short_result['failures'], short_result['skipped'])
 334         return True
 335
 336     def clean(self):
 337         """Clean created objects/functions."""
 338         try:
 339             if not self.orchestrator['requirements']['preserve_setup']:
 340                 self.__logger.info("Removing deployment files...")
 341                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 342                 if os.path.exists(testresult):
 343                     os.remove(testresult)
 344                 self.__logger.info("Removing %s file ", self.filename)
 345                 if os.path.exists(self.filename):
 346                     os.remove(self.filename)
 347                 self.__logger.info("Destroying Orchestrator...")
 348                 os.system('juju destroy-controller -y abot-controller '
 349                           '--destroy-all-models')
 350         except Exception:  # pylint: disable=broad-except
 351             self.__logger.warn("Some issue during the undeployment ..")
 352             self.__logger.warn("Tenant clean continue ..")
 353
 354         if not self.orchestrator['requirements']['preserve_setup']:
 355             self.__logger.info('Remove the Abot_epc OS object ..')
 356             for creator in reversed(self.created_object):
 357                 try:
 358                     creator.clean()
 359                 except Exception as exc:  # pylint: disable=broad-except
 360                     self.__logger.error('Unexpected error cleaning - %s', exc)
 361
 362             self.__logger.info("Releasing all the floating IPs")
 363             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 364             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 365                                                self.tenant_name)
 366             self.__logger.info("TENANT ID : %s", tenant_id)
 367             for item in floating_ips:
 368                 if item['tenant_id'] == tenant_id:
 369                     os_utils.delete_floating_ip(self.neutron_client,
 370                                                 item['id'])
 371             self.__logger.info("Cleaning Projects and Users")
 372             for creator in reversed(self.created_object):
 373                 try:
 374                     creator.clean()
 375                 except Exception as exc:  # pylint: disable=broad-except
 376                     self.__logger.error('Unexpected error cleaning - %s', exc)
 377         return True
 378
 379
 380 # ----------------------------------------------------------
 381 #
 382 #               YAML UTILS
 383 #
 384 # -----------------------------------------------------------
 385 def get_config(parameter, file_path):
 386     """
 387     Returns the value of a given parameter in file.yaml
 388     parameter must be given in string format with dots
 389     Example: general.openstack.image_name
 390     """
 391     with open(file_path) as config_file:
 392         file_yaml = yaml.safe_load(config_file)
 393     config_file.close()
 394     value = file_yaml
 395     for element in parameter.split("."):
 396         value = value.get(element)
 397         if value is None:
 398             raise ValueError("The parameter %s is not defined in"
 399                              " reporting.yaml" % parameter)
 400     return value
 401
 402
 403 def sig_test_format(sig_test):
 404     """
 405     Process the signaling result to have a short result
 406     """
 407     nb_passed = 0
 408     nb_failures = 0
 409     nb_skipped = 0
 410     for data_test in sig_test:
 411         if data_test['result'] == "passed":
 412             nb_passed += 1
 413         elif data_test['result'] == "failed":
 414             nb_failures += 1
 415         elif data_test['result'] == "skipped":
 416             nb_skipped += 1
 417     total_sig_test_result = {}
 418     total_sig_test_result['passed'] = nb_passed
 419     total_sig_test_result['failures'] = nb_failures
 420     total_sig_test_result['skipped'] = nb_skipped
 421     return total_sig_test_result
 422
 423
 424 def process_abot_test_result(file_path):
 425     """ Process ABoT Result """
 426     with open(file_path) as test_result:
 427         data = json.load(test_result)
 428         res = []
 429         for tests in data:
 430             tests = update_data(tests)
 431             try:
 432                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 433                 for steps in flatten_steps:
 434                     steps['result'] = steps['step_status']
 435                     res.append(steps)
 436             except:
 437                 logging.error("Could not post data to ElasticSearch host")
 438                 raise
 439         return res
 440
 441
 442 def update_data(obj):
 443     """ Update Result data"""
 444     try:
 445         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 446
 447         for element in obj['elements']:
 448             element['final_result'] = "passed"
 449             element['flatten_steps'] = []
 450
 451             for step in element['steps']:
 452                 temp_dict = {}
 453                 step['result'][step['result']['status']] = 1
 454                 if step['result']['status'].lower() in ['fail', 'failed']:
 455                     element['final_result'] = "failed"
 456
 457                 temp_dict['feature_file'] = obj['feature_file']
 458                 temp_dict['step_name'] = step['name']
 459                 temp_dict['step_status'] = step['result']['status']
 460                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 461                 temp_dict['step_' + step['result']['status']] = 1
 462                 element['flatten_steps'].append(deepcopy(temp_dict))
 463
 464             # Need to put the tag in OBJ and not ELEMENT
 465             if 'tags' in obj:
 466                 element['tags'] = deepcopy(obj['tags'])
 467                 for tag in obj['tags']:
 468                     element[tag['name']] = 1
 469             else:
 470                 for tag in element['tags']:
 471                     element[tag['name']] = 1
 472
 473     except Exception:  # pylint: disable=broad-except
 474         logging.error("Error in updating data, %s", sys.exc_info()[0])
 475         raise
 476
 477     return obj
 478
 479
 480 def get_instance_metadata(nova_client, instance):
 481     """ Get instance Metadata - Instance ID """
 482     try:
 483         instance = nova_client.servers.get(instance.id)
 484         return instance.metadata
 485     except Exception as exc:  # pylint: disable=broad-except
 486         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 487         return None
 488
 489
 490 CLOUD_TEMPLATE = """clouds:
 491     abot-epc:
 492       type: openstack
 493       auth-types: [userpass]
 494       endpoint: {url}
 495       regions:
 496         {region}:
 497           endpoint: {url}
 498 credentials:
 499   abot-epc:
 500     abot-epc:
 501       auth-type: userpass
 502       password: {pass}
 503       tenant-name: {tenant_n}
 504       username: {user_n}"""
 505
 506
 507 def write_config(fname, template, **kwargs):
 508     """ Generate yaml from template for addinh cloud in juju """
 509     with open(fname, 'w') as yfile:
 510         yfile.write(template.format(**kwargs))
 511
 512
 513 def append_config(file_name, p_domain, u_domain):
 514     """ Append values into a yaml file  """
 515     with open(file_name) as yfile:
 516         doc = yaml.load(yfile)
 517     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 518         p_domain)
 519     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 520         u_domain)
 521
 522     with open(file_name, 'w') as yfile:
 523         yaml.safe_dump(doc, yfile, default_flow_style=False)