functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 from urlparse import urljoin
  19
  20 import functest.core.vnf as vnf
  21 from functest.opnfv_tests.openstack.snaps import snaps_utils
  22 from functest.utils.constants import CONST
  23 import functest.utils.openstack_utils as os_utils
  24
  25 import pkg_resources
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.tests import openstack_tests
  33 from snaps.openstack.utils import keystone_utils
  34 import yaml
  35
  36 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  37 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  38
  39
  40 class JujuEpc(vnf.VnfOnBoarding):
  41     # pylint:disable=too-many-instance-attributes
  42     """Abot EPC deployed with JUJU Orchestrator Case"""
  43
  44     __logger = logging.getLogger(__name__)
  45
  46     default_region_name = "RegionOne"
  47
  48     def __init__(self, **kwargs):
  49         if "case_name" not in kwargs:
  50             kwargs["case_name"] = "juju_epc"
  51         super(JujuEpc, self).__init__(**kwargs)
  52
  53         # Retrieve the configuration
  54         self.case_dir = pkg_resources.resource_filename(
  55             'functest', 'opnfv_tests/vnf/epc')
  56         try:
  57             self.config = CONST.__getattribute__(
  58                 'vnf_{}_config'.format(self.case_name))
  59         except Exception:
  60             raise Exception("VNF config file not found")
  61         config_file = os.path.join(self.case_dir, self.config)
  62         self.orchestrator = dict(
  63             requirements=get_config("orchestrator.requirements", config_file),
  64         )
  65
  66         self.created_object = []
  67         self.snaps_creds = ''
  68
  69         self.os_creds = openstack_tests.get_credentials(
  70             os_env_file=CONST.__getattribute__('openstack_creds'))
  71
  72         self.details['orchestrator'] = dict(
  73             name=get_config("orchestrator.name", config_file),
  74             version=get_config("orchestrator.version", config_file),
  75             status='ERROR',
  76             result=''
  77         )
  78
  79         self.vnf = dict(
  80             descriptor=get_config("vnf.descriptor", config_file),
  81             requirements=get_config("vnf.requirements", config_file)
  82         )
  83         self.details['vnf'] = dict(
  84             descriptor_version=self.vnf['descriptor']['version'],
  85             name=get_config("vnf.name", config_file),
  86             version=get_config("vnf.version", config_file),
  87         )
  88         self.__logger.debug("VNF configuration: %s", self.vnf)
  89
  90         self.details['test_vnf'] = dict(
  91             name=get_config("vnf_test_suite.name", config_file),
  92             version=get_config("vnf_test_suite.version", config_file),
  93             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  94         )
  95         self.images = get_config("tenant_images", config_file)
  96         self.__logger.info("Images needed for vEPC: %s", self.images)
  97         self.keystone_client = os_utils.get_keystone_client()
  98         self.glance_client = os_utils.get_glance_client()
  99         self.neutron_client = os_utils.get_neutron_client()
 100         self.nova_client = os_utils.get_nova_client()
 101         self.sec_group_id = None
 102         self.public_auth_url = None
 103         self.creds = None
 104         self.filename = None
 105
 106     def prepare(self):
 107         """Prepare testcase (Additional pre-configuration steps)."""
 108         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 109
 110         super(JujuEpc, self).prepare()
 111
 112         self.__logger.info("Additional pre-configuration steps")
 113         self.public_auth_url = keystone_utils.get_endpoint(
 114             self.snaps_creds, 'identity')
 115         # it enforces a versioned public identity endpoint as juju simply
 116         # adds /auth/tokens wich fails vs an unversioned endpoint.
 117         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 118             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 119
 120         self.creds = {
 121             "tenant": self.tenant_name,
 122             "username": self.tenant_name,
 123             "password": self.tenant_name,
 124             "auth_url": os_utils.get_credentials()['auth_url']
 125             }
 126
 127         self.snaps_creds = OSCreds(
 128             username=self.creds['username'],
 129             password=self.creds['password'],
 130             auth_url=self.creds['auth_url'],
 131             project_name=self.creds['tenant'],
 132             identity_api_version=int(os_utils.get_keystone_client_version()))
 133
 134         cloud_data = {
 135             'url': self.public_auth_url,
 136             'pass': self.tenant_name,
 137             'tenant_n': self.tenant_name,
 138             'user_n': self.tenant_name,
 139             'region': os.environ.get(
 140                 "OS_REGION_NAME", self.default_region_name)
 141         }
 142         self.__logger.info("Cloud DATA:  %s", cloud_data)
 143         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 144         self.__logger.info("Create  %s to add cloud info", self.filename)
 145         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 146
 147         if self.snaps_creds.identity_api_version == 3:
 148             append_config(self.filename, '{}'.format(
 149                 os_utils.get_credentials()['project_domain_name']),
 150                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 151
 152         self.__logger.info("Upload some OS images if it doesn't exist")
 153         for image_name, image_file in self.images.iteritems():
 154             self.__logger.info("image: %s, file: %s", image_name, image_file)
 155             if image_file and image_name:
 156                 image_creator = OpenStackImage(
 157                     self.snaps_creds,
 158                     ImageSettings(name=image_name,
 159                                   image_user='cloud',
 160                                   img_format='qcow2',
 161                                   image_file=image_file))
 162                 image_creator.create()
 163                 self.created_object.append(image_creator)
 164
 165     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 166         """
 167         Create network, subnet, router
 168
 169         Bootstrap juju
 170         """
 171         self.__logger.info("Deployed Orchestrator")
 172         private_net_name = CONST.__getattribute__(
 173             'vnf_{}_private_net_name'.format(self.case_name))
 174         private_subnet_name = CONST.__getattribute__(
 175             'vnf_{}_private_subnet_name'.format(self.case_name))
 176         private_subnet_cidr = CONST.__getattribute__(
 177             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 178         abot_router = CONST.__getattribute__(
 179             'vnf_{}_external_router'.format(self.case_name))
 180         dns_nameserver = CONST.__getattribute__(
 181             'vnf_{}_dns_nameserver'.format(self.case_name))
 182         ext_net_name = CONST.__getattribute__(
 183             'vnf_{}_external_network_name'.format(self.case_name))
 184
 185         self.__logger.info("Creating full network ...")
 186         subnet_settings = SubnetSettings(name=private_subnet_name,
 187                                          cidr=private_subnet_cidr,
 188                                          dns_nameservers=dns_nameserver)
 189         network_settings = NetworkSettings(name=private_net_name,
 190                                            subnet_settings=[subnet_settings])
 191         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 192         network_creator.create()
 193         self.created_object.append(network_creator)
 194
 195         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 196         self.__logger.info("Creating network Router ....")
 197         router_creator = OpenStackRouter(
 198             self.snaps_creds,
 199             RouterSettings(
 200                 name=abot_router,
 201                 external_gateway=ext_net_name,
 202                 internal_subnets=[subnet_settings.name]))
 203         router_creator.create()
 204         self.created_object.append(router_creator)
 205         self.__logger.info("Creating Flavor ....")
 206         flavor_settings = FlavorSettings(
 207             name=self.orchestrator['requirements']['flavor']['name'],
 208             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 209             disk=10,
 210             vcpus=1)
 211         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 212         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 213         flavor_creator.create()
 214         self.created_object.append(flavor_creator)
 215         self.__logger.info("Installing Dependency Packages .......")
 216         source_dir = "/src/epc-requirements/juju_bin_build"
 217         if os.path.exists(source_dir):
 218             shutil.rmtree(source_dir)
 219         os.makedirs(source_dir)
 220         os.environ['GOPATH'] = str(source_dir)
 221         os.environ['GOBIN'] = str(source_dir) + "/bin"
 222         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 223                               (os.path.expandvars('$GOBIN')) + ":" +
 224                               (os.path.expandvars('$PATH')))
 225         os.system('go get -d -v github.com/juju/juju/...')
 226         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 227         os.system('git checkout tags/juju-2.2.5')
 228         os.system('go get github.com/rogpeppe/godeps')
 229         os.system('godeps -u dependencies.tsv')
 230         os.system('go install -v github.com/juju/juju/...')
 231         self.__logger.info("Creating Cloud for Abot-epc .....")
 232         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 233         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 234         for image_name in self.images.keys():
 235             self.__logger.info("Generating Metadata for %s", image_name)
 236             image_id = os_utils.get_image_id(self.glance_client, image_name)
 237             os.system(
 238                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 239                 '{} -u {}'.format(
 240                     image_id, image_name,
 241                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 242                     self.public_auth_url))
 243         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 244         self.__logger.info("Credential information  : %s", net_id)
 245         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 246                                   '--config network={} --metadata-source ~  '
 247                                   '--config ssl-hostname-verification=false '
 248                                   '--constraints mem=2G --bootstrap-series '
 249                                   'xenial '
 250                                   '--config use-floating-ip=true --debug'.
 251                                   format(net_id))
 252         os.system(juju_bootstrap_command)
 253         return True
 254
 255     def deploy_vnf(self):
 256         """Deploy ABOT-OAI-EPC."""
 257         self.__logger.info("Upload VNFD")
 258         descriptor = self.vnf['descriptor']
 259         self.__logger.info("Get or create flavor for all Abot-EPC")
 260         flavor_settings = FlavorSettings(
 261             name=self.vnf['requirements']['flavor']['name'],
 262             ram=self.vnf['requirements']['flavor']['ram_min'],
 263             disk=10,
 264             vcpus=1)
 265         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 266         flavor_creator.create()
 267         self.created_object.append(flavor_creator)
 268         self.__logger.info("Deploying Abot-epc bundle file ...")
 269         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 270         self.__logger.info("Waiting for instances .....")
 271         status = os.system('juju-wait')
 272         self.__logger.info("juju wait completed: %s", status)
 273         self.__logger.info("Deployed Abot-epc on Openstack")
 274         if status == 0:
 275             instances = os_utils.get_instances(self.nova_client)
 276             for items in instances:
 277                 metadata = get_instance_metadata(self.nova_client, items)
 278                 if 'juju-units-deployed' in metadata:
 279                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 280                                  '-' + metadata['juju-model-uuid'])
 281                     self.sec_group_id = os_utils.get_security_group_id(
 282                         self.neutron_client, sec_group)
 283                     break
 284             self.__logger.info("Adding Security group rule....")
 285             os_utils.create_secgroup_rule(self.neutron_client,
 286                                           self.sec_group_id, 'ingress', 132)
 287             self.__logger.info("Copying the feature files to Abot_node ")
 288             os.system('juju scp -- -r {}/featureFiles abot-'
 289                       'epc-basic/0:~/'.format(self.case_dir))
 290             self.__logger.info("Copying the feature files in Abot_node ")
 291             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 292                       "~/featureFiles /etc/rebaca-test-suite"
 293                       "/featureFiles'")
 294             count = 0
 295             while count < 10:
 296                 epcstatus = os.system('juju status oai-epc | '
 297                                       'grep {} | grep {} | grep {}'
 298                                       .format('EPC', 'is', 'running'))
 299                 if epcstatus == 0:
 300                     break
 301                 else:
 302                     time.sleep(60)
 303                     count = count + 1
 304             os.system('juju-wait')
 305             return True
 306         return False
 307
 308     def test_vnf(self):
 309         """Run test on ABoT."""
 310         start_time = time.time()
 311         self.__logger.info("Running VNF Test cases....")
 312         os.system('juju run-action abot-epc-basic/0 run '
 313                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 314         os.system('juju-wait')
 315         duration = time.time() - start_time
 316         self.__logger.info("Getting results from Abot node....")
 317         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 318                   'epc-basic/artifacts/TestResults.json {}/.'
 319                   .format(self.case_dir))
 320         self.__logger.info("Parsing the Test results...")
 321         res = (process_abot_test_result('{}/TestResults.'
 322                                         'json'.format(self.case_dir)))
 323         short_result = sig_test_format(res)
 324         self.__logger.info(short_result)
 325         self.details['test_vnf'].update(status='PASS',
 326                                         result=short_result,
 327                                         full_result=res,
 328                                         duration=duration)
 329
 330         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 331                            "%d, Skipped: %d", short_result['passed'],
 332                            short_result['failures'], short_result['skipped'])
 333         return True
 334
 335     def clean(self):
 336         """Clean created objects/functions."""
 337         try:
 338             if not self.orchestrator['requirements']['preserve_setup']:
 339                 self.__logger.info("Removing deployment files...")
 340                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 341                 if os.path.exists(testresult):
 342                     os.remove(testresult)
 343                 self.__logger.info("Removing %s file ", self.filename)
 344                 if os.path.exists(self.filename):
 345                     os.remove(self.filename)
 346                 self.__logger.info("Destroying Orchestrator...")
 347                 os.system('juju destroy-controller -y abot-controller '
 348                           '--destroy-all-models')
 349         except Exception:  # pylint: disable=broad-except
 350             self.__logger.warn("Some issue during the undeployment ..")
 351             self.__logger.warn("Tenant clean continue ..")
 352
 353         if not self.orchestrator['requirements']['preserve_setup']:
 354             self.__logger.info('Remove the Abot_epc OS object ..')
 355             for creator in reversed(self.created_object):
 356                 try:
 357                     creator.clean()
 358                 except Exception as exc:  # pylint: disable=broad-except
 359                     self.__logger.error('Unexpected error cleaning - %s', exc)
 360
 361             self.__logger.info("Releasing all the floating IPs")
 362             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 363             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 364                                                self.tenant_name)
 365             self.__logger.info("TENANT ID : %s", tenant_id)
 366             for item in floating_ips:
 367                 if item['tenant_id'] == tenant_id:
 368                     os_utils.delete_floating_ip(self.neutron_client,
 369                                                 item['id'])
 370             self.__logger.info("Cleaning Projects and Users")
 371             for creator in reversed(self.created_object):
 372                 try:
 373                     creator.clean()
 374                 except Exception as exc:  # pylint: disable=broad-except
 375                     self.__logger.error('Unexpected error cleaning - %s', exc)
 376         return True
 377
 378
 379 # ----------------------------------------------------------
 380 #
 381 #               YAML UTILS
 382 #
 383 # -----------------------------------------------------------
 384 def get_config(parameter, file_path):
 385     """
 386     Returns the value of a given parameter in file.yaml
 387     parameter must be given in string format with dots
 388     Example: general.openstack.image_name
 389     """
 390     with open(file_path) as config_file:
 391         file_yaml = yaml.safe_load(config_file)
 392     config_file.close()
 393     value = file_yaml
 394     for element in parameter.split("."):
 395         value = value.get(element)
 396         if value is None:
 397             raise ValueError("The parameter %s is not defined in"
 398                              " reporting.yaml" % parameter)
 399     return value
 400
 401
 402 def sig_test_format(sig_test):
 403     """
 404     Process the signaling result to have a short result
 405     """
 406     nb_passed = 0
 407     nb_failures = 0
 408     nb_skipped = 0
 409     for data_test in sig_test:
 410         if data_test['result'] == "passed":
 411             nb_passed += 1
 412         elif data_test['result'] == "failed":
 413             nb_failures += 1
 414         elif data_test['result'] == "skipped":
 415             nb_skipped += 1
 416     total_sig_test_result = {}
 417     total_sig_test_result['passed'] = nb_passed
 418     total_sig_test_result['failures'] = nb_failures
 419     total_sig_test_result['skipped'] = nb_skipped
 420     return total_sig_test_result
 421
 422
 423 def process_abot_test_result(file_path):
 424     """ Process ABoT Result """
 425     with open(file_path) as test_result:
 426         data = json.load(test_result)
 427         res = []
 428         for tests in data:
 429             tests = update_data(tests)
 430             try:
 431                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 432                 for steps in flatten_steps:
 433                     steps['result'] = steps['step_status']
 434                     res.append(steps)
 435             except:
 436                 logging.error("Could not post data to ElasticSearch host")
 437                 raise
 438         return res
 439
 440
 441 def update_data(obj):
 442     """ Update Result data"""
 443     try:
 444         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 445
 446         for element in obj['elements']:
 447             element['final_result'] = "passed"
 448             element['flatten_steps'] = []
 449
 450             for step in element['steps']:
 451                 temp_dict = {}
 452                 step['result'][step['result']['status']] = 1
 453                 if step['result']['status'].lower() in ['fail', 'failed']:
 454                     element['final_result'] = "failed"
 455
 456                 temp_dict['feature_file'] = obj['feature_file']
 457                 temp_dict['step_name'] = step['name']
 458                 temp_dict['step_status'] = step['result']['status']
 459                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 460                 temp_dict['step_' + step['result']['status']] = 1
 461                 element['flatten_steps'].append(deepcopy(temp_dict))
 462
 463             # Need to put the tag in OBJ and not ELEMENT
 464             if 'tags' in obj:
 465                 element['tags'] = deepcopy(obj['tags'])
 466                 for tag in obj['tags']:
 467                     element[tag['name']] = 1
 468             else:
 469                 for tag in element['tags']:
 470                     element[tag['name']] = 1
 471
 472     except Exception:  # pylint: disable=broad-except
 473         logging.error("Error in updating data, %s", sys.exc_info()[0])
 474         raise
 475
 476     return obj
 477
 478
 479 def get_instance_metadata(nova_client, instance):
 480     """ Get instance Metadata - Instance ID """
 481     try:
 482         instance = nova_client.servers.get(instance.id)
 483         return instance.metadata
 484     except Exception as exc:  # pylint: disable=broad-except
 485         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 486         return None
 487
 488
 489 CLOUD_TEMPLATE = """clouds:
 490     abot-epc:
 491       type: openstack
 492       auth-types: [userpass]
 493       endpoint: {url}
 494       regions:
 495         {region}:
 496           endpoint: {url}
 497 credentials:
 498   abot-epc:
 499     abot-epc:
 500       auth-type: userpass
 501       password: {pass}
 502       tenant-name: {tenant_n}
 503       username: {user_n}"""
 504
 505
 506 def write_config(fname, template, **kwargs):
 507     """ Generate yaml from template for addinh cloud in juju """
 508     with open(fname, 'w') as yfile:
 509         yfile.write(template.format(**kwargs))
 510
 511
 512 def append_config(file_name, p_domain, u_domain):
 513     """ Append values into a yaml file  """
 514     with open(file_name) as yfile:
 515         doc = yaml.load(yfile)
 516     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 517         p_domain)
 518     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 519         u_domain)
 520
 521     with open(file_name, 'w') as yfile:
 522         yaml.safe_dump(doc, yfile, default_flow_style=False)