functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 import yaml
  19 import functest.utils.openstack_utils as os_utils
  20 import functest.core.vnf as vnf
  21 import pkg_resources
  22
  23 from functest.opnfv_tests.openstack.snaps import snaps_utils
  24 from functest.utils.constants import CONST
  25
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.tests import openstack_tests
  33 from snaps.openstack.utils import keystone_utils
  34
  35 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  36 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  37
  38
  39 class JujuEpc(vnf.VnfOnBoarding):
  40     """Abot EPC deployed with JUJU Orchestrator Case"""
  41
  42     __logger = logging.getLogger(__name__)
  43
  44     def __init__(self, **kwargs):
  45         if "case_name" not in kwargs:
  46             kwargs["case_name"] = "juju_epc"
  47         super(JujuEpc, self).__init__(**kwargs)
  48
  49         # Retrieve the configuration
  50         self.case_dir = pkg_resources.resource_filename(
  51             'functest', 'opnfv_tests/vnf/epc')
  52         try:
  53             self.config = CONST.__getattribute__(
  54                 'vnf_{}_config'.format(self.case_name))
  55         except Exception:
  56             raise Exception("VNF config file not found")
  57         config_file = os.path.join(self.case_dir, self.config)
  58         self.orchestrator = dict(
  59             requirements=get_config("orchestrator.requirements", config_file),
  60         )
  61
  62         self.created_object = []
  63         self.snaps_creds = ''
  64
  65         self.os_creds = openstack_tests.get_credentials(
  66             os_env_file=CONST.__getattribute__('openstack_creds'))
  67
  68         self.details['orchestrator'] = dict(
  69             name=get_config("orchestrator.name", config_file),
  70             version=get_config("orchestrator.version", config_file),
  71             status='ERROR',
  72             result=''
  73         )
  74
  75         self.vnf = dict(
  76             descriptor=get_config("vnf.descriptor", config_file),
  77             requirements=get_config("vnf.requirements", config_file)
  78         )
  79         self.details['vnf'] = dict(
  80             descriptor_version=self.vnf['descriptor']['version'],
  81             name=get_config("vnf.name", config_file),
  82             version=get_config("vnf.version", config_file),
  83         )
  84         self.__logger.debug("VNF configuration: %s", self.vnf)
  85
  86         self.details['test_vnf'] = dict(
  87             name=get_config("vnf_test_suite.name", config_file),
  88             version=get_config("vnf_test_suite.version", config_file),
  89             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  90         )
  91         self.images = get_config("tenant_images", config_file)
  92         self.__logger.info("Images needed for vEPC: %s", self.images)
  93         self.keystone_client = os_utils.get_keystone_client()
  94         self.glance_client = os_utils.get_glance_client()
  95         self.neutron_client = os_utils.get_neutron_client()
  96         self.nova_client = os_utils.get_nova_client()
  97
  98     def prepare(self):
  99         """Prepare testcase (Additional pre-configuration steps)."""
 100         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 101
 102         super(JujuEpc, self).prepare()
 103
 104         self.__logger.info("Additional pre-configuration steps")
 105         self.public_auth_url = keystone_utils.get_endpoint(
 106             self.snaps_creds, 'identity')
 107
 108         self.creds = {
 109             "tenant": self.tenant_name,
 110             "username": self.tenant_name,
 111             "password": self.tenant_name,
 112             "auth_url": os_utils.get_credentials()['auth_url']
 113             }
 114
 115         self.snaps_creds = OSCreds(
 116             username=self.creds['username'],
 117             password=self.creds['password'],
 118             auth_url=self.creds['auth_url'],
 119             project_name=self.creds['tenant'],
 120             identity_api_version=int(os_utils.get_keystone_client_version()))
 121
 122         cloud_data = {
 123             'url': self.public_auth_url,
 124             'pass': self.tenant_name,
 125             'tenant_n': self.tenant_name,
 126             'user_n': self.tenant_name
 127         }
 128         self.__logger.info("Cloud DATA:  %s", cloud_data)
 129         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 130         self.__logger.info("Cretae  %s to add cloud info", self.filename)
 131         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 132
 133         if self.snaps_creds.identity_api_version == 3:
 134             append_config(self.filename, '{}'.format(
 135                 os_utils.get_credentials()['project_domain_name']),
 136                           '{}'.format(os_utils.get_credentials()
 137                                       ['user_domain_name']))
 138
 139         self.__logger.info("Upload some OS images if it doesn't exist")
 140         for image_name, image_file in self.images.iteritems():
 141             self.__logger.info("image: %s, file: %s", image_name, image_file)
 142             if image_file and image_name:
 143                 image_creator = OpenStackImage(
 144                     self.snaps_creds,
 145                     ImageSettings(name=image_name,
 146                                   image_user='cloud',
 147                                   img_format='qcow2',
 148                                   image_file=image_file))
 149                 image_creator.create()
 150                 self.created_object.append(image_creator)
 151
 152     def deploy_orchestrator(self):
 153         """
 154         Create network, subnet, router
 155
 156         Bootstrap juju
 157         """
 158         self.__logger.info("Deployed Orchestrator")
 159         private_net_name = CONST.__getattribute__(
 160             'vnf_{}_private_net_name'.format(self.case_name))
 161         private_subnet_name = CONST.__getattribute__(
 162             'vnf_{}_private_subnet_name'.format(self.case_name))
 163         private_subnet_cidr = CONST.__getattribute__(
 164             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 165         abot_router = CONST.__getattribute__(
 166             'vnf_{}_external_router'.format(self.case_name))
 167         dns_nameserver = CONST.__getattribute__(
 168             'vnf_{}_dns_nameserver'.format(self.case_name))
 169         ext_net_name = CONST.__getattribute__(
 170             'vnf_{}_external_network_name'.format(self.case_name))
 171
 172         self.__logger.info("Creating full network ...")
 173         subnet_settings = SubnetSettings(name=private_subnet_name,
 174                                          cidr=private_subnet_cidr,
 175                                          dns_nameservers=dns_nameserver)
 176         network_settings = NetworkSettings(name=private_net_name,
 177                                            subnet_settings=[subnet_settings])
 178         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 179         network_creator.create()
 180         self.created_object.append(network_creator)
 181
 182         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 183         self.__logger.info("Creating network Router ....")
 184         router_creator = OpenStackRouter(
 185             self.snaps_creds,
 186             RouterSettings(
 187                 name=abot_router,
 188                 external_gateway=ext_net_name,
 189                 internal_subnets=[subnet_settings.name]))
 190         router_creator.create()
 191         self.created_object.append(router_creator)
 192         self.__logger.info("Creating Flavor ....")
 193         flavor_settings = FlavorSettings(
 194             name=self.orchestrator['requirements']['flavor']['name'],
 195             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 196             disk=10,
 197             vcpus=1)
 198         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 199         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 200         flavor_creator.create()
 201         self.created_object.append(flavor_creator)
 202         self.__logger.info("Installing Dependency Packages .......")
 203         source_dir = "/src/epc-requirements/juju_bin_build"
 204         if os.path.exists(source_dir):
 205             shutil.rmtree(source_dir)
 206         os.makedirs(source_dir)
 207         os.environ['GOPATH'] = str(source_dir)
 208         os.environ['GOBIN'] = str(source_dir) + "/bin"
 209         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 210                               (os.path.expandvars('$GOBIN')) + ":" +
 211                               (os.path.expandvars('$PATH')))
 212         os.system('go get -d -v github.com/juju/juju/...')
 213         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 214         os.system('git checkout tags/juju-2.2.5')
 215         os.system('go get github.com/rogpeppe/godeps')
 216         os.system('godeps -u dependencies.tsv')
 217         os.system('go install -v github.com/juju/juju/...')
 218         self.__logger.info("Creating Cloud for Abot-epc .....")
 219         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 220         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 221         for image_name in self.images.keys():
 222             self.__logger.info("Generating Metadata for %s", image_name)
 223             image_id = os_utils.get_image_id(self.glance_client, image_name)
 224             os.system('juju metadata generate-image -d ~ -i {} -s {} -r '
 225                       'RegionOne -u {}'.format(image_id,
 226                                                image_name,
 227                                                self.public_auth_url))
 228         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 229         self.__logger.info("Credential information  : %s", net_id)
 230         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 231                                   '--config network={} --metadata-source ~  '
 232                                   '--config ssl-hostname-verification=false '
 233                                   '--constraints mem=2G --bootstrap-series '
 234                                   'trusty '
 235                                   '--config use-floating-ip=true --debug'.
 236                                   format(net_id))
 237         os.system(juju_bootstrap_command)
 238         return True
 239
 240     def deploy_vnf(self):
 241         """Deploy ABOT-OAI-EPC."""
 242         self.__logger.info("Upload VNFD")
 243         descriptor = self.vnf['descriptor']
 244         self.__logger.info("Get or create flavor for all Abot-EPC")
 245         flavor_settings = FlavorSettings(
 246             name=self.vnf['requirements']['flavor']['name'],
 247             ram=self.vnf['requirements']['flavor']['ram_min'],
 248             disk=10,
 249             vcpus=1)
 250         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 251         flavor_creator.create()
 252         self.created_object.append(flavor_creator)
 253         self.__logger.info("Deploying Abot-epc bundle file ...")
 254         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 255         self.__logger.info("Waiting for instances .....")
 256         status = os.system('juju-wait')
 257         self.__logger.info("juju wait completed: %s", status)
 258         self.__logger.info("Deployed Abot-epc on Openstack")
 259         if status == 0:
 260             instances = os_utils.get_instances(self.nova_client)
 261             for items in instances:
 262                 metadata = get_instance_metadata(self.nova_client, items)
 263                 if 'juju-units-deployed' in metadata:
 264                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 265                                  '-' + metadata['juju-model-uuid'])
 266                     self.sec_group_id = os_utils.get_security_group_id(
 267                         self.neutron_client, sec_group)
 268                     break
 269             self.__logger.info("Adding Security group rule....")
 270             os_utils.create_secgroup_rule(self.neutron_client,
 271                                           self.sec_group_id, 'ingress', 132)
 272             self.__logger.info("Copying the feature files to Abot_node ")
 273             os.system('juju scp -- -r {}/featureFiles abot-'
 274                       'epc-basic/0:~/'.format(self.case_dir))
 275             self.__logger.info("Copying the feature files in Abot_node ")
 276             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 277                       "~/featureFiles /etc/rebaca-test-suite"
 278                       "/featureFiles'")
 279             count = 0
 280             while count < 10:
 281                 epcstatus = os.system('juju status oai-epc | '
 282                                       'grep {} | grep {} | grep {}'
 283                                       .format('EPC', 'is', 'running'))
 284                 if epcstatus == 0:
 285                     break
 286                 else:
 287                     time.sleep(60)
 288                     count = count + 1
 289             os.system('juju-wait')
 290             return True
 291         else:
 292             return False
 293
 294     def test_vnf(self):
 295         """Run test on ABoT."""
 296         start_time = time.time()
 297         self.__logger.info("Running VNF Test cases....")
 298         os.system('juju run-action abot-epc-basic/0 run '
 299                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 300         os.system('juju-wait')
 301         duration = time.time() - start_time
 302         self.__logger.info("Getting results from Abot node....")
 303         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 304                   'epc-basic/artifacts/TestResults.json {}/.'
 305                   .format(self.case_dir))
 306         self.__logger.info("Parsing the Test results...")
 307         res = (process_abot_test_result('{}/TestResults.'
 308                                         'json'.format(self.case_dir)))
 309         short_result = sig_test_format(res)
 310         self.__logger.info(short_result)
 311         self.details['test_vnf'].update(status='PASS',
 312                                         result=short_result,
 313                                         full_result=res,
 314                                         duration=duration)
 315
 316         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 317                            "%d, Skipped: %d", short_result['passed'],
 318                            short_result['failures'], short_result['skipped'])
 319         return True
 320
 321     def clean(self):
 322         """Clean created objects/functions."""
 323         try:
 324             if not self.orchestrator['requirements']['preserve_setup']:
 325                 self.__logger.info("Removing deployment files...")
 326                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 327                 if os.path.exists(testresult):
 328                     os.remove(testresult)
 329                 self.__logger.info("Removing %s file ", self.filename)
 330                 if os.path.exists(self.filename):
 331                     os.remove(self.filename)
 332                 self.__logger.info("Destroying Orchestrator...")
 333                 os.system('juju destroy-controller -y abot-controller '
 334                           '--destroy-all-models')
 335         except Exception:  # pylint: disable=broad-except
 336             self.__logger.warn("Some issue during the undeployment ..")
 337             self.__logger.warn("Tenant clean continue ..")
 338
 339         if not self.orchestrator['requirements']['preserve_setup']:
 340             self.__logger.info('Remove the Abot_epc OS object ..')
 341             for creator in reversed(self.created_object):
 342                 try:
 343                     creator.clean()
 344                 except Exception as exc:  # pylint: disable=broad-except
 345                     self.__logger.error('Unexpected error cleaning - %s', exc)
 346
 347             self.__logger.info("Releasing all the floating IPs")
 348             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 349             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 350                                                self.tenant_name)
 351             self.__logger.info("TENANT ID : %s", tenant_id)
 352             for item in floating_ips:
 353                 if item['tenant_id'] == tenant_id:
 354                     os_utils.delete_floating_ip(self.neutron_client,
 355                                                 item['id'])
 356             self.__logger.info("Cleaning Projects and Users")
 357             for creator in reversed(self.created_object):
 358                 try:
 359                     creator.clean()
 360                 except Exception as exc:  # pylint: disable=broad-except
 361                     self.__logger.error('Unexpected error cleaning - %s', exc)
 362         return True
 363
 364
 365 # ----------------------------------------------------------
 366 #
 367 #               YAML UTILS
 368 #
 369 # -----------------------------------------------------------
 370 def get_config(parameter, file_path):
 371     """
 372     Returns the value of a given parameter in file.yaml
 373     parameter must be given in string format with dots
 374     Example: general.openstack.image_name
 375     """
 376     with open(file_path) as config_file:
 377         file_yaml = yaml.safe_load(config_file)
 378     config_file.close()
 379     value = file_yaml
 380     for element in parameter.split("."):
 381         value = value.get(element)
 382         if value is None:
 383             raise ValueError("The parameter %s is not defined in"
 384                              " reporting.yaml" % parameter)
 385     return value
 386
 387
 388 def sig_test_format(sig_test):
 389     """
 390     Process the signaling result to have a short result
 391     """
 392     nb_passed = 0
 393     nb_failures = 0
 394     nb_skipped = 0
 395     for data_test in sig_test:
 396         if data_test['result'] == "passed":
 397             nb_passed += 1
 398         elif data_test['result'] == "failed":
 399             nb_failures += 1
 400         elif data_test['result'] == "skipped":
 401             nb_skipped += 1
 402     total_sig_test_result = {}
 403     total_sig_test_result['passed'] = nb_passed
 404     total_sig_test_result['failures'] = nb_failures
 405     total_sig_test_result['skipped'] = nb_skipped
 406     return total_sig_test_result
 407
 408
 409 def process_abot_test_result(file_path):
 410     """ Process ABoT Result """
 411     with open(file_path) as test_result:
 412         data = json.load(test_result)
 413         res = []
 414         for tests in data:
 415             tests = update_data(tests)
 416             try:
 417                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 418                 for steps in flatten_steps:
 419                     steps['result'] = steps['step_status']
 420                     res.append(steps)
 421             except:
 422                 logging.error("Could not post data to ElasticSearch host")
 423                 raise
 424         return res
 425
 426
 427 def update_data(obj):
 428     """ Update Result data"""
 429     try:
 430         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 431
 432         for element in obj['elements']:
 433             element['final_result'] = "passed"
 434             element['flatten_steps'] = []
 435
 436             for step in element['steps']:
 437                 temp_dict = {}
 438                 step['result'][step['result']['status']] = 1
 439                 if step['result']['status'].lower() in ['fail', 'failed']:
 440                     element['final_result'] = "failed"
 441
 442                 temp_dict['feature_file'] = obj['feature_file']
 443                 temp_dict['step_name'] = step['name']
 444                 temp_dict['step_status'] = step['result']['status']
 445                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 446                 temp_dict['step_' + step['result']['status']] = 1
 447                 element['flatten_steps'].append(deepcopy(temp_dict))
 448
 449             # Need to put the tag in OBJ and not ELEMENT
 450             if 'tags' in obj:
 451                 element['tags'] = deepcopy(obj['tags'])
 452                 for tag in obj['tags']:
 453                     element[tag['name']] = 1
 454             else:
 455                 for tag in element['tags']:
 456                     element[tag['name']] = 1
 457
 458     except:
 459         logging.error("Error in updating data, %s" % (sys.exc_info()[0]))
 460         raise
 461
 462     return obj
 463
 464
 465 def get_instance_metadata(nova_client, instance):
 466     """ Get instance Metadata - Instance ID """
 467     try:
 468         instance = nova_client.servers.get(instance.id)
 469         return instance.metadata
 470     except Exception as e:
 471         logging.error("Error [get_instance_status(nova_client)]: %s" % e)
 472         return None
 473
 474
 475 CLOUD_TEMPLATE = """clouds:
 476     abot-epc:
 477       type: openstack
 478       auth-types: [userpass]
 479       endpoint: {url}
 480       regions:
 481         RegionOne:
 482           endpoint: {url}
 483 credentials:
 484   abot-epc:
 485     abot-epc:
 486       auth-type: userpass
 487       password: {pass}
 488       tenant-name: {tenant_n}
 489       username: {user_n}"""
 490
 491
 492 def write_config(fname, template, **kwargs):
 493     """ Generate yaml from template for addinh cloud in juju """
 494     with open(fname, 'w') as yfile:
 495         yfile.write(template.format(**kwargs))
 496
 497
 498 def append_config(file_name, p_domain, u_domain):
 499     """ Append values into a yaml file  """
 500     with open(file_name) as yfile:
 501         doc = yaml.load(yfile)
 502     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 503         p_domain)
 504     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 505         u_domain)
 506
 507     with open(file_name, 'w') as yfile:
 508         yaml.safe_dump(doc, yfile, default_flow_style=False)