functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import logging
  12 import shutil
  13 import os
  14 import time
  15 import json
  16 import sys
  17 from copy import deepcopy
  18 from urlparse import urljoin
  19
  20 import functest.core.vnf as vnf
  21 from functest.opnfv_tests.openstack.snaps import snaps_utils
  22 from functest.utils.constants import CONST
  23 import functest.utils.openstack_utils as os_utils
  24
  25 import pkg_resources
  26 from snaps.openstack.os_credentials import OSCreds
  27 from snaps.openstack.create_network import (NetworkSettings,
  28                                             SubnetSettings, OpenStackNetwork)
  29 from snaps.openstack.create_router import (RouterSettings, OpenStackRouter)
  30 from snaps.openstack.create_flavor import (FlavorSettings, OpenStackFlavor)
  31 from snaps.openstack.create_image import (ImageSettings, OpenStackImage)
  32 from snaps.openstack.utils import keystone_utils
  33 import yaml
  34
  35 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  36 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  37
  38
  39 class JujuEpc(vnf.VnfOnBoarding):
  40     # pylint:disable=too-many-instance-attributes
  41     """Abot EPC deployed with JUJU Orchestrator Case"""
  42
  43     __logger = logging.getLogger(__name__)
  44
  45     default_region_name = "RegionOne"
  46
  47     def __init__(self, **kwargs):
  48         if "case_name" not in kwargs:
  49             kwargs["case_name"] = "juju_epc"
  50         super(JujuEpc, self).__init__(**kwargs)
  51
  52         # Retrieve the configuration
  53         self.case_dir = pkg_resources.resource_filename(
  54             'functest', 'opnfv_tests/vnf/epc')
  55         try:
  56             self.config = CONST.__getattribute__(
  57                 'vnf_{}_config'.format(self.case_name))
  58         except Exception:
  59             raise Exception("VNF config file not found")
  60         config_file = os.path.join(self.case_dir, self.config)
  61         self.orchestrator = dict(
  62             requirements=get_config("orchestrator.requirements", config_file),
  63         )
  64
  65         self.created_object = []
  66         self.snaps_creds = ''
  67
  68         self.os_creds = snaps_utils.get_credentials()
  69         self.details['orchestrator'] = dict(
  70             name=get_config("orchestrator.name", config_file),
  71             version=get_config("orchestrator.version", config_file),
  72             status='ERROR',
  73             result=''
  74         )
  75
  76         self.vnf = dict(
  77             descriptor=get_config("vnf.descriptor", config_file),
  78             requirements=get_config("vnf.requirements", config_file)
  79         )
  80         self.details['vnf'] = dict(
  81             descriptor_version=self.vnf['descriptor']['version'],
  82             name=get_config("vnf.name", config_file),
  83             version=get_config("vnf.version", config_file),
  84         )
  85         self.__logger.debug("VNF configuration: %s", self.vnf)
  86
  87         self.details['test_vnf'] = dict(
  88             name=get_config("vnf_test_suite.name", config_file),
  89             version=get_config("vnf_test_suite.version", config_file),
  90             tag_name=get_config("vnf_test_suite.tag_name", config_file)
  91         )
  92         self.images = get_config("tenant_images", config_file)
  93         self.__logger.info("Images needed for vEPC: %s", self.images)
  94         self.keystone_client = os_utils.get_keystone_client()
  95         self.glance_client = os_utils.get_glance_client()
  96         self.neutron_client = os_utils.get_neutron_client()
  97         self.nova_client = os_utils.get_nova_client()
  98         self.sec_group_id = None
  99         self.public_auth_url = None
 100         self.creds = None
 101         self.filename = None
 102
 103     def prepare(self):
 104         """Prepare testcase (Additional pre-configuration steps)."""
 105         self.__logger.debug("OS Credentials: %s", os_utils.get_credentials())
 106
 107         super(JujuEpc, self).prepare()
 108
 109         self.__logger.info("Additional pre-configuration steps")
 110         self.public_auth_url = keystone_utils.get_endpoint(
 111             self.snaps_creds, 'identity')
 112         # it enforces a versioned public identity endpoint as juju simply
 113         # adds /auth/tokens wich fails vs an unversioned endpoint.
 114         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 115             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 116
 117         self.creds = {
 118             "tenant": self.tenant_name,
 119             "username": self.tenant_name,
 120             "password": self.tenant_name,
 121             "auth_url": os_utils.get_credentials()['auth_url']
 122             }
 123
 124         self.snaps_creds = OSCreds(
 125             username=self.creds['username'],
 126             password=self.creds['password'],
 127             auth_url=self.creds['auth_url'],
 128             project_name=self.creds['tenant'],
 129             identity_api_version=int(os_utils.get_keystone_client_version()))
 130
 131         cloud_data = {
 132             'url': self.public_auth_url,
 133             'pass': self.tenant_name,
 134             'tenant_n': self.tenant_name,
 135             'user_n': self.tenant_name,
 136             'region': os.environ.get(
 137                 "OS_REGION_NAME", self.default_region_name)
 138         }
 139         self.__logger.info("Cloud DATA:  %s", cloud_data)
 140         self.filename = os.path.join(self.case_dir, 'abot-epc.yaml')
 141         self.__logger.info("Create  %s to add cloud info", self.filename)
 142         write_config(self.filename, CLOUD_TEMPLATE, **cloud_data)
 143
 144         if self.snaps_creds.identity_api_version == 3:
 145             append_config(
 146                 self.filename, '{}'.format(
 147                     os_utils.get_credentials()['project_domain_name']),
 148                 '{}'.format(os_utils.get_credentials()['user_domain_name']))
 149
 150         self.__logger.info("Upload some OS images if it doesn't exist")
 151         for image_name, image_file in self.images.iteritems():
 152             self.__logger.info("image: %s, file: %s", image_name, image_file)
 153             if image_file and image_name:
 154                 image_creator = OpenStackImage(
 155                     self.snaps_creds,
 156                     ImageSettings(name=image_name,
 157                                   image_user='cloud',
 158                                   img_format='qcow2',
 159                                   image_file=image_file))
 160                 image_creator.create()
 161                 self.created_object.append(image_creator)
 162
 163     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 164         """
 165         Create network, subnet, router
 166
 167         Bootstrap juju
 168         """
 169         self.__logger.info("Deployed Orchestrator")
 170         private_net_name = CONST.__getattribute__(
 171             'vnf_{}_private_net_name'.format(self.case_name))
 172         private_subnet_name = CONST.__getattribute__(
 173             'vnf_{}_private_subnet_name'.format(self.case_name))
 174         private_subnet_cidr = CONST.__getattribute__(
 175             'vnf_{}_private_subnet_cidr'.format(self.case_name))
 176         abot_router = CONST.__getattribute__(
 177             'vnf_{}_external_router'.format(self.case_name))
 178         dns_nameserver = CONST.__getattribute__(
 179             'vnf_{}_dns_nameserver'.format(self.case_name))
 180         ext_net_name = CONST.__getattribute__(
 181             'vnf_{}_external_network_name'.format(self.case_name))
 182
 183         self.__logger.info("Creating full network ...")
 184         subnet_settings = SubnetSettings(name=private_subnet_name,
 185                                          cidr=private_subnet_cidr,
 186                                          dns_nameservers=dns_nameserver)
 187         network_settings = NetworkSettings(name=private_net_name,
 188                                            subnet_settings=[subnet_settings])
 189         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 190         network_creator.create()
 191         self.created_object.append(network_creator)
 192
 193         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 194         self.__logger.info("Creating network Router ....")
 195         router_creator = OpenStackRouter(
 196             self.snaps_creds,
 197             RouterSettings(
 198                 name=abot_router,
 199                 external_gateway=ext_net_name,
 200                 internal_subnets=[subnet_settings.name]))
 201         router_creator.create()
 202         self.created_object.append(router_creator)
 203         self.__logger.info("Creating Flavor ....")
 204         flavor_settings = FlavorSettings(
 205             name=self.orchestrator['requirements']['flavor']['name'],
 206             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 207             disk=10,
 208             vcpus=1)
 209         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 210         self.__logger.info("Juju Bootstrap: Skip creation of flavors")
 211         flavor_creator.create()
 212         self.created_object.append(flavor_creator)
 213         self.__logger.info("Installing Dependency Packages .......")
 214         source_dir = "/src/epc-requirements/juju_bin_build"
 215         if os.path.exists(source_dir):
 216             shutil.rmtree(source_dir)
 217         os.makedirs(source_dir)
 218         os.environ['GOPATH'] = str(source_dir)
 219         os.environ['GOBIN'] = str(source_dir) + "/bin"
 220         os.environ['PATH'] = ((os.path.expandvars('$GOPATH')) + ":" +
 221                               (os.path.expandvars('$GOBIN')) + ":" +
 222                               (os.path.expandvars('$PATH')))
 223         os.system('go get -d -v github.com/juju/juju/...')
 224         os.chdir(source_dir + "/src" + "/github.com" + "/juju" + "/juju")
 225         os.system('git checkout tags/juju-2.2.5')
 226         os.system('go get github.com/rogpeppe/godeps')
 227         os.system('godeps -u dependencies.tsv')
 228         os.system('go install -v github.com/juju/juju/...')
 229         self.__logger.info("Creating Cloud for Abot-epc .....")
 230         os.system('juju add-cloud abot-epc -f {}'.format(self.filename))
 231         os.system('juju add-credential abot-epc -f {}'.format(self.filename))
 232         for image_name in self.images.keys():
 233             self.__logger.info("Generating Metadata for %s", image_name)
 234             image_id = os_utils.get_image_id(self.glance_client, image_name)
 235             os.system(
 236                 'juju metadata generate-image -d ~ -i {} -s {} -r '
 237                 '{} -u {}'.format(
 238                     image_id, image_name,
 239                     os.environ.get("OS_REGION_NAME", self.default_region_name),
 240                     self.public_auth_url))
 241         net_id = os_utils.get_network_id(self.neutron_client, private_net_name)
 242         self.__logger.info("Credential information  : %s", net_id)
 243         juju_bootstrap_command = ('juju bootstrap abot-epc abot-controller '
 244                                   '--config network={} --metadata-source ~  '
 245                                   '--config ssl-hostname-verification=false '
 246                                   '--constraints mem=2G --bootstrap-series '
 247                                   'xenial '
 248                                   '--config use-floating-ip=true --debug'.
 249                                   format(net_id))
 250         os.system(juju_bootstrap_command)
 251         return True
 252
 253     def deploy_vnf(self):
 254         """Deploy ABOT-OAI-EPC."""
 255         self.__logger.info("Upload VNFD")
 256         descriptor = self.vnf['descriptor']
 257         self.__logger.info("Get or create flavor for all Abot-EPC")
 258         flavor_settings = FlavorSettings(
 259             name=self.vnf['requirements']['flavor']['name'],
 260             ram=self.vnf['requirements']['flavor']['ram_min'],
 261             disk=10,
 262             vcpus=1)
 263         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 264         flavor_creator.create()
 265         self.created_object.append(flavor_creator)
 266         self.__logger.info("Deploying Abot-epc bundle file ...")
 267         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 268         self.__logger.info("Waiting for instances .....")
 269         status = os.system('juju-wait')
 270         self.__logger.info("juju wait completed: %s", status)
 271         self.__logger.info("Deployed Abot-epc on Openstack")
 272         if status == 0:
 273             instances = os_utils.get_instances(self.nova_client)
 274             for items in instances:
 275                 metadata = get_instance_metadata(self.nova_client, items)
 276                 if 'juju-units-deployed' in metadata:
 277                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 278                                  '-' + metadata['juju-model-uuid'])
 279                     self.sec_group_id = os_utils.get_security_group_id(
 280                         self.neutron_client, sec_group)
 281                     break
 282             self.__logger.info("Adding Security group rule....")
 283             os_utils.create_secgroup_rule(self.neutron_client,
 284                                           self.sec_group_id, 'ingress', 132)
 285             self.__logger.info("Copying the feature files to Abot_node ")
 286             os.system('juju scp -- -r {}/featureFiles abot-'
 287                       'epc-basic/0:~/'.format(self.case_dir))
 288             self.__logger.info("Copying the feature files in Abot_node ")
 289             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 290                       "~/featureFiles /etc/rebaca-test-suite"
 291                       "/featureFiles'")
 292             count = 0
 293             while count < 10:
 294                 epcstatus = os.system('juju status oai-epc | '
 295                                       'grep {} | grep {} | grep {}'
 296                                       .format('EPC', 'is', 'running'))
 297                 if epcstatus == 0:
 298                     break
 299                 else:
 300                     time.sleep(60)
 301                     count = count + 1
 302             os.system('juju-wait')
 303             return True
 304         return False
 305
 306     def test_vnf(self):
 307         """Run test on ABoT."""
 308         start_time = time.time()
 309         self.__logger.info("Running VNF Test cases....")
 310         os.system('juju run-action abot-epc-basic/0 run '
 311                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 312         os.system('juju-wait')
 313         duration = time.time() - start_time
 314         self.__logger.info("Getting results from Abot node....")
 315         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 316                   'epc-basic/artifacts/TestResults.json {}/.'
 317                   .format(self.case_dir))
 318         self.__logger.info("Parsing the Test results...")
 319         res = (process_abot_test_result('{}/TestResults.'
 320                                         'json'.format(self.case_dir)))
 321         short_result = sig_test_format(res)
 322         self.__logger.info(short_result)
 323         self.details['test_vnf'].update(status='PASS',
 324                                         result=short_result,
 325                                         full_result=res,
 326                                         duration=duration)
 327
 328         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 329                            "%d, Skipped: %d", short_result['passed'],
 330                            short_result['failures'], short_result['skipped'])
 331         return True
 332
 333     def clean(self):
 334         """Clean created objects/functions."""
 335         try:
 336             if not self.orchestrator['requirements']['preserve_setup']:
 337                 self.__logger.info("Removing deployment files...")
 338                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 339                 if os.path.exists(testresult):
 340                     os.remove(testresult)
 341                 self.__logger.info("Removing %s file ", self.filename)
 342                 if os.path.exists(self.filename):
 343                     os.remove(self.filename)
 344                 self.__logger.info("Destroying Orchestrator...")
 345                 os.system('juju destroy-controller -y abot-controller '
 346                           '--destroy-all-models')
 347         except Exception:  # pylint: disable=broad-except
 348             self.__logger.warn("Some issue during the undeployment ..")
 349             self.__logger.warn("Tenant clean continue ..")
 350
 351         if not self.orchestrator['requirements']['preserve_setup']:
 352             self.__logger.info('Remove the Abot_epc OS object ..')
 353             for creator in reversed(self.created_object):
 354                 try:
 355                     creator.clean()
 356                 except Exception as exc:  # pylint: disable=broad-except
 357                     self.__logger.error('Unexpected error cleaning - %s', exc)
 358
 359             self.__logger.info("Releasing all the floating IPs")
 360             floating_ips = os_utils.get_floating_ips(self.neutron_client)
 361             tenant_id = os_utils.get_tenant_id(self.keystone_client,
 362                                                self.tenant_name)
 363             self.__logger.info("TENANT ID : %s", tenant_id)
 364             for item in floating_ips:
 365                 if item['tenant_id'] == tenant_id:
 366                     os_utils.delete_floating_ip(self.neutron_client,
 367                                                 item['id'])
 368             self.__logger.info("Cleaning Projects and Users")
 369             for creator in reversed(self.created_object):
 370                 try:
 371                     creator.clean()
 372                 except Exception as exc:  # pylint: disable=broad-except
 373                     self.__logger.error('Unexpected error cleaning - %s', exc)
 374         return True
 375
 376
 377 # ----------------------------------------------------------
 378 #
 379 #               YAML UTILS
 380 #
 381 # -----------------------------------------------------------
 382 def get_config(parameter, file_path):
 383     """
 384     Returns the value of a given parameter in file.yaml
 385     parameter must be given in string format with dots
 386     Example: general.openstack.image_name
 387     """
 388     with open(file_path) as config_file:
 389         file_yaml = yaml.safe_load(config_file)
 390     config_file.close()
 391     value = file_yaml
 392     for element in parameter.split("."):
 393         value = value.get(element)
 394         if value is None:
 395             raise ValueError("The parameter %s is not defined in"
 396                              " reporting.yaml" % parameter)
 397     return value
 398
 399
 400 def sig_test_format(sig_test):
 401     """
 402     Process the signaling result to have a short result
 403     """
 404     nb_passed = 0
 405     nb_failures = 0
 406     nb_skipped = 0
 407     for data_test in sig_test:
 408         if data_test['result'] == "passed":
 409             nb_passed += 1
 410         elif data_test['result'] == "failed":
 411             nb_failures += 1
 412         elif data_test['result'] == "skipped":
 413             nb_skipped += 1
 414     total_sig_test_result = {}
 415     total_sig_test_result['passed'] = nb_passed
 416     total_sig_test_result['failures'] = nb_failures
 417     total_sig_test_result['skipped'] = nb_skipped
 418     return total_sig_test_result
 419
 420
 421 def process_abot_test_result(file_path):
 422     """ Process ABoT Result """
 423     with open(file_path) as test_result:
 424         data = json.load(test_result)
 425         res = []
 426         for tests in data:
 427             tests = update_data(tests)
 428             try:
 429                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 430                 for steps in flatten_steps:
 431                     steps['result'] = steps['step_status']
 432                     res.append(steps)
 433             except:
 434                 logging.error("Could not post data to ElasticSearch host")
 435                 raise
 436         return res
 437
 438
 439 def update_data(obj):
 440     """ Update Result data"""
 441     try:
 442         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 443
 444         for element in obj['elements']:
 445             element['final_result'] = "passed"
 446             element['flatten_steps'] = []
 447
 448             for step in element['steps']:
 449                 temp_dict = {}
 450                 step['result'][step['result']['status']] = 1
 451                 if step['result']['status'].lower() in ['fail', 'failed']:
 452                     element['final_result'] = "failed"
 453
 454                 temp_dict['feature_file'] = obj['feature_file']
 455                 temp_dict['step_name'] = step['name']
 456                 temp_dict['step_status'] = step['result']['status']
 457                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 458                 temp_dict['step_' + step['result']['status']] = 1
 459                 element['flatten_steps'].append(deepcopy(temp_dict))
 460
 461             # Need to put the tag in OBJ and not ELEMENT
 462             if 'tags' in obj:
 463                 element['tags'] = deepcopy(obj['tags'])
 464                 for tag in obj['tags']:
 465                     element[tag['name']] = 1
 466             else:
 467                 for tag in element['tags']:
 468                     element[tag['name']] = 1
 469
 470     except Exception:  # pylint: disable=broad-except
 471         logging.error("Error in updating data, %s", sys.exc_info()[0])
 472         raise
 473
 474     return obj
 475
 476
 477 def get_instance_metadata(nova_client, instance):
 478     """ Get instance Metadata - Instance ID """
 479     try:
 480         instance = nova_client.servers.get(instance.id)
 481         return instance.metadata
 482     except Exception as exc:  # pylint: disable=broad-except
 483         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 484         return None
 485
 486
 487 CLOUD_TEMPLATE = """clouds:
 488     abot-epc:
 489       type: openstack
 490       auth-types: [userpass]
 491       endpoint: {url}
 492       regions:
 493         {region}:
 494           endpoint: {url}
 495 credentials:
 496   abot-epc:
 497     abot-epc:
 498       auth-type: userpass
 499       password: {pass}
 500       tenant-name: {tenant_n}
 501       username: {user_n}"""
 502
 503
 504 def write_config(fname, template, **kwargs):
 505     """ Generate yaml from template for addinh cloud in juju """
 506     with open(fname, 'w') as yfile:
 507         yfile.write(template.format(**kwargs))
 508
 509
 510 def append_config(file_name, p_domain, u_domain):
 511     """ Append values into a yaml file  """
 512     with open(file_name) as yfile:
 513         doc = yaml.load(yfile)
 514     doc['credentials']['abot-epc']['abot-epc']['project-domain-name'] = (
 515         p_domain)
 516     doc['credentials']['abot-epc']['abot-epc']['user-domain-name'] = (
 517         u_domain)
 518
 519     with open(file_name, 'w') as yfile:
 520         yaml.safe_dump(doc, yfile, default_flow_style=False)