functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20 import pkg_resources
  21 import yaml
  22
  23 from functest.core import vnf
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.utils import config
  26 from functest.utils import env
  27
  28 from snaps.config.flavor import FlavorConfig
  29 from snaps.config.image import ImageConfig
  30 from snaps.config.network import NetworkConfig, SubnetConfig
  31 from snaps.config.router import RouterConfig
  32 from snaps.config.security_group import (
  33     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  34 from snaps.config.user import UserConfig
  35 from snaps.openstack.create_flavor import OpenStackFlavor
  36 from snaps.openstack.create_image import OpenStackImage
  37 from snaps.openstack.create_network import OpenStackNetwork
  38 from snaps.openstack.create_router import OpenStackRouter
  39 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  40 from snaps.openstack.create_user import OpenStackUser
  41 from snaps.openstack.utils import keystone_utils
  42 from snaps.openstack.utils import nova_utils
  43
  44 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  45 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  46
  47 CLOUD_TEMPLATE = """clouds:
  48   abot-epc:
  49     type: openstack
  50     auth-types: [userpass]
  51     endpoint: {url}
  52     regions:
  53       {region}:
  54         endpoint: {url}"""
  55
  56 CREDS_TEMPLATE2 = """credentials:
  57   abot-epc:
  58     default-credential: abot-epc
  59     abot-epc:
  60       auth-type: userpass
  61       password: {pass}
  62       project-domain-name: {project_domain_n}
  63       tenant-name: {tenant_n}"""
  64
  65 CREDS_TEMPLATE3 = """credentials:
  66   abot-epc:
  67     default-credential: abot-epc
  68     abot-epc:
  69       auth-type: userpass
  70       password: {pass}
  71       project-domain-name: {project_domain_n}
  72       tenant-name: {tenant_n}
  73       user-domain-name: {user_domain_n}
  74       username: {user_n}"""
  75
  76
  77 class JujuEpc(vnf.VnfOnBoarding):
  78     # pylint:disable=too-many-instance-attributes
  79     """Abot EPC deployed with JUJU Orchestrator Case"""
  80
  81     __logger = logging.getLogger(__name__)
  82
  83     def __init__(self, **kwargs):
  84         if "case_name" not in kwargs:
  85             kwargs["case_name"] = "juju_epc"
  86         super(JujuEpc, self).__init__(**kwargs)
  87
  88         # Retrieve the configuration
  89         self.case_dir = pkg_resources.resource_filename(
  90             'functest', 'opnfv_tests/vnf/epc')
  91         try:
  92             self.config = getattr(
  93                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  94         except Exception:
  95             raise Exception("VNF config file not found")
  96         self.config_file = os.path.join(self.case_dir, self.config)
  97         self.orchestrator = dict(requirements=get_config(
  98             "orchestrator.requirements", self.config_file))
  99
 100         self.created_object = []
 101         self.details['orchestrator'] = dict(
 102             name=get_config("orchestrator.name", self.config_file),
 103             version=get_config("orchestrator.version", self.config_file),
 104             status='ERROR',
 105             result=''
 106         )
 107
 108         self.vnf = dict(
 109             descriptor=get_config("vnf.descriptor", self.config_file),
 110             requirements=get_config("vnf.requirements", self.config_file)
 111         )
 112         self.details['vnf'] = dict(
 113             descriptor_version=self.vnf['descriptor']['version'],
 114             name=get_config("vnf.name", self.config_file),
 115             version=get_config("vnf.version", self.config_file),
 116         )
 117         self.__logger.debug("VNF configuration: %s", self.vnf)
 118
 119         self.details['test_vnf'] = dict(
 120             name=get_config("vnf_test_suite.name", self.config_file),
 121             version=get_config("vnf_test_suite.version", self.config_file),
 122             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 123         )
 124         self.public_auth_url = None
 125
 126         self.res_dir = os.path.join(
 127             getattr(config.CONF, 'dir_results'), self.case_name)
 128
 129     def _bypass_juju_network_discovery_bug(self, name):
 130         user_creator = OpenStackUser(
 131             self.snaps_creds,
 132             UserConfig(
 133                 name=name, password=str(uuid.uuid4()),
 134                 domain=self.snaps_creds.user_domain_name,
 135                 roles={'_member_': self.tenant_name}))
 136         user_creator.create()
 137         self.created_object.append(user_creator)
 138         return user_creator
 139
 140     def _register_cloud(self):
 141         self.__logger.info("Creating Cloud for Abot-epc .....")
 142         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 143         # It allows gating APEX and ensures this testcase is working till
 144         # https://jira.opnfv.org/browse/APEX-570 is fixed in APEX.
 145         # It must be removed as soon as possible to disable per installer
 146         # processing in Functest.
 147         region = self.snaps_creds.region_name
 148         if not region and env.get('INSTALLER_TYPE') == 'apex':
 149             region = "regionOne"
 150         cloud_data = {
 151             'url': self.public_auth_url,
 152             'region': region}
 153         with open(clouds_yaml, 'w') as yfile:
 154             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 155         if os.system(
 156                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 157             raise vnf.VnfPreparationException
 158
 159     def _register_credentials_v2(self):
 160         self.__logger.info("Creating Credentials for Abot-epc .....")
 161         user_creator = self._bypass_juju_network_discovery_bug(
 162             'juju_network_discovery_bug')
 163         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 164         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 165         creds_data = {
 166             'pass': snaps_creds.password,
 167             'tenant_n': snaps_creds.project_name,
 168             'user_n': snaps_creds.username}
 169         with open(credentials_yaml, 'w') as yfile:
 170             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 171         if os.system(
 172                 'juju add-credential abot-epc -f {} --replace'.format(
 173                     credentials_yaml)):
 174             raise vnf.VnfPreparationException
 175
 176     def _register_credentials_v3(self):
 177         self.__logger.info("Creating Credentials for Abot-epc .....")
 178         user_creator = self._bypass_juju_network_discovery_bug(
 179             'juju_network_discovery_bug')
 180         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 181         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 182         creds_data = {
 183             'pass': snaps_creds.password,
 184             'tenant_n': snaps_creds.project_name,
 185             'user_n': snaps_creds.username,
 186             'project_domain_n': snaps_creds.project_domain_name,
 187             'user_domain_n': snaps_creds.user_domain_name}
 188         with open(credentials_yaml, 'w') as yfile:
 189             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 190         if os.system(
 191                 'juju add-credential abot-epc -f {} --replace'.format(
 192                     credentials_yaml)):
 193             raise vnf.VnfPreparationException
 194
 195     def _add_custom_rule(self, sec_grp_name):
 196         """ To add custom rule for SCTP Traffic """
 197         sec_grp_rules = list()
 198         sec_grp_rules.append(
 199             SecurityGroupRuleConfig(
 200                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 201                 protocol=Protocol.sctp))
 202         security_group = OpenStackSecurityGroup(
 203             self.snaps_creds,
 204             SecurityGroupConfig(
 205                 name=sec_grp_name,
 206                 rule_settings=sec_grp_rules))
 207         security_group.create()
 208         self.created_object.append(security_group)
 209
 210     def prepare(self):
 211         """Prepare testcase (Additional pre-configuration steps)."""
 212         self.__logger.info("Additional pre-configuration steps")
 213         super(JujuEpc, self).prepare()
 214         try:
 215             os.makedirs(self.res_dir)
 216         except OSError as ex:
 217             if ex.errno != errno.EEXIST:
 218                 self.__logger.exception("Cannot create %s", self.res_dir)
 219                 raise vnf.VnfPreparationException
 220         self.public_auth_url = keystone_utils.get_endpoint(
 221             self.snaps_creds, 'identity')
 222         # it enforces a versioned public identity endpoint as juju simply
 223         # adds /auth/tokens wich fails vs an unversioned endpoint.
 224         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 225             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 226         self._register_cloud()
 227         if self.snaps_creds.identity_api_version == 3:
 228             self._register_credentials_v3()
 229         else:
 230             self._register_credentials_v2()
 231
 232     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 233         """
 234         Create network, subnet, router
 235
 236         Bootstrap juju
 237         """
 238         self.__logger.info("Deployed Orchestrator")
 239         private_net_name = getattr(
 240             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 241         private_subnet_name = '{}-{}'.format(
 242             getattr(config.CONF,
 243                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 244             self.uuid)
 245         private_subnet_cidr = getattr(
 246             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 247         abot_router = '{}-{}'.format(
 248             getattr(config.CONF,
 249                     'vnf_{}_external_router'.format(self.case_name)),
 250             self.uuid)
 251         self.__logger.info("Creating full network ...")
 252         subnet_settings = SubnetConfig(
 253             name=private_subnet_name, cidr=private_subnet_cidr)
 254         network_settings = NetworkConfig(
 255             name=private_net_name, subnet_settings=[subnet_settings])
 256         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 257         net_id = network_creator.create().id
 258         self.created_object.append(network_creator)
 259
 260         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 261         self.__logger.info("Creating network Router ....")
 262         router_creator = OpenStackRouter(
 263             self.snaps_creds, RouterConfig(
 264                 name=abot_router,
 265                 external_gateway=ext_net_name,
 266                 internal_subnets=[subnet_settings.name]))
 267         router_creator.create()
 268         self.created_object.append(router_creator)
 269         self.__logger.info("Creating Flavor ....")
 270         flavor_settings = FlavorConfig(
 271             name=self.orchestrator['requirements']['flavor']['name'],
 272             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 273             disk=10, vcpus=1)
 274         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 275         flavor_creator.create()
 276         self.created_object.append(flavor_creator)
 277         self.__logger.info("Upload some OS images if it doesn't exist")
 278         images = get_config("tenant_images", self.config_file)
 279         self.__logger.info("Images needed for vEPC: %s", images)
 280         for image_name, image_file in images.iteritems():
 281             self.__logger.info("image: %s, file: %s", image_name, image_file)
 282             if image_file and image_name:
 283                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 284                     name=image_name, image_user='cloud', img_format='qcow2',
 285                     image_file=image_file))
 286                 image_id = image_creator.create().id
 287                 os.system(
 288                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 289                     '{} -u {}'.format(
 290                         image_id, image_name, self.snaps_creds.region_name,
 291                         self.public_auth_url))
 292                 self.created_object.append(image_creator)
 293         self.__logger.info("Network ID  : %s", net_id)
 294         juju_bootstrap_command = (
 295             'juju bootstrap abot-epc abot-controller --config network={} '
 296             '--metadata-source ~  --config ssl-hostname-verification=false '
 297             '--constraints mem=2G --bootstrap-series xenial '
 298             '--config use-floating-ip=true --debug '
 299             '--config use-default-secgroup=true'.format(net_id))
 300         os.system(juju_bootstrap_command)
 301         return True
 302
 303     def deploy_vnf(self):
 304         """Deploy ABOT-OAI-EPC."""
 305         self.__logger.info("Upload VNFD")
 306         descriptor = self.vnf['descriptor']
 307         self.__logger.info("Get or create flavor for all Abot-EPC")
 308         flavor_settings = FlavorConfig(
 309             name=self.vnf['requirements']['flavor']['name'],
 310             ram=self.vnf['requirements']['flavor']['ram_min'],
 311             disk=10,
 312             vcpus=1)
 313         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 314         flavor_creator.create()
 315         self.created_object.append(flavor_creator)
 316         self.__logger.info("Deploying Abot-epc bundle file ...")
 317         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 318         self.__logger.info("Waiting for instances .....")
 319         status = os.system('juju-wait')
 320         self.__logger.info("juju wait completed: %s", status)
 321         self.__logger.info("Deployed Abot-epc on Openstack")
 322         nova_client = nova_utils.nova_client(self.snaps_creds)
 323         if status == 0:
 324             instances = get_instances(nova_client)
 325             self.__logger.info("List of Instance: %s", instances)
 326             for items in instances:
 327                 metadata = get_instance_metadata(nova_client, items)
 328                 if 'juju-units-deployed' in metadata:
 329                     sec_group = ('juju-' +
 330                                  metadata['juju-controller-uuid'] +
 331                                  '-' + metadata['juju-model-uuid'])
 332                     self.__logger.info("Instance: %s", sec_group)
 333                     break
 334             self.__logger.info("Adding Security group rule....")
 335             # This will add sctp rule to a common Security Group Created
 336             # by juju and shared to all deployed units.
 337             self._add_custom_rule(sec_group)
 338             self.__logger.info("Copying the feature files to Abot_node ")
 339             os.system('juju scp -- -r {}/featureFiles abot-'
 340                       'epc-basic/0:~/'.format(self.case_dir))
 341             self.__logger.info("Copying the feature files in Abot_node ")
 342             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 343                       "~/featureFiles /etc/rebaca-test-suite"
 344                       "/featureFiles'")
 345             count = 0
 346             while count < 10:
 347                 epcstatus = os.system('juju status oai-epc | '
 348                                       'grep {} | grep {} | grep {}'
 349                                       .format('EPC', 'is', 'running'))
 350                 if epcstatus == 0:
 351                     break
 352                 else:
 353                     time.sleep(60)
 354                     count = count + 1
 355             os.system('juju-wait')
 356             return True
 357         return False
 358
 359     def test_vnf(self):
 360         """Run test on ABoT."""
 361         start_time = time.time()
 362         self.__logger.info("Running VNF Test cases....")
 363         os.system('juju run-action abot-epc-basic/0 run '
 364                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 365         os.system('juju-wait')
 366         duration = time.time() - start_time
 367         self.__logger.info("Getting results from Abot node....")
 368         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 369                   'epc-basic/artifacts/TestResults.json {}/.'
 370                   .format(self.case_dir))
 371         self.__logger.info("Parsing the Test results...")
 372         res = (process_abot_test_result('{}/TestResults.'
 373                                         'json'.format(self.case_dir)))
 374         short_result = sig_test_format(res)
 375         self.__logger.info(short_result)
 376         self.details['test_vnf'].update(status='PASS',
 377                                         result=short_result,
 378                                         full_result=res,
 379                                         duration=duration)
 380
 381         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 382                            "%d, Skipped: %d", short_result['passed'],
 383                            short_result['failures'], short_result['skipped'])
 384         return True
 385
 386     def clean(self):
 387         """Clean created objects/functions."""
 388         try:
 389             if not self.orchestrator['requirements']['preserve_setup']:
 390                 self.__logger.info("Removing deployment files...")
 391                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 392                 if os.path.exists(testresult):
 393                     os.remove(testresult)
 394                 self.__logger.info("Destroying Orchestrator...")
 395                 os.system('juju destroy-controller -y abot-controller '
 396                           '--destroy-all-models')
 397         except Exception:  # pylint: disable=broad-except
 398             self.__logger.warn("Some issue during the undeployment ..")
 399             self.__logger.warn("Tenant clean continue ..")
 400
 401         if not self.orchestrator['requirements']['preserve_setup']:
 402             self.__logger.info('Remove the Abot_epc OS object ..')
 403             super(JujuEpc, self).clean()
 404
 405         return True
 406
 407
 408 # ----------------------------------------------------------
 409 #
 410 #               YAML UTILS
 411 #
 412 # -----------------------------------------------------------
 413 def get_config(parameter, file_path):
 414     """
 415     Returns the value of a given parameter in file.yaml
 416     parameter must be given in string format with dots
 417     Example: general.openstack.image_name
 418     """
 419     with open(file_path) as config_file:
 420         file_yaml = yaml.safe_load(config_file)
 421     config_file.close()
 422     value = file_yaml
 423     for element in parameter.split("."):
 424         value = value.get(element)
 425         if value is None:
 426             raise ValueError("The parameter %s is not defined in"
 427                              " reporting.yaml" % parameter)
 428     return value
 429
 430
 431 def sig_test_format(sig_test):
 432     """
 433     Process the signaling result to have a short result
 434     """
 435     nb_passed = 0
 436     nb_failures = 0
 437     nb_skipped = 0
 438     for data_test in sig_test:
 439         if data_test['result'] == "passed":
 440             nb_passed += 1
 441         elif data_test['result'] == "failed":
 442             nb_failures += 1
 443         elif data_test['result'] == "skipped":
 444             nb_skipped += 1
 445     total_sig_test_result = {}
 446     total_sig_test_result['passed'] = nb_passed
 447     total_sig_test_result['failures'] = nb_failures
 448     total_sig_test_result['skipped'] = nb_skipped
 449     return total_sig_test_result
 450
 451
 452 def process_abot_test_result(file_path):
 453     """ Process ABoT Result """
 454     with open(file_path) as test_result:
 455         data = json.load(test_result)
 456         res = []
 457         for tests in data:
 458             tests = update_data(tests)
 459             try:
 460                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 461                 for steps in flatten_steps:
 462                     steps['result'] = steps['step_status']
 463                     res.append(steps)
 464             except:
 465                 logging.error("Could not post data to ElasticSearch host")
 466                 raise
 467         return res
 468
 469
 470 def update_data(obj):
 471     """ Update Result data"""
 472     try:
 473         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 474
 475         for element in obj['elements']:
 476             element['final_result'] = "passed"
 477             element['flatten_steps'] = []
 478
 479             for step in element['steps']:
 480                 temp_dict = {}
 481                 step['result'][step['result']['status']] = 1
 482                 if step['result']['status'].lower() in ['fail', 'failed']:
 483                     element['final_result'] = "failed"
 484
 485                 temp_dict['feature_file'] = obj['feature_file']
 486                 temp_dict['step_name'] = step['name']
 487                 temp_dict['step_status'] = step['result']['status']
 488                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 489                 temp_dict['step_' + step['result']['status']] = 1
 490                 element['flatten_steps'].append(deepcopy(temp_dict))
 491
 492             # Need to put the tag in OBJ and not ELEMENT
 493             if 'tags' in obj:
 494                 element['tags'] = deepcopy(obj['tags'])
 495                 for tag in obj['tags']:
 496                     element[tag['name']] = 1
 497             else:
 498                 for tag in element['tags']:
 499                     element[tag['name']] = 1
 500
 501     except Exception:  # pylint: disable=broad-except
 502         logging.error("Error in updating data, %s", sys.exc_info()[0])
 503         raise
 504
 505     return obj
 506
 507
 508 def get_instances(nova_client):
 509     """ To get all vm info of a project """
 510     try:
 511         instances = nova_client.servers.list()
 512         return instances
 513     except Exception as exc:  # pylint: disable=broad-except
 514         logging.error("Error [get_instances(nova_client)]: %s", exc)
 515         return None
 516
 517
 518 def get_instance_metadata(nova_client, instance):
 519     """ Get instance Metadata - Instance ID """
 520     try:
 521         instance = nova_client.servers.get(instance.id)
 522         return instance.metadata
 523     except Exception as exc:  # pylint: disable=broad-except
 524         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 525         return None