functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20 import pkg_resources
  21 import yaml
  22
  23 from functest.core import vnf
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.utils.constants import CONST
  26
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.security_group import (
  32     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  33 from snaps.config.user import UserConfig
  34 from snaps.openstack.create_flavor import OpenStackFlavor
  35 from snaps.openstack.create_image import OpenStackImage
  36 from snaps.openstack.create_network import OpenStackNetwork
  37 from snaps.openstack.create_router import OpenStackRouter
  38 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  39 from snaps.openstack.create_user import OpenStackUser
  40 from snaps.openstack.utils import keystone_utils
  41 from snaps.openstack.utils import nova_utils
  42
  43 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  44 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  45
  46 CLOUD_TEMPLATE = """clouds:
  47   abot-epc:
  48     type: openstack
  49     auth-types: [userpass]
  50     endpoint: {url}
  51     regions:
  52       {region}:
  53         endpoint: {url}"""
  54
  55 CREDS_TEMPLATE2 = """credentials:
  56   abot-epc:
  57     default-credential: abot-epc
  58     abot-epc:
  59       auth-type: userpass
  60       password: {pass}
  61       project-domain-name: {project_domain_n}
  62       tenant-name: {tenant_n}"""
  63
  64 CREDS_TEMPLATE3 = """credentials:
  65   abot-epc:
  66     default-credential: abot-epc
  67     abot-epc:
  68       auth-type: userpass
  69       password: {pass}
  70       project-domain-name: {project_domain_n}
  71       tenant-name: {tenant_n}
  72       user-domain-name: {user_domain_n}
  73       username: {user_n}"""
  74
  75
  76 class JujuEpc(vnf.VnfOnBoarding):
  77     # pylint:disable=too-many-instance-attributes
  78     """Abot EPC deployed with JUJU Orchestrator Case"""
  79
  80     __logger = logging.getLogger(__name__)
  81
  82     default_region_name = "RegionOne"
  83
  84     def __init__(self, **kwargs):
  85         if "case_name" not in kwargs:
  86             kwargs["case_name"] = "juju_epc"
  87         super(JujuEpc, self).__init__(**kwargs)
  88
  89         # Retrieve the configuration
  90         self.case_dir = pkg_resources.resource_filename(
  91             'functest', 'opnfv_tests/vnf/epc')
  92         try:
  93             self.config = getattr(
  94                 CONST, 'vnf_{}_config'.format(self.case_name))
  95         except Exception:
  96             raise Exception("VNF config file not found")
  97         self.config_file = os.path.join(self.case_dir, self.config)
  98         self.orchestrator = dict(requirements=get_config(
  99             "orchestrator.requirements", self.config_file))
 100
 101         self.created_object = []
 102         self.details['orchestrator'] = dict(
 103             name=get_config("orchestrator.name", self.config_file),
 104             version=get_config("orchestrator.version", self.config_file),
 105             status='ERROR',
 106             result=''
 107         )
 108
 109         self.vnf = dict(
 110             descriptor=get_config("vnf.descriptor", self.config_file),
 111             requirements=get_config("vnf.requirements", self.config_file)
 112         )
 113         self.details['vnf'] = dict(
 114             descriptor_version=self.vnf['descriptor']['version'],
 115             name=get_config("vnf.name", self.config_file),
 116             version=get_config("vnf.version", self.config_file),
 117         )
 118         self.__logger.debug("VNF configuration: %s", self.vnf)
 119
 120         self.details['test_vnf'] = dict(
 121             name=get_config("vnf_test_suite.name", self.config_file),
 122             version=get_config("vnf_test_suite.version", self.config_file),
 123             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 124         )
 125         self.public_auth_url = None
 126
 127         self.res_dir = os.path.join(
 128             getattr(CONST, 'dir_results'), self.case_name)
 129
 130     def _bypass_juju_network_discovery_bug(self, name):
 131         user_creator = OpenStackUser(
 132             self.snaps_creds,
 133             UserConfig(
 134                 name=name, password=str(uuid.uuid4()),
 135                 roles={'_member_': self.tenant_name}))
 136         user_creator.create()
 137         self.created_object.append(user_creator)
 138         return user_creator
 139
 140     def _register_cloud(self):
 141         self.__logger.info("Creating Cloud for Abot-epc .....")
 142         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 143         cloud_data = {
 144             'url': self.public_auth_url,
 145             'region': os.environ.get(
 146                 "OS_REGION_NAME", self.default_region_name)}
 147         with open(clouds_yaml, 'w') as yfile:
 148             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 149         if os.system(
 150                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 151             raise vnf.VnfPreparationException
 152
 153     def _register_credentials_v2(self):
 154         self.__logger.info("Creating Credentials for Abot-epc .....")
 155         user_creator = self._bypass_juju_network_discovery_bug(
 156             'juju_network_discovery_bug')
 157         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 158         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 159         creds_data = {
 160             'pass': snaps_creds.password,
 161             'tenant_n': snaps_creds.project_name,
 162             'user_n': snaps_creds.username}
 163         with open(credentials_yaml, 'w') as yfile:
 164             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 165         if os.system(
 166                 'juju add-credential abot-epc -f {} --replace'.format(
 167                     credentials_yaml)):
 168             raise vnf.VnfPreparationException
 169
 170     def _register_credentials_v3(self):
 171         self.__logger.info("Creating Credentials for Abot-epc .....")
 172         user_creator = self._bypass_juju_network_discovery_bug(
 173             'juju_network_discovery_bug')
 174         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 175         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 176         creds_data = {
 177             'pass': snaps_creds.password,
 178             'tenant_n': snaps_creds.project_name,
 179             'user_n': snaps_creds.username,
 180             'project_domain_n': snaps_creds.project_domain_name,
 181             'user_domain_n': snaps_creds.user_domain_name}
 182         with open(credentials_yaml, 'w') as yfile:
 183             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 184         if os.system(
 185                 'juju add-credential abot-epc -f {} --replace'.format(
 186                     credentials_yaml)):
 187             raise vnf.VnfPreparationException
 188
 189     def _add_custom_rule(self, sec_grp_name):
 190         """ To add custom rule for SCTP Traffic """
 191         sec_grp_rules = list()
 192         security_group_init = OpenStackSecurityGroup(
 193             self.snaps_creds,
 194             SecurityGroupConfig(
 195                 name=sec_grp_name,
 196                 rule_settings=sec_grp_rules))
 197         security_group_init.initialize()
 198         sctp_rule = SecurityGroupRuleConfig(
 199             sec_grp_name=sec_grp_name, direction=Direction.ingress,
 200             protocol=Protocol.sctp)
 201         security_group_init.add_rule(sctp_rule)
 202         self.created_object.append(security_group_init)
 203
 204     def prepare(self):
 205         """Prepare testcase (Additional pre-configuration steps)."""
 206         self.__logger.info("Additional pre-configuration steps")
 207         super(JujuEpc, self).prepare()
 208         try:
 209             os.makedirs(self.res_dir)
 210         except OSError as ex:
 211             if ex.errno != errno.EEXIST:
 212                 self.__logger.exception("Cannot create %s", self.res_dir)
 213                 raise vnf.VnfPreparationException
 214         self.public_auth_url = keystone_utils.get_endpoint(
 215             self.snaps_creds, 'identity')
 216         # it enforces a versioned public identity endpoint as juju simply
 217         # adds /auth/tokens wich fails vs an unversioned endpoint.
 218         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 219             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 220         self._register_cloud()
 221         if self.snaps_creds.identity_api_version == 3:
 222             self._register_credentials_v3()
 223         else:
 224             self._register_credentials_v2()
 225
 226     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 227         """
 228         Create network, subnet, router
 229
 230         Bootstrap juju
 231         """
 232         self.__logger.info("Deployed Orchestrator")
 233         private_net_name = getattr(
 234             CONST, 'vnf_{}_private_net_name'.format(self.case_name))
 235         private_subnet_name = getattr(
 236             CONST, 'vnf_{}_private_subnet_name'.format(self.case_name))
 237         private_subnet_cidr = getattr(
 238             CONST, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 239         abot_router = getattr(
 240             CONST, 'vnf_{}_external_router'.format(self.case_name))
 241
 242         self.__logger.info("Creating full network ...")
 243         subnet_settings = SubnetConfig(
 244             name=private_subnet_name, cidr=private_subnet_cidr)
 245         network_settings = NetworkConfig(
 246             name=private_net_name, subnet_settings=[subnet_settings])
 247         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 248         net_id = network_creator.create().id
 249         self.created_object.append(network_creator)
 250
 251         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 252         self.__logger.info("Creating network Router ....")
 253         router_creator = OpenStackRouter(
 254             self.snaps_creds, RouterConfig(
 255                 name=abot_router,
 256                 external_gateway=ext_net_name,
 257                 internal_subnets=[subnet_settings.name]))
 258         router_creator.create()
 259         self.created_object.append(router_creator)
 260         self.__logger.info("Creating Flavor ....")
 261         flavor_settings = FlavorConfig(
 262             name=self.orchestrator['requirements']['flavor']['name'],
 263             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 264             disk=10, vcpus=1)
 265         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 266         flavor_creator.create()
 267         self.created_object.append(flavor_creator)
 268         self.__logger.info("Upload some OS images if it doesn't exist")
 269         images = get_config("tenant_images", self.config_file)
 270         self.__logger.info("Images needed for vEPC: %s", images)
 271         for image_name, image_file in images.iteritems():
 272             self.__logger.info("image: %s, file: %s", image_name, image_file)
 273             if image_file and image_name:
 274                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 275                     name=image_name, image_user='cloud', img_format='qcow2',
 276                     image_file=image_file))
 277                 image_id = image_creator.create().id
 278                 os.system(
 279                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 280                     '{} -u {}'.format(
 281                         image_id, image_name,
 282                         os.environ.get(
 283                             "OS_REGION_NAME", self.default_region_name),
 284                         self.public_auth_url))
 285                 self.created_object.append(image_creator)
 286         self.__logger.info("Network ID  : %s", net_id)
 287         juju_bootstrap_command = (
 288             'juju bootstrap abot-epc abot-controller --config network={} '
 289             '--metadata-source ~  --config ssl-hostname-verification=false '
 290             '--constraints mem=2G --bootstrap-series xenial '
 291             '--config use-floating-ip=true --debug '
 292             '--config use-default-secgroup=true'.format(net_id))
 293         os.system(juju_bootstrap_command)
 294         return True
 295
 296     def deploy_vnf(self):
 297         """Deploy ABOT-OAI-EPC."""
 298         self.__logger.info("Upload VNFD")
 299         descriptor = self.vnf['descriptor']
 300         self.__logger.info("Get or create flavor for all Abot-EPC")
 301         flavor_settings = FlavorConfig(
 302             name=self.vnf['requirements']['flavor']['name'],
 303             ram=self.vnf['requirements']['flavor']['ram_min'],
 304             disk=10,
 305             vcpus=1)
 306         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 307         flavor_creator.create()
 308         self.created_object.append(flavor_creator)
 309         self.__logger.info("Deploying Abot-epc bundle file ...")
 310         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 311         self.__logger.info("Waiting for instances .....")
 312         status = os.system('juju-wait')
 313         self.__logger.info("juju wait completed: %s", status)
 314         self.__logger.info("Deployed Abot-epc on Openstack")
 315         nova_client = nova_utils.nova_client(self.snaps_creds)
 316         if status == 0:
 317             instances = get_instances(nova_client)
 318             self.__logger.info("List of Instance: %s", instances)
 319             for items in instances:
 320                 metadata = get_instance_metadata(nova_client, items)
 321                 if 'juju-units-deployed' in metadata:
 322                     sec_group = ('juju-' +
 323                                  metadata['juju-controller-uuid'] +
 324                                  '-' + metadata['juju-model-uuid'])
 325                     self.__logger.info("Instance: %s", sec_group)
 326                     break
 327             self.__logger.info("Adding Security group rule....")
 328             # This will add sctp rule to a common Security Group Created
 329             # by juju and shared to all deployed units.
 330             self._add_custom_rule(sec_group)
 331             self.__logger.info("Copying the feature files to Abot_node ")
 332             os.system('juju scp -- -r {}/featureFiles abot-'
 333                       'epc-basic/0:~/'.format(self.case_dir))
 334             self.__logger.info("Copying the feature files in Abot_node ")
 335             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 336                       "~/featureFiles /etc/rebaca-test-suite"
 337                       "/featureFiles'")
 338             count = 0
 339             while count < 10:
 340                 epcstatus = os.system('juju status oai-epc | '
 341                                       'grep {} | grep {} | grep {}'
 342                                       .format('EPC', 'is', 'running'))
 343                 if epcstatus == 0:
 344                     break
 345                 else:
 346                     time.sleep(60)
 347                     count = count + 1
 348             os.system('juju-wait')
 349             return True
 350         return False
 351
 352     def test_vnf(self):
 353         """Run test on ABoT."""
 354         start_time = time.time()
 355         self.__logger.info("Running VNF Test cases....")
 356         os.system('juju run-action abot-epc-basic/0 run '
 357                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 358         os.system('juju-wait')
 359         duration = time.time() - start_time
 360         self.__logger.info("Getting results from Abot node....")
 361         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 362                   'epc-basic/artifacts/TestResults.json {}/.'
 363                   .format(self.case_dir))
 364         self.__logger.info("Parsing the Test results...")
 365         res = (process_abot_test_result('{}/TestResults.'
 366                                         'json'.format(self.case_dir)))
 367         short_result = sig_test_format(res)
 368         self.__logger.info(short_result)
 369         self.details['test_vnf'].update(status='PASS',
 370                                         result=short_result,
 371                                         full_result=res,
 372                                         duration=duration)
 373
 374         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 375                            "%d, Skipped: %d", short_result['passed'],
 376                            short_result['failures'], short_result['skipped'])
 377         return True
 378
 379     def clean(self):
 380         """Clean created objects/functions."""
 381         try:
 382             if not self.orchestrator['requirements']['preserve_setup']:
 383                 self.__logger.info("Removing deployment files...")
 384                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 385                 if os.path.exists(testresult):
 386                     os.remove(testresult)
 387                 self.__logger.info("Destroying Orchestrator...")
 388                 os.system('juju destroy-controller -y abot-controller '
 389                           '--destroy-all-models')
 390         except Exception:  # pylint: disable=broad-except
 391             self.__logger.warn("Some issue during the undeployment ..")
 392             self.__logger.warn("Tenant clean continue ..")
 393
 394         if not self.orchestrator['requirements']['preserve_setup']:
 395             self.__logger.info('Remove the Abot_epc OS object ..')
 396             super(JujuEpc, self).clean()
 397
 398         return True
 399
 400
 401 # ----------------------------------------------------------
 402 #
 403 #               YAML UTILS
 404 #
 405 # -----------------------------------------------------------
 406 def get_config(parameter, file_path):
 407     """
 408     Returns the value of a given parameter in file.yaml
 409     parameter must be given in string format with dots
 410     Example: general.openstack.image_name
 411     """
 412     with open(file_path) as config_file:
 413         file_yaml = yaml.safe_load(config_file)
 414     config_file.close()
 415     value = file_yaml
 416     for element in parameter.split("."):
 417         value = value.get(element)
 418         if value is None:
 419             raise ValueError("The parameter %s is not defined in"
 420                              " reporting.yaml" % parameter)
 421     return value
 422
 423
 424 def sig_test_format(sig_test):
 425     """
 426     Process the signaling result to have a short result
 427     """
 428     nb_passed = 0
 429     nb_failures = 0
 430     nb_skipped = 0
 431     for data_test in sig_test:
 432         if data_test['result'] == "passed":
 433             nb_passed += 1
 434         elif data_test['result'] == "failed":
 435             nb_failures += 1
 436         elif data_test['result'] == "skipped":
 437             nb_skipped += 1
 438     total_sig_test_result = {}
 439     total_sig_test_result['passed'] = nb_passed
 440     total_sig_test_result['failures'] = nb_failures
 441     total_sig_test_result['skipped'] = nb_skipped
 442     return total_sig_test_result
 443
 444
 445 def process_abot_test_result(file_path):
 446     """ Process ABoT Result """
 447     with open(file_path) as test_result:
 448         data = json.load(test_result)
 449         res = []
 450         for tests in data:
 451             tests = update_data(tests)
 452             try:
 453                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 454                 for steps in flatten_steps:
 455                     steps['result'] = steps['step_status']
 456                     res.append(steps)
 457             except:
 458                 logging.error("Could not post data to ElasticSearch host")
 459                 raise
 460         return res
 461
 462
 463 def update_data(obj):
 464     """ Update Result data"""
 465     try:
 466         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 467
 468         for element in obj['elements']:
 469             element['final_result'] = "passed"
 470             element['flatten_steps'] = []
 471
 472             for step in element['steps']:
 473                 temp_dict = {}
 474                 step['result'][step['result']['status']] = 1
 475                 if step['result']['status'].lower() in ['fail', 'failed']:
 476                     element['final_result'] = "failed"
 477
 478                 temp_dict['feature_file'] = obj['feature_file']
 479                 temp_dict['step_name'] = step['name']
 480                 temp_dict['step_status'] = step['result']['status']
 481                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 482                 temp_dict['step_' + step['result']['status']] = 1
 483                 element['flatten_steps'].append(deepcopy(temp_dict))
 484
 485             # Need to put the tag in OBJ and not ELEMENT
 486             if 'tags' in obj:
 487                 element['tags'] = deepcopy(obj['tags'])
 488                 for tag in obj['tags']:
 489                     element[tag['name']] = 1
 490             else:
 491                 for tag in element['tags']:
 492                     element[tag['name']] = 1
 493
 494     except Exception:  # pylint: disable=broad-except
 495         logging.error("Error in updating data, %s", sys.exc_info()[0])
 496         raise
 497
 498     return obj
 499
 500
 501 def get_instances(nova_client):
 502     """ To get all vm info of a project """
 503     try:
 504         instances = nova_client.servers.list()
 505         return instances
 506     except Exception as exc:  # pylint: disable=broad-except
 507         logging.error("Error [get_instances(nova_client)]: %s", exc)
 508         return None
 509
 510
 511 def get_instance_metadata(nova_client, instance):
 512     """ Get instance Metadata - Instance ID """
 513     try:
 514         instance = nova_client.servers.get(instance.id)
 515         return instance.metadata
 516     except Exception as exc:  # pylint: disable=broad-except
 517         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 518         return None