functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20 import pkg_resources
  21 import yaml
  22
  23 from functest.core import vnf
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.utils.constants import CONST
  26
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.security_group import (
  32     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  33 from snaps.config.user import UserConfig
  34 from snaps.openstack.create_flavor import OpenStackFlavor
  35 from snaps.openstack.create_image import OpenStackImage
  36 from snaps.openstack.create_network import OpenStackNetwork
  37 from snaps.openstack.create_router import OpenStackRouter
  38 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  39 from snaps.openstack.create_user import OpenStackUser
  40 from snaps.openstack.utils import keystone_utils
  41 from snaps.openstack.utils import nova_utils
  42
  43 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  44 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  45
  46 CLOUD_TEMPLATE = """clouds:
  47   abot-epc:
  48     type: openstack
  49     auth-types: [userpass]
  50     endpoint: {url}
  51     regions:
  52       {region}:
  53         endpoint: {url}"""
  54
  55 CREDS_TEMPLATE2 = """credentials:
  56   abot-epc:
  57     default-credential: abot-epc
  58     abot-epc:
  59       auth-type: userpass
  60       password: {pass}
  61       project-domain-name: {project_domain_n}
  62       tenant-name: {tenant_n}"""
  63
  64 CREDS_TEMPLATE3 = """credentials:
  65   abot-epc:
  66     default-credential: abot-epc
  67     abot-epc:
  68       auth-type: userpass
  69       password: {pass}
  70       project-domain-name: {project_domain_n}
  71       tenant-name: {tenant_n}
  72       user-domain-name: {user_domain_n}
  73       username: {user_n}"""
  74
  75
  76 class JujuEpc(vnf.VnfOnBoarding):
  77     # pylint:disable=too-many-instance-attributes
  78     """Abot EPC deployed with JUJU Orchestrator Case"""
  79
  80     __logger = logging.getLogger(__name__)
  81
  82     def __init__(self, **kwargs):
  83         if "case_name" not in kwargs:
  84             kwargs["case_name"] = "juju_epc"
  85         super(JujuEpc, self).__init__(**kwargs)
  86
  87         # Retrieve the configuration
  88         self.case_dir = pkg_resources.resource_filename(
  89             'functest', 'opnfv_tests/vnf/epc')
  90         try:
  91             self.config = getattr(
  92                 CONST, 'vnf_{}_config'.format(self.case_name))
  93         except Exception:
  94             raise Exception("VNF config file not found")
  95         self.config_file = os.path.join(self.case_dir, self.config)
  96         self.orchestrator = dict(requirements=get_config(
  97             "orchestrator.requirements", self.config_file))
  98
  99         self.created_object = []
 100         self.details['orchestrator'] = dict(
 101             name=get_config("orchestrator.name", self.config_file),
 102             version=get_config("orchestrator.version", self.config_file),
 103             status='ERROR',
 104             result=''
 105         )
 106
 107         self.vnf = dict(
 108             descriptor=get_config("vnf.descriptor", self.config_file),
 109             requirements=get_config("vnf.requirements", self.config_file)
 110         )
 111         self.details['vnf'] = dict(
 112             descriptor_version=self.vnf['descriptor']['version'],
 113             name=get_config("vnf.name", self.config_file),
 114             version=get_config("vnf.version", self.config_file),
 115         )
 116         self.__logger.debug("VNF configuration: %s", self.vnf)
 117
 118         self.details['test_vnf'] = dict(
 119             name=get_config("vnf_test_suite.name", self.config_file),
 120             version=get_config("vnf_test_suite.version", self.config_file),
 121             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 122         )
 123         self.public_auth_url = None
 124
 125         self.res_dir = os.path.join(
 126             getattr(CONST, 'dir_results'), self.case_name)
 127
 128     def _bypass_juju_network_discovery_bug(self, name):
 129         user_creator = OpenStackUser(
 130             self.snaps_creds,
 131             UserConfig(
 132                 name=name, password=str(uuid.uuid4()),
 133                 roles={'_member_': self.tenant_name}))
 134         user_creator.create()
 135         self.created_object.append(user_creator)
 136         return user_creator
 137
 138     def _register_cloud(self):
 139         self.__logger.info("Creating Cloud for Abot-epc .....")
 140         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 141         cloud_data = {
 142             'url': self.public_auth_url,
 143             'region': self.snaps_creds.region_name}
 144         with open(clouds_yaml, 'w') as yfile:
 145             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 146         if os.system(
 147                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 148             raise vnf.VnfPreparationException
 149
 150     def _register_credentials_v2(self):
 151         self.__logger.info("Creating Credentials for Abot-epc .....")
 152         user_creator = self._bypass_juju_network_discovery_bug(
 153             'juju_network_discovery_bug')
 154         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 155         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 156         creds_data = {
 157             'pass': snaps_creds.password,
 158             'tenant_n': snaps_creds.project_name,
 159             'user_n': snaps_creds.username}
 160         with open(credentials_yaml, 'w') as yfile:
 161             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 162         if os.system(
 163                 'juju add-credential abot-epc -f {} --replace'.format(
 164                     credentials_yaml)):
 165             raise vnf.VnfPreparationException
 166
 167     def _register_credentials_v3(self):
 168         self.__logger.info("Creating Credentials for Abot-epc .....")
 169         user_creator = self._bypass_juju_network_discovery_bug(
 170             'juju_network_discovery_bug')
 171         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 172         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 173         creds_data = {
 174             'pass': snaps_creds.password,
 175             'tenant_n': snaps_creds.project_name,
 176             'user_n': snaps_creds.username,
 177             'project_domain_n': snaps_creds.project_domain_name,
 178             'user_domain_n': snaps_creds.user_domain_name}
 179         with open(credentials_yaml, 'w') as yfile:
 180             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 181         if os.system(
 182                 'juju add-credential abot-epc -f {} --replace'.format(
 183                     credentials_yaml)):
 184             raise vnf.VnfPreparationException
 185
 186     def _add_custom_rule(self, sec_grp_name):
 187         """ To add custom rule for SCTP Traffic """
 188         sec_grp_rules = list()
 189         security_group_init = OpenStackSecurityGroup(
 190             self.snaps_creds,
 191             SecurityGroupConfig(
 192                 name=sec_grp_name,
 193                 rule_settings=sec_grp_rules))
 194         security_group_init.initialize()
 195         sctp_rule = SecurityGroupRuleConfig(
 196             sec_grp_name=sec_grp_name, direction=Direction.ingress,
 197             protocol=Protocol.sctp)
 198         security_group_init.add_rule(sctp_rule)
 199         self.created_object.append(security_group_init)
 200
 201     def prepare(self):
 202         """Prepare testcase (Additional pre-configuration steps)."""
 203         self.__logger.info("Additional pre-configuration steps")
 204         super(JujuEpc, self).prepare()
 205         try:
 206             os.makedirs(self.res_dir)
 207         except OSError as ex:
 208             if ex.errno != errno.EEXIST:
 209                 self.__logger.exception("Cannot create %s", self.res_dir)
 210                 raise vnf.VnfPreparationException
 211         self.public_auth_url = keystone_utils.get_endpoint(
 212             self.snaps_creds, 'identity')
 213         # it enforces a versioned public identity endpoint as juju simply
 214         # adds /auth/tokens wich fails vs an unversioned endpoint.
 215         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 216             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 217         self._register_cloud()
 218         if self.snaps_creds.identity_api_version == 3:
 219             self._register_credentials_v3()
 220         else:
 221             self._register_credentials_v2()
 222
 223     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 224         """
 225         Create network, subnet, router
 226
 227         Bootstrap juju
 228         """
 229         self.__logger.info("Deployed Orchestrator")
 230         private_net_name = getattr(
 231             CONST, 'vnf_{}_private_net_name'.format(self.case_name))
 232         private_subnet_name = getattr(
 233             CONST, 'vnf_{}_private_subnet_name'.format(self.case_name))
 234         private_subnet_cidr = getattr(
 235             CONST, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 236         abot_router = getattr(
 237             CONST, 'vnf_{}_external_router'.format(self.case_name))
 238
 239         self.__logger.info("Creating full network ...")
 240         subnet_settings = SubnetConfig(
 241             name=private_subnet_name, cidr=private_subnet_cidr)
 242         network_settings = NetworkConfig(
 243             name=private_net_name, subnet_settings=[subnet_settings])
 244         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 245         net_id = network_creator.create().id
 246         self.created_object.append(network_creator)
 247
 248         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 249         self.__logger.info("Creating network Router ....")
 250         router_creator = OpenStackRouter(
 251             self.snaps_creds, RouterConfig(
 252                 name=abot_router,
 253                 external_gateway=ext_net_name,
 254                 internal_subnets=[subnet_settings.name]))
 255         router_creator.create()
 256         self.created_object.append(router_creator)
 257         self.__logger.info("Creating Flavor ....")
 258         flavor_settings = FlavorConfig(
 259             name=self.orchestrator['requirements']['flavor']['name'],
 260             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 261             disk=10, vcpus=1)
 262         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 263         flavor_creator.create()
 264         self.created_object.append(flavor_creator)
 265         self.__logger.info("Upload some OS images if it doesn't exist")
 266         images = get_config("tenant_images", self.config_file)
 267         self.__logger.info("Images needed for vEPC: %s", images)
 268         for image_name, image_file in images.iteritems():
 269             self.__logger.info("image: %s, file: %s", image_name, image_file)
 270             if image_file and image_name:
 271                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 272                     name=image_name, image_user='cloud', img_format='qcow2',
 273                     image_file=image_file))
 274                 image_id = image_creator.create().id
 275                 os.system(
 276                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 277                     '{} -u {}'.format(
 278                         image_id, image_name, self.snaps_creds.region_name,
 279                         self.public_auth_url))
 280                 self.created_object.append(image_creator)
 281         self.__logger.info("Network ID  : %s", net_id)
 282         juju_bootstrap_command = (
 283             'juju bootstrap abot-epc abot-controller --config network={} '
 284             '--metadata-source ~  --config ssl-hostname-verification=false '
 285             '--constraints mem=2G --bootstrap-series xenial '
 286             '--config use-floating-ip=true --debug '
 287             '--config use-default-secgroup=true'.format(net_id))
 288         os.system(juju_bootstrap_command)
 289         return True
 290
 291     def deploy_vnf(self):
 292         """Deploy ABOT-OAI-EPC."""
 293         self.__logger.info("Upload VNFD")
 294         descriptor = self.vnf['descriptor']
 295         self.__logger.info("Get or create flavor for all Abot-EPC")
 296         flavor_settings = FlavorConfig(
 297             name=self.vnf['requirements']['flavor']['name'],
 298             ram=self.vnf['requirements']['flavor']['ram_min'],
 299             disk=10,
 300             vcpus=1)
 301         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 302         flavor_creator.create()
 303         self.created_object.append(flavor_creator)
 304         self.__logger.info("Deploying Abot-epc bundle file ...")
 305         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 306         self.__logger.info("Waiting for instances .....")
 307         status = os.system('juju-wait')
 308         self.__logger.info("juju wait completed: %s", status)
 309         self.__logger.info("Deployed Abot-epc on Openstack")
 310         nova_client = nova_utils.nova_client(self.snaps_creds)
 311         if status == 0:
 312             instances = get_instances(nova_client)
 313             self.__logger.info("List of Instance: %s", instances)
 314             for items in instances:
 315                 metadata = get_instance_metadata(nova_client, items)
 316                 if 'juju-units-deployed' in metadata:
 317                     sec_group = ('juju-' +
 318                                  metadata['juju-controller-uuid'] +
 319                                  '-' + metadata['juju-model-uuid'])
 320                     self.__logger.info("Instance: %s", sec_group)
 321                     break
 322             self.__logger.info("Adding Security group rule....")
 323             # This will add sctp rule to a common Security Group Created
 324             # by juju and shared to all deployed units.
 325             self._add_custom_rule(sec_group)
 326             self.__logger.info("Copying the feature files to Abot_node ")
 327             os.system('juju scp -- -r {}/featureFiles abot-'
 328                       'epc-basic/0:~/'.format(self.case_dir))
 329             self.__logger.info("Copying the feature files in Abot_node ")
 330             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 331                       "~/featureFiles /etc/rebaca-test-suite"
 332                       "/featureFiles'")
 333             count = 0
 334             while count < 10:
 335                 epcstatus = os.system('juju status oai-epc | '
 336                                       'grep {} | grep {} | grep {}'
 337                                       .format('EPC', 'is', 'running'))
 338                 if epcstatus == 0:
 339                     break
 340                 else:
 341                     time.sleep(60)
 342                     count = count + 1
 343             os.system('juju-wait')
 344             return True
 345         return False
 346
 347     def test_vnf(self):
 348         """Run test on ABoT."""
 349         start_time = time.time()
 350         self.__logger.info("Running VNF Test cases....")
 351         os.system('juju run-action abot-epc-basic/0 run '
 352                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 353         os.system('juju-wait')
 354         duration = time.time() - start_time
 355         self.__logger.info("Getting results from Abot node....")
 356         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 357                   'epc-basic/artifacts/TestResults.json {}/.'
 358                   .format(self.case_dir))
 359         self.__logger.info("Parsing the Test results...")
 360         res = (process_abot_test_result('{}/TestResults.'
 361                                         'json'.format(self.case_dir)))
 362         short_result = sig_test_format(res)
 363         self.__logger.info(short_result)
 364         self.details['test_vnf'].update(status='PASS',
 365                                         result=short_result,
 366                                         full_result=res,
 367                                         duration=duration)
 368
 369         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 370                            "%d, Skipped: %d", short_result['passed'],
 371                            short_result['failures'], short_result['skipped'])
 372         return True
 373
 374     def clean(self):
 375         """Clean created objects/functions."""
 376         try:
 377             if not self.orchestrator['requirements']['preserve_setup']:
 378                 self.__logger.info("Removing deployment files...")
 379                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 380                 if os.path.exists(testresult):
 381                     os.remove(testresult)
 382                 self.__logger.info("Destroying Orchestrator...")
 383                 os.system('juju destroy-controller -y abot-controller '
 384                           '--destroy-all-models')
 385         except Exception:  # pylint: disable=broad-except
 386             self.__logger.warn("Some issue during the undeployment ..")
 387             self.__logger.warn("Tenant clean continue ..")
 388
 389         if not self.orchestrator['requirements']['preserve_setup']:
 390             self.__logger.info('Remove the Abot_epc OS object ..')
 391             super(JujuEpc, self).clean()
 392
 393         return True
 394
 395
 396 # ----------------------------------------------------------
 397 #
 398 #               YAML UTILS
 399 #
 400 # -----------------------------------------------------------
 401 def get_config(parameter, file_path):
 402     """
 403     Returns the value of a given parameter in file.yaml
 404     parameter must be given in string format with dots
 405     Example: general.openstack.image_name
 406     """
 407     with open(file_path) as config_file:
 408         file_yaml = yaml.safe_load(config_file)
 409     config_file.close()
 410     value = file_yaml
 411     for element in parameter.split("."):
 412         value = value.get(element)
 413         if value is None:
 414             raise ValueError("The parameter %s is not defined in"
 415                              " reporting.yaml" % parameter)
 416     return value
 417
 418
 419 def sig_test_format(sig_test):
 420     """
 421     Process the signaling result to have a short result
 422     """
 423     nb_passed = 0
 424     nb_failures = 0
 425     nb_skipped = 0
 426     for data_test in sig_test:
 427         if data_test['result'] == "passed":
 428             nb_passed += 1
 429         elif data_test['result'] == "failed":
 430             nb_failures += 1
 431         elif data_test['result'] == "skipped":
 432             nb_skipped += 1
 433     total_sig_test_result = {}
 434     total_sig_test_result['passed'] = nb_passed
 435     total_sig_test_result['failures'] = nb_failures
 436     total_sig_test_result['skipped'] = nb_skipped
 437     return total_sig_test_result
 438
 439
 440 def process_abot_test_result(file_path):
 441     """ Process ABoT Result """
 442     with open(file_path) as test_result:
 443         data = json.load(test_result)
 444         res = []
 445         for tests in data:
 446             tests = update_data(tests)
 447             try:
 448                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 449                 for steps in flatten_steps:
 450                     steps['result'] = steps['step_status']
 451                     res.append(steps)
 452             except:
 453                 logging.error("Could not post data to ElasticSearch host")
 454                 raise
 455         return res
 456
 457
 458 def update_data(obj):
 459     """ Update Result data"""
 460     try:
 461         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 462
 463         for element in obj['elements']:
 464             element['final_result'] = "passed"
 465             element['flatten_steps'] = []
 466
 467             for step in element['steps']:
 468                 temp_dict = {}
 469                 step['result'][step['result']['status']] = 1
 470                 if step['result']['status'].lower() in ['fail', 'failed']:
 471                     element['final_result'] = "failed"
 472
 473                 temp_dict['feature_file'] = obj['feature_file']
 474                 temp_dict['step_name'] = step['name']
 475                 temp_dict['step_status'] = step['result']['status']
 476                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 477                 temp_dict['step_' + step['result']['status']] = 1
 478                 element['flatten_steps'].append(deepcopy(temp_dict))
 479
 480             # Need to put the tag in OBJ and not ELEMENT
 481             if 'tags' in obj:
 482                 element['tags'] = deepcopy(obj['tags'])
 483                 for tag in obj['tags']:
 484                     element[tag['name']] = 1
 485             else:
 486                 for tag in element['tags']:
 487                     element[tag['name']] = 1
 488
 489     except Exception:  # pylint: disable=broad-except
 490         logging.error("Error in updating data, %s", sys.exc_info()[0])
 491         raise
 492
 493     return obj
 494
 495
 496 def get_instances(nova_client):
 497     """ To get all vm info of a project """
 498     try:
 499         instances = nova_client.servers.list()
 500         return instances
 501     except Exception as exc:  # pylint: disable=broad-except
 502         logging.error("Error [get_instances(nova_client)]: %s", exc)
 503         return None
 504
 505
 506 def get_instance_metadata(nova_client, instance):
 507     """ Get instance Metadata - Instance ID """
 508     try:
 509         instance = nova_client.servers.get(instance.id)
 510         return instance.metadata
 511     except Exception as exc:  # pylint: disable=broad-except
 512         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 513         return None