functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20 import pkg_resources
  21 import yaml
  22
  23 from functest.core import vnf
  24 from functest.opnfv_tests.openstack.snaps import snaps_utils
  25 from functest.utils import config
  26
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.security_group import (
  32     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  33 from snaps.config.user import UserConfig
  34 from snaps.openstack.create_flavor import OpenStackFlavor
  35 from snaps.openstack.create_image import OpenStackImage
  36 from snaps.openstack.create_network import OpenStackNetwork
  37 from snaps.openstack.create_router import OpenStackRouter
  38 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  39 from snaps.openstack.create_user import OpenStackUser
  40 from snaps.openstack.utils import keystone_utils
  41 from snaps.openstack.utils import nova_utils
  42
  43 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  44 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  45
  46 CLOUD_TEMPLATE = """clouds:
  47   abot-epc:
  48     type: openstack
  49     auth-types: [userpass]
  50     endpoint: {url}
  51     regions:
  52       {region}:
  53         endpoint: {url}"""
  54
  55 CREDS_TEMPLATE2 = """credentials:
  56   abot-epc:
  57     default-credential: abot-epc
  58     abot-epc:
  59       auth-type: userpass
  60       password: {pass}
  61       project-domain-name: {project_domain_n}
  62       tenant-name: {tenant_n}"""
  63
  64 CREDS_TEMPLATE3 = """credentials:
  65   abot-epc:
  66     default-credential: abot-epc
  67     abot-epc:
  68       auth-type: userpass
  69       password: {pass}
  70       project-domain-name: {project_domain_n}
  71       tenant-name: {tenant_n}
  72       user-domain-name: {user_domain_n}
  73       username: {user_n}"""
  74
  75
  76 class JujuEpc(vnf.VnfOnBoarding):
  77     # pylint:disable=too-many-instance-attributes
  78     """Abot EPC deployed with JUJU Orchestrator Case"""
  79
  80     __logger = logging.getLogger(__name__)
  81
  82     def __init__(self, **kwargs):
  83         if "case_name" not in kwargs:
  84             kwargs["case_name"] = "juju_epc"
  85         super(JujuEpc, self).__init__(**kwargs)
  86
  87         # Retrieve the configuration
  88         self.case_dir = pkg_resources.resource_filename(
  89             'functest', 'opnfv_tests/vnf/epc')
  90         try:
  91             self.config = getattr(
  92                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  93         except Exception:
  94             raise Exception("VNF config file not found")
  95         self.config_file = os.path.join(self.case_dir, self.config)
  96         self.orchestrator = dict(requirements=get_config(
  97             "orchestrator.requirements", self.config_file))
  98
  99         self.created_object = []
 100         self.details['orchestrator'] = dict(
 101             name=get_config("orchestrator.name", self.config_file),
 102             version=get_config("orchestrator.version", self.config_file),
 103             status='ERROR',
 104             result=''
 105         )
 106
 107         self.vnf = dict(
 108             descriptor=get_config("vnf.descriptor", self.config_file),
 109             requirements=get_config("vnf.requirements", self.config_file)
 110         )
 111         self.details['vnf'] = dict(
 112             descriptor_version=self.vnf['descriptor']['version'],
 113             name=get_config("vnf.name", self.config_file),
 114             version=get_config("vnf.version", self.config_file),
 115         )
 116         self.__logger.debug("VNF configuration: %s", self.vnf)
 117
 118         self.details['test_vnf'] = dict(
 119             name=get_config("vnf_test_suite.name", self.config_file),
 120             version=get_config("vnf_test_suite.version", self.config_file),
 121             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 122         )
 123         self.public_auth_url = None
 124
 125         self.res_dir = os.path.join(
 126             getattr(config.CONF, 'dir_results'), self.case_name)
 127
 128     def _bypass_juju_network_discovery_bug(self, name):
 129         user_creator = OpenStackUser(
 130             self.snaps_creds,
 131             UserConfig(
 132                 name=name, password=str(uuid.uuid4()),
 133                 roles={'_member_': self.tenant_name}))
 134         user_creator.create()
 135         self.created_object.append(user_creator)
 136         return user_creator
 137
 138     def _register_cloud(self):
 139         self.__logger.info("Creating Cloud for Abot-epc .....")
 140         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 141         cloud_data = {
 142             'url': self.public_auth_url,
 143             'region': self.snaps_creds.region_name}
 144         with open(clouds_yaml, 'w') as yfile:
 145             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 146         if os.system(
 147                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 148             raise vnf.VnfPreparationException
 149
 150     def _register_credentials_v2(self):
 151         self.__logger.info("Creating Credentials for Abot-epc .....")
 152         user_creator = self._bypass_juju_network_discovery_bug(
 153             'juju_network_discovery_bug')
 154         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 155         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 156         creds_data = {
 157             'pass': snaps_creds.password,
 158             'tenant_n': snaps_creds.project_name,
 159             'user_n': snaps_creds.username}
 160         with open(credentials_yaml, 'w') as yfile:
 161             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 162         if os.system(
 163                 'juju add-credential abot-epc -f {} --replace'.format(
 164                     credentials_yaml)):
 165             raise vnf.VnfPreparationException
 166
 167     def _register_credentials_v3(self):
 168         self.__logger.info("Creating Credentials for Abot-epc .....")
 169         user_creator = self._bypass_juju_network_discovery_bug(
 170             'juju_network_discovery_bug')
 171         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 172         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 173         creds_data = {
 174             'pass': snaps_creds.password,
 175             'tenant_n': snaps_creds.project_name,
 176             'user_n': snaps_creds.username,
 177             'project_domain_n': snaps_creds.project_domain_name,
 178             'user_domain_n': snaps_creds.user_domain_name}
 179         with open(credentials_yaml, 'w') as yfile:
 180             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 181         if os.system(
 182                 'juju add-credential abot-epc -f {} --replace'.format(
 183                     credentials_yaml)):
 184             raise vnf.VnfPreparationException
 185
 186     def _add_custom_rule(self, sec_grp_name):
 187         """ To add custom rule for SCTP Traffic """
 188         sec_grp_rules = list()
 189         sec_grp_rules.append(
 190             SecurityGroupRuleConfig(
 191                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 192                 protocol=Protocol.sctp))
 193         security_group = OpenStackSecurityGroup(
 194             self.snaps_creds,
 195             SecurityGroupConfig(
 196                 name=sec_grp_name,
 197                 rule_settings=sec_grp_rules))
 198         security_group.create()
 199         self.created_object.append(security_group)
 200
 201     def prepare(self):
 202         """Prepare testcase (Additional pre-configuration steps)."""
 203         self.__logger.info("Additional pre-configuration steps")
 204         super(JujuEpc, self).prepare()
 205         try:
 206             os.makedirs(self.res_dir)
 207         except OSError as ex:
 208             if ex.errno != errno.EEXIST:
 209                 self.__logger.exception("Cannot create %s", self.res_dir)
 210                 raise vnf.VnfPreparationException
 211         self.public_auth_url = keystone_utils.get_endpoint(
 212             self.snaps_creds, 'identity')
 213         # it enforces a versioned public identity endpoint as juju simply
 214         # adds /auth/tokens wich fails vs an unversioned endpoint.
 215         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 216             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 217         self._register_cloud()
 218         if self.snaps_creds.identity_api_version == 3:
 219             self._register_credentials_v3()
 220         else:
 221             self._register_credentials_v2()
 222
 223     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 224         """
 225         Create network, subnet, router
 226
 227         Bootstrap juju
 228         """
 229         self.__logger.info("Deployed Orchestrator")
 230         private_net_name = getattr(
 231             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 232         private_subnet_name = '{}-{}'.format(
 233             getattr(config.CONF,
 234                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 235             self.uuid)
 236         private_subnet_cidr = getattr(
 237             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 238         abot_router = '{}-{}'.format(
 239             getattr(config.CONF,
 240                     'vnf_{}_external_router'.format(self.case_name)),
 241             self.uuid)
 242         self.__logger.info("Creating full network ...")
 243         subnet_settings = SubnetConfig(
 244             name=private_subnet_name, cidr=private_subnet_cidr)
 245         network_settings = NetworkConfig(
 246             name=private_net_name, subnet_settings=[subnet_settings])
 247         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 248         net_id = network_creator.create().id
 249         self.created_object.append(network_creator)
 250
 251         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 252         self.__logger.info("Creating network Router ....")
 253         router_creator = OpenStackRouter(
 254             self.snaps_creds, RouterConfig(
 255                 name=abot_router,
 256                 external_gateway=ext_net_name,
 257                 internal_subnets=[subnet_settings.name]))
 258         router_creator.create()
 259         self.created_object.append(router_creator)
 260         self.__logger.info("Creating Flavor ....")
 261         flavor_settings = FlavorConfig(
 262             name=self.orchestrator['requirements']['flavor']['name'],
 263             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 264             disk=10, vcpus=1)
 265         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 266         flavor_creator.create()
 267         self.created_object.append(flavor_creator)
 268         self.__logger.info("Upload some OS images if it doesn't exist")
 269         images = get_config("tenant_images", self.config_file)
 270         self.__logger.info("Images needed for vEPC: %s", images)
 271         for image_name, image_file in images.iteritems():
 272             self.__logger.info("image: %s, file: %s", image_name, image_file)
 273             if image_file and image_name:
 274                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 275                     name=image_name, image_user='cloud', img_format='qcow2',
 276                     image_file=image_file))
 277                 image_id = image_creator.create().id
 278                 os.system(
 279                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 280                     '{} -u {}'.format(
 281                         image_id, image_name, self.snaps_creds.region_name,
 282                         self.public_auth_url))
 283                 self.created_object.append(image_creator)
 284         self.__logger.info("Network ID  : %s", net_id)
 285         juju_bootstrap_command = (
 286             'juju bootstrap abot-epc abot-controller --config network={} '
 287             '--metadata-source ~  --config ssl-hostname-verification=false '
 288             '--constraints mem=2G --bootstrap-series xenial '
 289             '--config use-floating-ip=true --debug '
 290             '--config use-default-secgroup=true'.format(net_id))
 291         os.system(juju_bootstrap_command)
 292         return True
 293
 294     def deploy_vnf(self):
 295         """Deploy ABOT-OAI-EPC."""
 296         self.__logger.info("Upload VNFD")
 297         descriptor = self.vnf['descriptor']
 298         self.__logger.info("Get or create flavor for all Abot-EPC")
 299         flavor_settings = FlavorConfig(
 300             name=self.vnf['requirements']['flavor']['name'],
 301             ram=self.vnf['requirements']['flavor']['ram_min'],
 302             disk=10,
 303             vcpus=1)
 304         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 305         flavor_creator.create()
 306         self.created_object.append(flavor_creator)
 307         self.__logger.info("Deploying Abot-epc bundle file ...")
 308         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 309         self.__logger.info("Waiting for instances .....")
 310         status = os.system('juju-wait')
 311         self.__logger.info("juju wait completed: %s", status)
 312         self.__logger.info("Deployed Abot-epc on Openstack")
 313         nova_client = nova_utils.nova_client(self.snaps_creds)
 314         if status == 0:
 315             instances = get_instances(nova_client)
 316             self.__logger.info("List of Instance: %s", instances)
 317             for items in instances:
 318                 metadata = get_instance_metadata(nova_client, items)
 319                 if 'juju-units-deployed' in metadata:
 320                     sec_group = ('juju-' +
 321                                  metadata['juju-controller-uuid'] +
 322                                  '-' + metadata['juju-model-uuid'])
 323                     self.__logger.info("Instance: %s", sec_group)
 324                     break
 325             self.__logger.info("Adding Security group rule....")
 326             # This will add sctp rule to a common Security Group Created
 327             # by juju and shared to all deployed units.
 328             self._add_custom_rule(sec_group)
 329             self.__logger.info("Copying the feature files to Abot_node ")
 330             os.system('juju scp -- -r {}/featureFiles abot-'
 331                       'epc-basic/0:~/'.format(self.case_dir))
 332             self.__logger.info("Copying the feature files in Abot_node ")
 333             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 334                       "~/featureFiles /etc/rebaca-test-suite"
 335                       "/featureFiles'")
 336             count = 0
 337             while count < 10:
 338                 epcstatus = os.system('juju status oai-epc | '
 339                                       'grep {} | grep {} | grep {}'
 340                                       .format('EPC', 'is', 'running'))
 341                 if epcstatus == 0:
 342                     break
 343                 else:
 344                     time.sleep(60)
 345                     count = count + 1
 346             os.system('juju-wait')
 347             return True
 348         return False
 349
 350     def test_vnf(self):
 351         """Run test on ABoT."""
 352         start_time = time.time()
 353         self.__logger.info("Running VNF Test cases....")
 354         os.system('juju run-action abot-epc-basic/0 run '
 355                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 356         os.system('juju-wait')
 357         duration = time.time() - start_time
 358         self.__logger.info("Getting results from Abot node....")
 359         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 360                   'epc-basic/artifacts/TestResults.json {}/.'
 361                   .format(self.case_dir))
 362         self.__logger.info("Parsing the Test results...")
 363         res = (process_abot_test_result('{}/TestResults.'
 364                                         'json'.format(self.case_dir)))
 365         short_result = sig_test_format(res)
 366         self.__logger.info(short_result)
 367         self.details['test_vnf'].update(status='PASS',
 368                                         result=short_result,
 369                                         full_result=res,
 370                                         duration=duration)
 371
 372         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 373                            "%d, Skipped: %d", short_result['passed'],
 374                            short_result['failures'], short_result['skipped'])
 375         return True
 376
 377     def clean(self):
 378         """Clean created objects/functions."""
 379         try:
 380             if not self.orchestrator['requirements']['preserve_setup']:
 381                 self.__logger.info("Removing deployment files...")
 382                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 383                 if os.path.exists(testresult):
 384                     os.remove(testresult)
 385                 self.__logger.info("Destroying Orchestrator...")
 386                 os.system('juju destroy-controller -y abot-controller '
 387                           '--destroy-all-models')
 388         except Exception:  # pylint: disable=broad-except
 389             self.__logger.warn("Some issue during the undeployment ..")
 390             self.__logger.warn("Tenant clean continue ..")
 391
 392         if not self.orchestrator['requirements']['preserve_setup']:
 393             self.__logger.info('Remove the Abot_epc OS object ..')
 394             super(JujuEpc, self).clean()
 395
 396         return True
 397
 398
 399 # ----------------------------------------------------------
 400 #
 401 #               YAML UTILS
 402 #
 403 # -----------------------------------------------------------
 404 def get_config(parameter, file_path):
 405     """
 406     Returns the value of a given parameter in file.yaml
 407     parameter must be given in string format with dots
 408     Example: general.openstack.image_name
 409     """
 410     with open(file_path) as config_file:
 411         file_yaml = yaml.safe_load(config_file)
 412     config_file.close()
 413     value = file_yaml
 414     for element in parameter.split("."):
 415         value = value.get(element)
 416         if value is None:
 417             raise ValueError("The parameter %s is not defined in"
 418                              " reporting.yaml" % parameter)
 419     return value
 420
 421
 422 def sig_test_format(sig_test):
 423     """
 424     Process the signaling result to have a short result
 425     """
 426     nb_passed = 0
 427     nb_failures = 0
 428     nb_skipped = 0
 429     for data_test in sig_test:
 430         if data_test['result'] == "passed":
 431             nb_passed += 1
 432         elif data_test['result'] == "failed":
 433             nb_failures += 1
 434         elif data_test['result'] == "skipped":
 435             nb_skipped += 1
 436     total_sig_test_result = {}
 437     total_sig_test_result['passed'] = nb_passed
 438     total_sig_test_result['failures'] = nb_failures
 439     total_sig_test_result['skipped'] = nb_skipped
 440     return total_sig_test_result
 441
 442
 443 def process_abot_test_result(file_path):
 444     """ Process ABoT Result """
 445     with open(file_path) as test_result:
 446         data = json.load(test_result)
 447         res = []
 448         for tests in data:
 449             tests = update_data(tests)
 450             try:
 451                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 452                 for steps in flatten_steps:
 453                     steps['result'] = steps['step_status']
 454                     res.append(steps)
 455             except:
 456                 logging.error("Could not post data to ElasticSearch host")
 457                 raise
 458         return res
 459
 460
 461 def update_data(obj):
 462     """ Update Result data"""
 463     try:
 464         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 465
 466         for element in obj['elements']:
 467             element['final_result'] = "passed"
 468             element['flatten_steps'] = []
 469
 470             for step in element['steps']:
 471                 temp_dict = {}
 472                 step['result'][step['result']['status']] = 1
 473                 if step['result']['status'].lower() in ['fail', 'failed']:
 474                     element['final_result'] = "failed"
 475
 476                 temp_dict['feature_file'] = obj['feature_file']
 477                 temp_dict['step_name'] = step['name']
 478                 temp_dict['step_status'] = step['result']['status']
 479                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 480                 temp_dict['step_' + step['result']['status']] = 1
 481                 element['flatten_steps'].append(deepcopy(temp_dict))
 482
 483             # Need to put the tag in OBJ and not ELEMENT
 484             if 'tags' in obj:
 485                 element['tags'] = deepcopy(obj['tags'])
 486                 for tag in obj['tags']:
 487                     element[tag['name']] = 1
 488             else:
 489                 for tag in element['tags']:
 490                     element[tag['name']] = 1
 491
 492     except Exception:  # pylint: disable=broad-except
 493         logging.error("Error in updating data, %s", sys.exc_info()[0])
 494         raise
 495
 496     return obj
 497
 498
 499 def get_instances(nova_client):
 500     """ To get all vm info of a project """
 501     try:
 502         instances = nova_client.servers.list()
 503         return instances
 504     except Exception as exc:  # pylint: disable=broad-except
 505         logging.error("Error [get_instances(nova_client)]: %s", exc)
 506         return None
 507
 508
 509 def get_instance_metadata(nova_client, instance):
 510     """ Get instance Metadata - Instance ID """
 511     try:
 512         instance = nova_client.servers.get(instance.id)
 513         return instance.metadata
 514     except Exception as exc:  # pylint: disable=broad-except
 515         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 516         return None