functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import re
  17 import subprocess
  18 import sys
  19 import uuid
  20 from copy import deepcopy
  21 from urlparse import urljoin
  22 import pkg_resources
  23 import yaml
  24
  25 from functest.core import vnf
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.utils import config
  28 from functest.utils import env
  29
  30 from snaps.config.flavor import FlavorConfig
  31 from snaps.config.image import ImageConfig
  32 from snaps.config.network import NetworkConfig, SubnetConfig
  33 from snaps.config.router import RouterConfig
  34 from snaps.config.security_group import (
  35     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  36 from snaps.config.user import UserConfig
  37 from snaps.openstack.create_flavor import OpenStackFlavor
  38 from snaps.openstack.create_image import OpenStackImage
  39 from snaps.openstack.create_network import OpenStackNetwork
  40 from snaps.openstack.create_router import OpenStackRouter
  41 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  42 from snaps.openstack.create_user import OpenStackUser
  43 from snaps.openstack.utils import keystone_utils
  44 from snaps.openstack.utils import nova_utils
  45
  46 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  47 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  48
  49 CLOUD_TEMPLATE = """clouds:
  50   abot-epc:
  51     type: openstack
  52     auth-types: [userpass]
  53     endpoint: {url}
  54     regions:
  55       {region}:
  56         endpoint: {url}"""
  57
  58 CREDS_TEMPLATE2 = """credentials:
  59   abot-epc:
  60     default-credential: abot-epc
  61     abot-epc:
  62       auth-type: userpass
  63       password: {pass}
  64       project-domain-name: {project_domain_n}
  65       tenant-name: {tenant_n}"""
  66
  67 CREDS_TEMPLATE3 = """credentials:
  68   abot-epc:
  69     default-credential: abot-epc
  70     abot-epc:
  71       auth-type: userpass
  72       password: {pass}
  73       project-domain-name: {project_domain_n}
  74       tenant-name: {tenant_n}
  75       user-domain-name: {user_domain_n}
  76       username: {user_n}"""
  77
  78
  79 class JujuEpc(vnf.VnfOnBoarding):
  80     # pylint:disable=too-many-instance-attributes
  81     """Abot EPC deployed with JUJU Orchestrator Case"""
  82
  83     __logger = logging.getLogger(__name__)
  84
  85     juju_wait_timeout = '3600'
  86
  87     def __init__(self, **kwargs):
  88         if "case_name" not in kwargs:
  89             kwargs["case_name"] = "juju_epc"
  90         super(JujuEpc, self).__init__(**kwargs)
  91
  92         # Retrieve the configuration
  93         self.case_dir = pkg_resources.resource_filename(
  94             'functest', 'opnfv_tests/vnf/epc')
  95         try:
  96             self.config = getattr(
  97                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  98         except Exception:
  99             raise Exception("VNF config file not found")
 100         self.config_file = os.path.join(self.case_dir, self.config)
 101         self.orchestrator = dict(requirements=get_config(
 102             "orchestrator.requirements", self.config_file))
 103
 104         self.created_object = []
 105         self.details['orchestrator'] = dict(
 106             name=get_config("orchestrator.name", self.config_file),
 107             version=get_config("orchestrator.version", self.config_file),
 108             status='ERROR',
 109             result=''
 110         )
 111
 112         self.vnf = dict(
 113             descriptor=get_config("vnf.descriptor", self.config_file),
 114             requirements=get_config("vnf.requirements", self.config_file)
 115         )
 116         self.details['vnf'] = dict(
 117             descriptor_version=self.vnf['descriptor']['version'],
 118             name=get_config("vnf.name", self.config_file),
 119             version=get_config("vnf.version", self.config_file),
 120         )
 121         self.__logger.debug("VNF configuration: %s", self.vnf)
 122
 123         self.details['test_vnf'] = dict(
 124             name=get_config("vnf_test_suite.name", self.config_file),
 125             version=get_config("vnf_test_suite.version", self.config_file),
 126             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 127         )
 128         self.public_auth_url = None
 129
 130         self.res_dir = os.path.join(
 131             getattr(config.CONF, 'dir_results'), self.case_name)
 132
 133     def _bypass_juju_network_discovery_bug(self, name):
 134         user_creator = OpenStackUser(
 135             self.snaps_creds,
 136             UserConfig(
 137                 name=name,
 138                 password=str(uuid.uuid4()),
 139                 project_name=self.tenant_name,
 140                 domain=self.snaps_creds.user_domain_name,
 141                 roles={'_member_': self.tenant_name}))
 142         user_creator.create()
 143         self.created_object.append(user_creator)
 144         return user_creator
 145
 146     def _register_cloud(self):
 147         self.__logger.info("Creating Cloud for Abot-epc .....")
 148         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 149         cloud_data = {
 150             'url': self.public_auth_url,
 151             'region': self.snaps_creds.region_name}
 152         with open(clouds_yaml, 'w') as yfile:
 153             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 154         cmd = ['juju', 'add-cloud', 'abot-epc', '-f', clouds_yaml, '--replace']
 155         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 156         self.__logger.info("%s\n%s", " ".join(cmd), output)
 157
 158     def _register_credentials_v2(self):
 159         self.__logger.info("Creating Credentials for Abot-epc .....")
 160         user_creator = self._bypass_juju_network_discovery_bug(
 161             'juju_network_discovery_bug')
 162         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 163         self.__logger.debug("snaps creds: %s", snaps_creds)
 164         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 165         creds_data = {
 166             'pass': snaps_creds.password,
 167             'tenant_n': snaps_creds.project_name,
 168             'user_n': snaps_creds.username}
 169         with open(credentials_yaml, 'w') as yfile:
 170             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 171         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 172                '--replace']
 173         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 174         self.__logger.info("%s\n%s", " ".join(cmd), output)
 175
 176     def _register_credentials_v3(self):
 177         self.__logger.info("Creating Credentials for Abot-epc .....")
 178         user_creator = self._bypass_juju_network_discovery_bug(
 179             'juju_network_discovery_bug')
 180         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 181         self.__logger.debug("snaps creds: %s", snaps_creds)
 182         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 183         creds_data = {
 184             'pass': snaps_creds.password,
 185             'tenant_n': snaps_creds.project_name,
 186             'user_n': snaps_creds.username,
 187             'project_domain_n': snaps_creds.project_domain_name,
 188             'user_domain_n': snaps_creds.user_domain_name}
 189         with open(credentials_yaml, 'w') as yfile:
 190             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 191         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 192                '--replace']
 193         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 194         self.__logger.info("%s\n%s", " ".join(cmd), output)
 195
 196     def _add_custom_rule(self, sec_grp_name):
 197         """ To add custom rule for SCTP Traffic """
 198         sec_grp_rules = list()
 199         sec_grp_rules.append(
 200             SecurityGroupRuleConfig(
 201                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 202                 protocol=Protocol.sctp))
 203         security_group = OpenStackSecurityGroup(
 204             self.snaps_creds,
 205             SecurityGroupConfig(
 206                 name=sec_grp_name,
 207                 rule_settings=sec_grp_rules))
 208         security_group.create()
 209         self.created_object.append(security_group)
 210
 211     def prepare(self):
 212         """Prepare testcase (Additional pre-configuration steps)."""
 213         self.__logger.info("Additional pre-configuration steps")
 214         super(JujuEpc, self).prepare()
 215         try:
 216             os.makedirs(self.res_dir)
 217         except OSError as ex:
 218             if ex.errno != errno.EEXIST:
 219                 self.__logger.exception("Cannot create %s", self.res_dir)
 220                 raise vnf.VnfPreparationException
 221         self.public_auth_url = keystone_utils.get_endpoint(
 222             self.snaps_creds, 'identity')
 223         # it enforces a versioned public identity endpoint as juju simply
 224         # adds /auth/tokens wich fails vs an unversioned endpoint.
 225         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 226             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 227         self._register_cloud()
 228         if self.snaps_creds.identity_api_version == 3:
 229             self._register_credentials_v3()
 230         else:
 231             self._register_credentials_v2()
 232
 233     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 234         """
 235         Create network, subnet, router
 236
 237         Bootstrap juju
 238         """
 239         self.__logger.info("Deployed Orchestrator")
 240         private_net_name = getattr(
 241             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 242         private_subnet_name = '{}-{}'.format(
 243             getattr(config.CONF,
 244                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 245             self.uuid)
 246         private_subnet_cidr = getattr(
 247             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 248         abot_router = '{}-{}'.format(
 249             getattr(config.CONF,
 250                     'vnf_{}_external_router'.format(self.case_name)),
 251             self.uuid)
 252         self.__logger.info("Creating full network ...")
 253         subnet_settings = SubnetConfig(
 254             name=private_subnet_name,
 255             cidr=private_subnet_cidr,
 256             dns_nameservers=[env.get('NAMESERVER')])
 257         network_settings = NetworkConfig(
 258             name=private_net_name, subnet_settings=[subnet_settings])
 259         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 260         net_id = network_creator.create().id
 261         self.created_object.append(network_creator)
 262
 263         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 264         self.__logger.info("Creating network Router ....")
 265         router_creator = OpenStackRouter(
 266             self.snaps_creds, RouterConfig(
 267                 name=abot_router,
 268                 external_gateway=ext_net_name,
 269                 internal_subnets=[subnet_settings.name]))
 270         router_creator.create()
 271         self.created_object.append(router_creator)
 272         self.__logger.info("Creating Flavor ....")
 273         flavor_settings = FlavorConfig(
 274             name=self.orchestrator['requirements']['flavor']['name'],
 275             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 276             disk=10, vcpus=1)
 277         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 278         flavor_creator.create()
 279         self.created_object.append(flavor_creator)
 280         self.__logger.info("Upload some OS images if it doesn't exist")
 281         images = get_config("tenant_images", self.config_file)
 282         self.__logger.info("Images needed for vEPC: %s", images)
 283         for image_name, image_file in images.iteritems():
 284             self.__logger.info("image: %s, file: %s", image_name, image_file)
 285             if image_file and image_name:
 286                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 287                     name=image_name, image_user='cloud', img_format='qcow2',
 288                     image_file=image_file))
 289                 image_id = image_creator.create().id
 290                 cmd = ['juju', 'metadata', 'generate-image', '-d', '/root',
 291                        '-i', image_id, '-s', image_name,
 292                        '-r', self.snaps_creds.region_name,
 293                        '-u', self.public_auth_url]
 294                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 295                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 296                 self.created_object.append(image_creator)
 297         self.__logger.info("Network ID  : %s", net_id)
 298         cmd = ['juju', 'bootstrap', 'abot-epc', 'abot-controller',
 299                '--metadata-source', '/root',
 300                '--constraints', 'mem=2G',
 301                '--bootstrap-series', 'xenial',
 302                '--config', 'network={}'.format(net_id),
 303                '--config', 'ssl-hostname-verification=false',
 304                '--config', 'use-floating-ip=true',
 305                '--config', 'use-default-secgroup=true',
 306                '--debug']
 307         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 308         self.__logger.info("%s\n%s", " ".join(cmd), output)
 309         return True
 310
 311     def check_app(self, name='abot-epc-basic', status='active'):
 312         """Check application status."""
 313         cmd = ['juju', 'status', '--format', 'short', name]
 314         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 315         self.__logger.info("%s\n%s", " ".join(cmd), output)
 316         ret = re.search(r'(?=workload:({})\))'.format(status), output)
 317         if ret:
 318             self.__logger.info("%s workload is %s", name, status)
 319             return True
 320         self.__logger.error("%s workload differs from %s", name, status)
 321         return False
 322
 323     def deploy_vnf(self):
 324         """Deploy ABOT-OAI-EPC."""
 325         self.__logger.info("Upload VNFD")
 326         descriptor = self.vnf['descriptor']
 327         self.__logger.info("Get or create flavor for all Abot-EPC")
 328         flavor_settings = FlavorConfig(
 329             name=self.vnf['requirements']['flavor']['name'],
 330             ram=self.vnf['requirements']['flavor']['ram_min'],
 331             disk=10,
 332             vcpus=1)
 333         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 334         flavor_creator.create()
 335         self.created_object.append(flavor_creator)
 336         self.__logger.info("Deploying Abot-epc bundle file ...")
 337         cmd = ['juju', 'deploy', '{}'.format(descriptor.get('file_name'))]
 338         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 339         self.__logger.info("%s\n%s", " ".join(cmd), output)
 340         self.__logger.info("Waiting for instances .....")
 341         cmd = ['timeout', '-t', JujuEpc.juju_wait_timeout, 'juju-wait']
 342         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 343         self.__logger.info("%s\n%s", " ".join(cmd), output)
 344         self.__logger.info("Deployed Abot-epc on Openstack")
 345         nova_client = nova_utils.nova_client(self.snaps_creds)
 346         instances = get_instances(nova_client)
 347         self.__logger.info("List of Instance: %s", instances)
 348         for items in instances:
 349             metadata = get_instance_metadata(nova_client, items)
 350             if 'juju-units-deployed' in metadata:
 351                 sec_group = 'juju-{}-{}'.format(
 352                     metadata['juju-controller-uuid'],
 353                     metadata['juju-model-uuid'])
 354                 self.__logger.info("Instance: %s", sec_group)
 355                 break
 356         self.__logger.info("Adding Security group rule....")
 357         # This will add sctp rule to a common Security Group Created
 358         # by juju and shared to all deployed units.
 359         self._add_custom_rule(sec_group)
 360         cmd = ['juju', 'status']
 361         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 362         self.__logger.debug("%s\n%s", " ".join(cmd), output)
 363         for app in ['abot-epc-basic', 'oai-epc', 'oai-hss']:
 364             if not self.check_app(app):
 365                 return False
 366         self.__logger.info("Copying the feature files to Abot_node ")
 367         cmd = ['juju', 'scp', '--', '-r', '-v',
 368                '{}/featureFiles'.format(self.case_dir), 'abot-epc-basic/0:~/']
 369         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 370         self.__logger.info("%s\n%s", " ".join(cmd), output)
 371         self.__logger.info("Copying the feature files in Abot_node ")
 372         cmd = ['juju', 'ssh', 'abot-epc-basic/0',
 373                'sudo', 'rsync', '-azvv', '~/featureFiles',
 374                '/etc/rebaca-test-suite/featureFiles']
 375         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 376         self.__logger.info("%s\n%s", " ".join(cmd), output)
 377         return True
 378
 379     def test_vnf(self):
 380         """Run test on ABoT."""
 381         start_time = time.time()
 382         self.__logger.info("Running VNF Test cases....")
 383         cmd = ['juju', 'run-action', 'abot-epc-basic/0', 'run',
 384                'tagnames={}'.format(self.details['test_vnf']['tag_name'])]
 385         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 386         self.__logger.info("%s\n%s", " ".join(cmd), output)
 387         cmd = ['timeout', '-t', JujuEpc.juju_wait_timeout, 'juju-wait']
 388         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 389         self.__logger.info("%s\n%s", " ".join(cmd), output)
 390         duration = time.time() - start_time
 391         self.__logger.info("Getting results from Abot node....")
 392         cmd = ['juju', 'scp', '--', '-v',
 393                'abot-epc-basic/0:'
 394                '/var/lib/abot-epc-basic/artifacts/TestResults.json',
 395                '{}/.'.format(self.res_dir)]
 396         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 397         self.__logger.info("%s\n%s", " ".join(cmd), output)
 398         self.__logger.info("Parsing the Test results...")
 399         res = (process_abot_test_result('{}/TestResults.json'.format(
 400             self.res_dir)))
 401         short_result = sig_test_format(res)
 402         self.__logger.info(short_result)
 403         self.details['test_vnf'].update(
 404             status='PASS', result=short_result, full_result=res,
 405             duration=duration)
 406         self.__logger.info(
 407             "Test VNF result: Passed: %d, Failed:%d, Skipped: %d",
 408             short_result['passed'],
 409             short_result['failures'], short_result['skipped'])
 410         return True
 411
 412     def clean(self):
 413         """Clean created objects/functions."""
 414         try:
 415             cmd = ['juju', 'debug-log', '--replay', '--no-tail']
 416             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 417             self.__logger.debug("%s\n%s", " ".join(cmd), output)
 418             if not self.orchestrator['requirements']['preserve_setup']:
 419                 self.__logger.info("Destroying Orchestrator...")
 420                 cmd = ['juju', 'destroy-controller', '-y', 'abot-controller',
 421                        '--destroy-all-models']
 422                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 423                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 424         except Exception:  # pylint: disable=broad-except
 425             self.__logger.exception("Some issue during the undeployment ..")
 426
 427         if not self.orchestrator['requirements']['preserve_setup']:
 428             self.__logger.info('Remove the Abot_epc OS object ..')
 429             super(JujuEpc, self).clean()
 430
 431         return True
 432
 433
 434 # ----------------------------------------------------------
 435 #
 436 #               YAML UTILS
 437 #
 438 # -----------------------------------------------------------
 439 def get_config(parameter, file_path):
 440     """
 441     Returns the value of a given parameter in file.yaml
 442     parameter must be given in string format with dots
 443     Example: general.openstack.image_name
 444     """
 445     with open(file_path) as config_file:
 446         file_yaml = yaml.safe_load(config_file)
 447     config_file.close()
 448     value = file_yaml
 449     for element in parameter.split("."):
 450         value = value.get(element)
 451         if value is None:
 452             raise ValueError("The parameter %s is not defined in"
 453                              " reporting.yaml" % parameter)
 454     return value
 455
 456
 457 def sig_test_format(sig_test):
 458     """
 459     Process the signaling result to have a short result
 460     """
 461     nb_passed = 0
 462     nb_failures = 0
 463     nb_skipped = 0
 464     for data_test in sig_test:
 465         if data_test['result'] == "passed":
 466             nb_passed += 1
 467         elif data_test['result'] == "failed":
 468             nb_failures += 1
 469         elif data_test['result'] == "skipped":
 470             nb_skipped += 1
 471     total_sig_test_result = {}
 472     total_sig_test_result['passed'] = nb_passed
 473     total_sig_test_result['failures'] = nb_failures
 474     total_sig_test_result['skipped'] = nb_skipped
 475     return total_sig_test_result
 476
 477
 478 def process_abot_test_result(file_path):
 479     """ Process ABoT Result """
 480     with open(file_path) as test_result:
 481         data = json.load(test_result)
 482         res = []
 483         for tests in data:
 484             tests = update_data(tests)
 485             try:
 486                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 487                 for steps in flatten_steps:
 488                     steps['result'] = steps['step_status']
 489                     res.append(steps)
 490             except:
 491                 logging.error("Could not post data to ElasticSearch host")
 492                 raise
 493         return res
 494
 495
 496 def update_data(obj):
 497     """ Update Result data"""
 498     try:
 499         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 500
 501         for element in obj['elements']:
 502             element['final_result'] = "passed"
 503             element['flatten_steps'] = []
 504
 505             for step in element['steps']:
 506                 temp_dict = {}
 507                 step['result'][step['result']['status']] = 1
 508                 if step['result']['status'].lower() in ['fail', 'failed']:
 509                     element['final_result'] = "failed"
 510
 511                 temp_dict['feature_file'] = obj['feature_file']
 512                 temp_dict['step_name'] = step['name']
 513                 temp_dict['step_status'] = step['result']['status']
 514                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 515                 temp_dict['step_' + step['result']['status']] = 1
 516                 element['flatten_steps'].append(deepcopy(temp_dict))
 517
 518             # Need to put the tag in OBJ and not ELEMENT
 519             if 'tags' in obj:
 520                 element['tags'] = deepcopy(obj['tags'])
 521                 for tag in obj['tags']:
 522                     element[tag['name']] = 1
 523             else:
 524                 for tag in element['tags']:
 525                     element[tag['name']] = 1
 526
 527     except Exception:  # pylint: disable=broad-except
 528         logging.error("Error in updating data, %s", sys.exc_info()[0])
 529         raise
 530
 531     return obj
 532
 533
 534 def get_instances(nova_client):
 535     """ To get all vm info of a project """
 536     try:
 537         instances = nova_client.servers.list()
 538         return instances
 539     except Exception as exc:  # pylint: disable=broad-except
 540         logging.error("Error [get_instances(nova_client)]: %s", exc)
 541         return None
 542
 543
 544 def get_instance_metadata(nova_client, instance):
 545     """ Get instance Metadata - Instance ID """
 546     try:
 547         instance = nova_client.servers.get(instance.id)
 548         return instance.metadata
 549     except Exception as exc:  # pylint: disable=broad-except
 550         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 551         return None