functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import subprocess
  17 import sys
  18 import uuid
  19 from copy import deepcopy
  20 from urlparse import urljoin
  21 import pkg_resources
  22 import yaml
  23
  24 from functest.core import vnf
  25 from functest.opnfv_tests.openstack.snaps import snaps_utils
  26 from functest.utils import config
  27 from functest.utils import env
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.image import ImageConfig
  31 from snaps.config.network import NetworkConfig, SubnetConfig
  32 from snaps.config.router import RouterConfig
  33 from snaps.config.security_group import (
  34     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  35 from snaps.config.user import UserConfig
  36 from snaps.openstack.create_flavor import OpenStackFlavor
  37 from snaps.openstack.create_image import OpenStackImage
  38 from snaps.openstack.create_network import OpenStackNetwork
  39 from snaps.openstack.create_router import OpenStackRouter
  40 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  41 from snaps.openstack.create_user import OpenStackUser
  42 from snaps.openstack.utils import keystone_utils
  43 from snaps.openstack.utils import nova_utils
  44
  45 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  46 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  47
  48 CLOUD_TEMPLATE = """clouds:
  49   abot-epc:
  50     type: openstack
  51     auth-types: [userpass]
  52     endpoint: {url}
  53     regions:
  54       {region}:
  55         endpoint: {url}"""
  56
  57 CREDS_TEMPLATE2 = """credentials:
  58   abot-epc:
  59     default-credential: abot-epc
  60     abot-epc:
  61       auth-type: userpass
  62       password: {pass}
  63       project-domain-name: {project_domain_n}
  64       tenant-name: {tenant_n}"""
  65
  66 CREDS_TEMPLATE3 = """credentials:
  67   abot-epc:
  68     default-credential: abot-epc
  69     abot-epc:
  70       auth-type: userpass
  71       password: {pass}
  72       project-domain-name: {project_domain_n}
  73       tenant-name: {tenant_n}
  74       user-domain-name: {user_domain_n}
  75       username: {user_n}"""
  76
  77
  78 class JujuEpc(vnf.VnfOnBoarding):
  79     # pylint:disable=too-many-instance-attributes
  80     """Abot EPC deployed with JUJU Orchestrator Case"""
  81
  82     __logger = logging.getLogger(__name__)
  83
  84     def __init__(self, **kwargs):
  85         if "case_name" not in kwargs:
  86             kwargs["case_name"] = "juju_epc"
  87         super(JujuEpc, self).__init__(**kwargs)
  88
  89         # Retrieve the configuration
  90         self.case_dir = pkg_resources.resource_filename(
  91             'functest', 'opnfv_tests/vnf/epc')
  92         try:
  93             self.config = getattr(
  94                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  95         except Exception:
  96             raise Exception("VNF config file not found")
  97         self.config_file = os.path.join(self.case_dir, self.config)
  98         self.orchestrator = dict(requirements=get_config(
  99             "orchestrator.requirements", self.config_file))
 100
 101         self.created_object = []
 102         self.details['orchestrator'] = dict(
 103             name=get_config("orchestrator.name", self.config_file),
 104             version=get_config("orchestrator.version", self.config_file),
 105             status='ERROR',
 106             result=''
 107         )
 108
 109         self.vnf = dict(
 110             descriptor=get_config("vnf.descriptor", self.config_file),
 111             requirements=get_config("vnf.requirements", self.config_file)
 112         )
 113         self.details['vnf'] = dict(
 114             descriptor_version=self.vnf['descriptor']['version'],
 115             name=get_config("vnf.name", self.config_file),
 116             version=get_config("vnf.version", self.config_file),
 117         )
 118         self.__logger.debug("VNF configuration: %s", self.vnf)
 119
 120         self.details['test_vnf'] = dict(
 121             name=get_config("vnf_test_suite.name", self.config_file),
 122             version=get_config("vnf_test_suite.version", self.config_file),
 123             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 124         )
 125         self.public_auth_url = None
 126
 127         self.res_dir = os.path.join(
 128             getattr(config.CONF, 'dir_results'), self.case_name)
 129
 130     def _bypass_juju_network_discovery_bug(self, name):
 131         user_creator = OpenStackUser(
 132             self.snaps_creds,
 133             UserConfig(
 134                 name=name,
 135                 password=str(uuid.uuid4()),
 136                 project_name=self.tenant_name,
 137                 domain=self.snaps_creds.user_domain_name,
 138                 roles={'_member_': self.tenant_name}))
 139         user_creator.create()
 140         self.created_object.append(user_creator)
 141         return user_creator
 142
 143     def _register_cloud(self):
 144         self.__logger.info("Creating Cloud for Abot-epc .....")
 145         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 146         cloud_data = {
 147             'url': self.public_auth_url,
 148             'region': self.snaps_creds.region_name}
 149         with open(clouds_yaml, 'w') as yfile:
 150             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 151         cmd = ['juju', 'add-cloud', 'abot-epc', '-f', clouds_yaml, '--replace']
 152         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 153         self.__logger.info("%s\n%s", " ".join(cmd), output)
 154
 155     def _register_credentials_v2(self):
 156         self.__logger.info("Creating Credentials for Abot-epc .....")
 157         user_creator = self._bypass_juju_network_discovery_bug(
 158             'juju_network_discovery_bug')
 159         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 160         self.__logger.debug("snaps creds: %s", snaps_creds)
 161         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 162         creds_data = {
 163             'pass': snaps_creds.password,
 164             'tenant_n': snaps_creds.project_name,
 165             'user_n': snaps_creds.username}
 166         with open(credentials_yaml, 'w') as yfile:
 167             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 168         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 169                '--replace']
 170         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 171         self.__logger.info("%s\n%s", " ".join(cmd), output)
 172
 173     def _register_credentials_v3(self):
 174         self.__logger.info("Creating Credentials for Abot-epc .....")
 175         user_creator = self._bypass_juju_network_discovery_bug(
 176             'juju_network_discovery_bug')
 177         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 178         self.__logger.debug("snaps creds: %s", snaps_creds)
 179         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 180         creds_data = {
 181             'pass': snaps_creds.password,
 182             'tenant_n': snaps_creds.project_name,
 183             'user_n': snaps_creds.username,
 184             'project_domain_n': snaps_creds.project_domain_name,
 185             'user_domain_n': snaps_creds.user_domain_name}
 186         with open(credentials_yaml, 'w') as yfile:
 187             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 188         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 189                '--replace']
 190         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 191         self.__logger.info("%s\n%s", " ".join(cmd), output)
 192
 193     def _add_custom_rule(self, sec_grp_name):
 194         """ To add custom rule for SCTP Traffic """
 195         sec_grp_rules = list()
 196         sec_grp_rules.append(
 197             SecurityGroupRuleConfig(
 198                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 199                 protocol=Protocol.sctp))
 200         security_group = OpenStackSecurityGroup(
 201             self.snaps_creds,
 202             SecurityGroupConfig(
 203                 name=sec_grp_name,
 204                 rule_settings=sec_grp_rules))
 205         security_group.create()
 206         self.created_object.append(security_group)
 207
 208     def prepare(self):
 209         """Prepare testcase (Additional pre-configuration steps)."""
 210         self.__logger.info("Additional pre-configuration steps")
 211         super(JujuEpc, self).prepare()
 212         try:
 213             os.makedirs(self.res_dir)
 214         except OSError as ex:
 215             if ex.errno != errno.EEXIST:
 216                 self.__logger.exception("Cannot create %s", self.res_dir)
 217                 raise vnf.VnfPreparationException
 218         self.public_auth_url = keystone_utils.get_endpoint(
 219             self.snaps_creds, 'identity')
 220         # it enforces a versioned public identity endpoint as juju simply
 221         # adds /auth/tokens wich fails vs an unversioned endpoint.
 222         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 223             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 224         self._register_cloud()
 225         if self.snaps_creds.identity_api_version == 3:
 226             self._register_credentials_v3()
 227         else:
 228             self._register_credentials_v2()
 229
 230     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 231         """
 232         Create network, subnet, router
 233
 234         Bootstrap juju
 235         """
 236         self.__logger.info("Deployed Orchestrator")
 237         private_net_name = getattr(
 238             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 239         private_subnet_name = '{}-{}'.format(
 240             getattr(config.CONF,
 241                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 242             self.uuid)
 243         private_subnet_cidr = getattr(
 244             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 245         abot_router = '{}-{}'.format(
 246             getattr(config.CONF,
 247                     'vnf_{}_external_router'.format(self.case_name)),
 248             self.uuid)
 249         self.__logger.info("Creating full network ...")
 250         subnet_settings = SubnetConfig(
 251             name=private_subnet_name,
 252             cidr=private_subnet_cidr,
 253             dns_nameservers=[env.get('NAMESERVER')])
 254         network_settings = NetworkConfig(
 255             name=private_net_name, subnet_settings=[subnet_settings])
 256         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 257         net_id = network_creator.create().id
 258         self.created_object.append(network_creator)
 259
 260         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 261         self.__logger.info("Creating network Router ....")
 262         router_creator = OpenStackRouter(
 263             self.snaps_creds, RouterConfig(
 264                 name=abot_router,
 265                 external_gateway=ext_net_name,
 266                 internal_subnets=[subnet_settings.name]))
 267         router_creator.create()
 268         self.created_object.append(router_creator)
 269         self.__logger.info("Creating Flavor ....")
 270         flavor_settings = FlavorConfig(
 271             name=self.orchestrator['requirements']['flavor']['name'],
 272             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 273             disk=10, vcpus=1)
 274         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 275         flavor_creator.create()
 276         self.created_object.append(flavor_creator)
 277         self.__logger.info("Upload some OS images if it doesn't exist")
 278         images = get_config("tenant_images", self.config_file)
 279         self.__logger.info("Images needed for vEPC: %s", images)
 280         for image_name, image_file in images.iteritems():
 281             self.__logger.info("image: %s, file: %s", image_name, image_file)
 282             if image_file and image_name:
 283                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 284                     name=image_name, image_user='cloud', img_format='qcow2',
 285                     image_file=image_file))
 286                 image_id = image_creator.create().id
 287                 cmd = ['juju', 'metadata', 'generate-image', '-d', '/root',
 288                        '-i', image_id, '-s', image_name,
 289                        '-r', self.snaps_creds.region_name,
 290                        '-u', self.public_auth_url]
 291                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 292                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 293                 self.created_object.append(image_creator)
 294         self.__logger.info("Network ID  : %s", net_id)
 295         cmd = ['juju', 'bootstrap', 'abot-epc', 'abot-controller',
 296                '--metadata-source', '/root',
 297                '--constraints', 'mem=2G',
 298                '--bootstrap-series', 'xenial',
 299                '--config', 'network={}'.format(net_id),
 300                '--config', 'ssl-hostname-verification=false',
 301                '--config', 'use-floating-ip=true',
 302                '--config', 'use-default-secgroup=true',
 303                '--debug']
 304         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 305         self.__logger.info("%s\n%s", " ".join(cmd), output)
 306         return True
 307
 308     def deploy_vnf(self):
 309         """Deploy ABOT-OAI-EPC."""
 310         self.__logger.info("Upload VNFD")
 311         descriptor = self.vnf['descriptor']
 312         self.__logger.info("Get or create flavor for all Abot-EPC")
 313         flavor_settings = FlavorConfig(
 314             name=self.vnf['requirements']['flavor']['name'],
 315             ram=self.vnf['requirements']['flavor']['ram_min'],
 316             disk=10,
 317             vcpus=1)
 318         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 319         flavor_creator.create()
 320         self.created_object.append(flavor_creator)
 321         self.__logger.info("Deploying Abot-epc bundle file ...")
 322         cmd = ['juju', 'deploy', '/{}'.format(descriptor.get('file_name'))]
 323         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 324         self.__logger.info("%s\n%s", " ".join(cmd), output)
 325         self.__logger.info("Waiting for instances .....")
 326         cmd = ['juju-wait']
 327         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 328         self.__logger.info("%s\n%s", " ".join(cmd), output)
 329         self.__logger.info("Deployed Abot-epc on Openstack")
 330         nova_client = nova_utils.nova_client(self.snaps_creds)
 331         instances = get_instances(nova_client)
 332         self.__logger.info("List of Instance: %s", instances)
 333         for items in instances:
 334             metadata = get_instance_metadata(nova_client, items)
 335             if 'juju-units-deployed' in metadata:
 336                 sec_group = 'juju-{}-{}'.format(
 337                     metadata['juju-controller-uuid'],
 338                     metadata['juju-model-uuid'])
 339                 self.__logger.info("Instance: %s", sec_group)
 340                 break
 341         self.__logger.info("Adding Security group rule....")
 342         # This will add sctp rule to a common Security Group Created
 343         # by juju and shared to all deployed units.
 344         self._add_custom_rule(sec_group)
 345         self.__logger.info("Copying the feature files to Abot_node ")
 346         cmd = ['juju', 'scp', '--', '-r',
 347                '{}/featureFiles'.format(self.case_dir), 'abot-epc-basic/0:~/']
 348         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 349         self.__logger.info("%s\n%s", " ".join(cmd), output)
 350         self.__logger.info("Copying the feature files in Abot_node ")
 351         cmd = ['juju', 'ssh', 'abot-epc-basic/0',
 352                'sudo', 'rsync', '-azvv', '~/featureFiles',
 353                '/etc/rebaca-test-suite/featureFiles']
 354         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 355         self.__logger.info("%s\n%s", " ".join(cmd), output)
 356         count = 0
 357         epcstatus = 1
 358         while count < 10:
 359             epcstatus = os.system(
 360                 'juju status oai-epc | grep {} | grep {} | grep {}'.format(
 361                     'EPC', 'is', 'running'))
 362             if epcstatus == 0:
 363                 break
 364             else:
 365                 time.sleep(60)
 366                 count = count + 1
 367         if epcstatus != 0:
 368             return False
 369         cmd = ['juju-wait']
 370         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 371         self.__logger.info("%s\n%s", " ".join(cmd), output)
 372         return True
 373
 374     def test_vnf(self):
 375         """Run test on ABoT."""
 376         start_time = time.time()
 377         self.__logger.info("Running VNF Test cases....")
 378         cmd = ['juju', 'run-action', 'abot-epc-basic/0', 'run',
 379                'tagnames={}'.format(self.details['test_vnf']['tag_name'])]
 380         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 381         self.__logger.info("%s\n%s", " ".join(cmd), output)
 382         cmd = ['juju-wait']
 383         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 384         self.__logger.info("%s\n%s", " ".join(cmd), output)
 385         duration = time.time() - start_time
 386         self.__logger.info("Getting results from Abot node....")
 387         cmd = ['juju', 'scp',
 388                'abot-epc-basic/0:'
 389                '/var/lib/abot-epc-basic/artifacts/TestResults.json',
 390                '{}/.'.format(self.res_dir)]
 391         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 392         self.__logger.info("%s\n%s", " ".join(cmd), output)
 393         self.__logger.info("Parsing the Test results...")
 394         res = (process_abot_test_result('{}/TestResults.json'.format(
 395             self.res_dir)))
 396         short_result = sig_test_format(res)
 397         self.__logger.info(short_result)
 398         self.details['test_vnf'].update(
 399             status='PASS', result=short_result, full_result=res,
 400             duration=duration)
 401         self.__logger.info(
 402             "Test VNF result: Passed: %d, Failed:%d, Skipped: %d",
 403             short_result['passed'],
 404             short_result['failures'], short_result['skipped'])
 405         return True
 406
 407     def clean(self):
 408         """Clean created objects/functions."""
 409         try:
 410             if not self.orchestrator['requirements']['preserve_setup']:
 411                 self.__logger.info("Destroying Orchestrator...")
 412                 cmd = ['juju', 'destroy-controller', '-y', 'abot-controller',
 413                        '--destroy-all-models']
 414                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 415                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 416         except Exception:  # pylint: disable=broad-except
 417             self.__logger.warn("Some issue during the undeployment ..")
 418             self.__logger.warn("Tenant clean continue ..")
 419
 420         if not self.orchestrator['requirements']['preserve_setup']:
 421             self.__logger.info('Remove the Abot_epc OS object ..')
 422             super(JujuEpc, self).clean()
 423
 424         return True
 425
 426
 427 # ----------------------------------------------------------
 428 #
 429 #               YAML UTILS
 430 #
 431 # -----------------------------------------------------------
 432 def get_config(parameter, file_path):
 433     """
 434     Returns the value of a given parameter in file.yaml
 435     parameter must be given in string format with dots
 436     Example: general.openstack.image_name
 437     """
 438     with open(file_path) as config_file:
 439         file_yaml = yaml.safe_load(config_file)
 440     config_file.close()
 441     value = file_yaml
 442     for element in parameter.split("."):
 443         value = value.get(element)
 444         if value is None:
 445             raise ValueError("The parameter %s is not defined in"
 446                              " reporting.yaml" % parameter)
 447     return value
 448
 449
 450 def sig_test_format(sig_test):
 451     """
 452     Process the signaling result to have a short result
 453     """
 454     nb_passed = 0
 455     nb_failures = 0
 456     nb_skipped = 0
 457     for data_test in sig_test:
 458         if data_test['result'] == "passed":
 459             nb_passed += 1
 460         elif data_test['result'] == "failed":
 461             nb_failures += 1
 462         elif data_test['result'] == "skipped":
 463             nb_skipped += 1
 464     total_sig_test_result = {}
 465     total_sig_test_result['passed'] = nb_passed
 466     total_sig_test_result['failures'] = nb_failures
 467     total_sig_test_result['skipped'] = nb_skipped
 468     return total_sig_test_result
 469
 470
 471 def process_abot_test_result(file_path):
 472     """ Process ABoT Result """
 473     with open(file_path) as test_result:
 474         data = json.load(test_result)
 475         res = []
 476         for tests in data:
 477             tests = update_data(tests)
 478             try:
 479                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 480                 for steps in flatten_steps:
 481                     steps['result'] = steps['step_status']
 482                     res.append(steps)
 483             except:
 484                 logging.error("Could not post data to ElasticSearch host")
 485                 raise
 486         return res
 487
 488
 489 def update_data(obj):
 490     """ Update Result data"""
 491     try:
 492         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 493
 494         for element in obj['elements']:
 495             element['final_result'] = "passed"
 496             element['flatten_steps'] = []
 497
 498             for step in element['steps']:
 499                 temp_dict = {}
 500                 step['result'][step['result']['status']] = 1
 501                 if step['result']['status'].lower() in ['fail', 'failed']:
 502                     element['final_result'] = "failed"
 503
 504                 temp_dict['feature_file'] = obj['feature_file']
 505                 temp_dict['step_name'] = step['name']
 506                 temp_dict['step_status'] = step['result']['status']
 507                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 508                 temp_dict['step_' + step['result']['status']] = 1
 509                 element['flatten_steps'].append(deepcopy(temp_dict))
 510
 511             # Need to put the tag in OBJ and not ELEMENT
 512             if 'tags' in obj:
 513                 element['tags'] = deepcopy(obj['tags'])
 514                 for tag in obj['tags']:
 515                     element[tag['name']] = 1
 516             else:
 517                 for tag in element['tags']:
 518                     element[tag['name']] = 1
 519
 520     except Exception:  # pylint: disable=broad-except
 521         logging.error("Error in updating data, %s", sys.exc_info()[0])
 522         raise
 523
 524     return obj
 525
 526
 527 def get_instances(nova_client):
 528     """ To get all vm info of a project """
 529     try:
 530         instances = nova_client.servers.list()
 531         return instances
 532     except Exception as exc:  # pylint: disable=broad-except
 533         logging.error("Error [get_instances(nova_client)]: %s", exc)
 534         return None
 535
 536
 537 def get_instance_metadata(nova_client, instance):
 538     """ Get instance Metadata - Instance ID """
 539     try:
 540         instance = nova_client.servers.get(instance.id)
 541         return instance.metadata
 542     except Exception as exc:  # pylint: disable=broad-except
 543         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 544         return None