functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import subprocess
  17 import sys
  18 import uuid
  19 from copy import deepcopy
  20 from urlparse import urljoin
  21 import pkg_resources
  22 import yaml
  23
  24 from functest.core import vnf
  25 from functest.opnfv_tests.openstack.snaps import snaps_utils
  26 from functest.utils import config
  27 from functest.utils import env
  28
  29 from snaps.config.flavor import FlavorConfig
  30 from snaps.config.image import ImageConfig
  31 from snaps.config.network import NetworkConfig, SubnetConfig
  32 from snaps.config.router import RouterConfig
  33 from snaps.config.security_group import (
  34     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  35 from snaps.config.user import UserConfig
  36 from snaps.openstack.create_flavor import OpenStackFlavor
  37 from snaps.openstack.create_image import OpenStackImage
  38 from snaps.openstack.create_network import OpenStackNetwork
  39 from snaps.openstack.create_router import OpenStackRouter
  40 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  41 from snaps.openstack.create_user import OpenStackUser
  42 from snaps.openstack.utils import keystone_utils
  43 from snaps.openstack.utils import nova_utils
  44
  45 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  46 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  47
  48 CLOUD_TEMPLATE = """clouds:
  49   abot-epc:
  50     type: openstack
  51     auth-types: [userpass]
  52     endpoint: {url}
  53     regions:
  54       {region}:
  55         endpoint: {url}"""
  56
  57 CREDS_TEMPLATE2 = """credentials:
  58   abot-epc:
  59     default-credential: abot-epc
  60     abot-epc:
  61       auth-type: userpass
  62       password: {pass}
  63       project-domain-name: {project_domain_n}
  64       tenant-name: {tenant_n}"""
  65
  66 CREDS_TEMPLATE3 = """credentials:
  67   abot-epc:
  68     default-credential: abot-epc
  69     abot-epc:
  70       auth-type: userpass
  71       password: {pass}
  72       project-domain-name: {project_domain_n}
  73       tenant-name: {tenant_n}
  74       user-domain-name: {user_domain_n}
  75       username: {user_n}"""
  76
  77
  78 class JujuEpc(vnf.VnfOnBoarding):
  79     # pylint:disable=too-many-instance-attributes
  80     """Abot EPC deployed with JUJU Orchestrator Case"""
  81
  82     __logger = logging.getLogger(__name__)
  83
  84     def __init__(self, **kwargs):
  85         if "case_name" not in kwargs:
  86             kwargs["case_name"] = "juju_epc"
  87         super(JujuEpc, self).__init__(**kwargs)
  88
  89         # Retrieve the configuration
  90         self.case_dir = pkg_resources.resource_filename(
  91             'functest', 'opnfv_tests/vnf/epc')
  92         try:
  93             self.config = getattr(
  94                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  95         except Exception:
  96             raise Exception("VNF config file not found")
  97         self.config_file = os.path.join(self.case_dir, self.config)
  98         self.orchestrator = dict(requirements=get_config(
  99             "orchestrator.requirements", self.config_file))
 100
 101         self.created_object = []
 102         self.details['orchestrator'] = dict(
 103             name=get_config("orchestrator.name", self.config_file),
 104             version=get_config("orchestrator.version", self.config_file),
 105             status='ERROR',
 106             result=''
 107         )
 108
 109         self.vnf = dict(
 110             descriptor=get_config("vnf.descriptor", self.config_file),
 111             requirements=get_config("vnf.requirements", self.config_file)
 112         )
 113         self.details['vnf'] = dict(
 114             descriptor_version=self.vnf['descriptor']['version'],
 115             name=get_config("vnf.name", self.config_file),
 116             version=get_config("vnf.version", self.config_file),
 117         )
 118         self.__logger.debug("VNF configuration: %s", self.vnf)
 119
 120         self.details['test_vnf'] = dict(
 121             name=get_config("vnf_test_suite.name", self.config_file),
 122             version=get_config("vnf_test_suite.version", self.config_file),
 123             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 124         )
 125         self.public_auth_url = None
 126
 127         self.res_dir = os.path.join(
 128             getattr(config.CONF, 'dir_results'), self.case_name)
 129
 130     def _bypass_juju_network_discovery_bug(self, name):
 131         user_creator = OpenStackUser(
 132             self.snaps_creds,
 133             UserConfig(
 134                 name=name,
 135                 password=str(uuid.uuid4()),
 136                 project_name=self.tenant_name,
 137                 domain=self.snaps_creds.user_domain_name,
 138                 roles={'_member_': self.tenant_name}))
 139         user_creator.create()
 140         self.created_object.append(user_creator)
 141         return user_creator
 142
 143     def _register_cloud(self):
 144         self.__logger.info("Creating Cloud for Abot-epc .....")
 145         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 146         cloud_data = {
 147             'url': self.public_auth_url,
 148             'region': self.snaps_creds.region_name}
 149         with open(clouds_yaml, 'w') as yfile:
 150             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 151         cmd = ['juju', 'add-cloud', 'abot-epc', '-f', clouds_yaml, '--replace']
 152         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 153         self.__logger.info("%s\n%s", " ".join(cmd), output)
 154
 155     def _register_credentials_v2(self):
 156         self.__logger.info("Creating Credentials for Abot-epc .....")
 157         user_creator = self._bypass_juju_network_discovery_bug(
 158             'juju_network_discovery_bug')
 159         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 160         self.__logger.debug("snaps creds: %s", snaps_creds)
 161         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 162         creds_data = {
 163             'pass': snaps_creds.password,
 164             'tenant_n': snaps_creds.project_name,
 165             'user_n': snaps_creds.username}
 166         with open(credentials_yaml, 'w') as yfile:
 167             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 168         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 169                '--replace']
 170         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 171         self.__logger.info("%s\n%s", " ".join(cmd), output)
 172
 173     def _register_credentials_v3(self):
 174         self.__logger.info("Creating Credentials for Abot-epc .....")
 175         user_creator = self._bypass_juju_network_discovery_bug(
 176             'juju_network_discovery_bug')
 177         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 178         self.__logger.debug("snaps creds: %s", snaps_creds)
 179         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 180         creds_data = {
 181             'pass': snaps_creds.password,
 182             'tenant_n': snaps_creds.project_name,
 183             'user_n': snaps_creds.username,
 184             'project_domain_n': snaps_creds.project_domain_name,
 185             'user_domain_n': snaps_creds.user_domain_name}
 186         with open(credentials_yaml, 'w') as yfile:
 187             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 188         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 189                '--replace']
 190         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 191         self.__logger.info("%s\n%s", " ".join(cmd), output)
 192
 193     def _add_custom_rule(self, sec_grp_name):
 194         """ To add custom rule for SCTP Traffic """
 195         sec_grp_rules = list()
 196         sec_grp_rules.append(
 197             SecurityGroupRuleConfig(
 198                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 199                 protocol=Protocol.sctp))
 200         security_group = OpenStackSecurityGroup(
 201             self.snaps_creds,
 202             SecurityGroupConfig(
 203                 name=sec_grp_name,
 204                 rule_settings=sec_grp_rules))
 205         security_group.create()
 206         self.created_object.append(security_group)
 207
 208     def prepare(self):
 209         """Prepare testcase (Additional pre-configuration steps)."""
 210         self.__logger.info("Additional pre-configuration steps")
 211         super(JujuEpc, self).prepare()
 212         try:
 213             os.makedirs(self.res_dir)
 214         except OSError as ex:
 215             if ex.errno != errno.EEXIST:
 216                 self.__logger.exception("Cannot create %s", self.res_dir)
 217                 raise vnf.VnfPreparationException
 218         self.public_auth_url = keystone_utils.get_endpoint(
 219             self.snaps_creds, 'identity')
 220         # it enforces a versioned public identity endpoint as juju simply
 221         # adds /auth/tokens wich fails vs an unversioned endpoint.
 222         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 223             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 224         self._register_cloud()
 225         if self.snaps_creds.identity_api_version == 3:
 226             self._register_credentials_v3()
 227         else:
 228             self._register_credentials_v2()
 229
 230     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 231         """
 232         Create network, subnet, router
 233
 234         Bootstrap juju
 235         """
 236         self.__logger.info("Deployed Orchestrator")
 237         private_net_name = getattr(
 238             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 239         private_subnet_name = '{}-{}'.format(
 240             getattr(config.CONF,
 241                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 242             self.uuid)
 243         private_subnet_cidr = getattr(
 244             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 245         abot_router = '{}-{}'.format(
 246             getattr(config.CONF,
 247                     'vnf_{}_external_router'.format(self.case_name)),
 248             self.uuid)
 249         self.__logger.info("Creating full network ...")
 250         subnet_settings = SubnetConfig(
 251             name=private_subnet_name,
 252             cidr=private_subnet_cidr,
 253             dns_nameservers=[env.get('NAMESERVER')])
 254         network_settings = NetworkConfig(
 255             name=private_net_name, subnet_settings=[subnet_settings])
 256         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 257         net_id = network_creator.create().id
 258         self.created_object.append(network_creator)
 259
 260         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 261         self.__logger.info("Creating network Router ....")
 262         router_creator = OpenStackRouter(
 263             self.snaps_creds, RouterConfig(
 264                 name=abot_router,
 265                 external_gateway=ext_net_name,
 266                 internal_subnets=[subnet_settings.name]))
 267         router_creator.create()
 268         self.created_object.append(router_creator)
 269         self.__logger.info("Creating Flavor ....")
 270         flavor_settings = FlavorConfig(
 271             name=self.orchestrator['requirements']['flavor']['name'],
 272             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 273             disk=10, vcpus=1)
 274         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 275         flavor_creator.create()
 276         self.created_object.append(flavor_creator)
 277         self.__logger.info("Upload some OS images if it doesn't exist")
 278         images = get_config("tenant_images", self.config_file)
 279         self.__logger.info("Images needed for vEPC: %s", images)
 280         for image_name, image_file in images.iteritems():
 281             self.__logger.info("image: %s, file: %s", image_name, image_file)
 282             if image_file and image_name:
 283                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 284                     name=image_name, image_user='cloud', img_format='qcow2',
 285                     image_file=image_file))
 286                 image_id = image_creator.create().id
 287                 cmd = ['juju', 'metadata', 'generate-image', '-d', '/root',
 288                        '-i', image_id, '-s', image_name,
 289                        '-r', self.snaps_creds.region_name,
 290                        '-u', self.public_auth_url]
 291                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 292                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 293                 self.created_object.append(image_creator)
 294         self.__logger.info("Network ID  : %s", net_id)
 295         cmd = ['juju', 'bootstrap', 'abot-epc', 'abot-controller',
 296                '--metadata-source', '/root',
 297                '--constraints', 'mem=2G',
 298                '--bootstrap-series', 'xenial',
 299                '--config', 'network={}'.format(net_id),
 300                '--config', 'ssl-hostname-verification=false',
 301                '--config', 'use-floating-ip=true',
 302                '--config', 'use-default-secgroup=true',
 303                '--debug']
 304         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 305         self.__logger.info("%s\n%s", " ".join(cmd), output)
 306         return True
 307
 308     def deploy_vnf(self):
 309         """Deploy ABOT-OAI-EPC."""
 310         self.__logger.info("Upload VNFD")
 311         descriptor = self.vnf['descriptor']
 312         self.__logger.info("Get or create flavor for all Abot-EPC")
 313         flavor_settings = FlavorConfig(
 314             name=self.vnf['requirements']['flavor']['name'],
 315             ram=self.vnf['requirements']['flavor']['ram_min'],
 316             disk=10,
 317             vcpus=1)
 318         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 319         flavor_creator.create()
 320         self.created_object.append(flavor_creator)
 321         self.__logger.info("Deploying Abot-epc bundle file ...")
 322         cmd = ['juju', 'deploy', '{}'.format(descriptor.get('file_name'))]
 323         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 324         self.__logger.info("%s\n%s", " ".join(cmd), output)
 325         self.__logger.info("Waiting for instances .....")
 326         cmd = ['juju-wait']
 327         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 328         self.__logger.info("%s\n%s", " ".join(cmd), output)
 329         self.__logger.info("Deployed Abot-epc on Openstack")
 330         nova_client = nova_utils.nova_client(self.snaps_creds)
 331         instances = get_instances(nova_client)
 332         self.__logger.info("List of Instance: %s", instances)
 333         for items in instances:
 334             metadata = get_instance_metadata(nova_client, items)
 335             if 'juju-units-deployed' in metadata:
 336                 sec_group = 'juju-{}-{}'.format(
 337                     metadata['juju-controller-uuid'],
 338                     metadata['juju-model-uuid'])
 339                 self.__logger.info("Instance: %s", sec_group)
 340                 break
 341         self.__logger.info("Adding Security group rule....")
 342         # This will add sctp rule to a common Security Group Created
 343         # by juju and shared to all deployed units.
 344         self._add_custom_rule(sec_group)
 345         cmd = ['juju', 'status']
 346         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 347         self.__logger.debug("%s\n%s", " ".join(cmd), output)
 348         self.__logger.info("Copying the feature files to Abot_node ")
 349         cmd = ['juju', 'scp', '--', '-r', '-v',
 350                '{}/featureFiles'.format(self.case_dir), 'abot-epc-basic/0:~/']
 351         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 352         self.__logger.info("%s\n%s", " ".join(cmd), output)
 353         self.__logger.info("Copying the feature files in Abot_node ")
 354         cmd = ['juju', 'ssh', 'abot-epc-basic/0',
 355                'sudo', 'rsync', '-azvv', '~/featureFiles',
 356                '/etc/rebaca-test-suite/featureFiles']
 357         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 358         self.__logger.info("%s\n%s", " ".join(cmd), output)
 359         count = 0
 360         epcstatus = 1
 361         while count < 10:
 362             epcstatus = os.system(
 363                 'juju status oai-epc | grep {} | grep {} | grep {}'.format(
 364                     'EPC', 'is', 'running'))
 365             if epcstatus == 0:
 366                 break
 367             else:
 368                 time.sleep(60)
 369                 count = count + 1
 370         if epcstatus != 0:
 371             return False
 372         cmd = ['juju-wait']
 373         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 374         self.__logger.info("%s\n%s", " ".join(cmd), output)
 375         return True
 376
 377     def test_vnf(self):
 378         """Run test on ABoT."""
 379         start_time = time.time()
 380         self.__logger.info("Running VNF Test cases....")
 381         cmd = ['juju', 'run-action', 'abot-epc-basic/0', 'run',
 382                'tagnames={}'.format(self.details['test_vnf']['tag_name'])]
 383         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 384         self.__logger.info("%s\n%s", " ".join(cmd), output)
 385         cmd = ['juju-wait']
 386         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 387         self.__logger.info("%s\n%s", " ".join(cmd), output)
 388         duration = time.time() - start_time
 389         self.__logger.info("Getting results from Abot node....")
 390         cmd = ['juju', 'scp', '--', '-v',
 391                'abot-epc-basic/0:'
 392                '/var/lib/abot-epc-basic/artifacts/TestResults.json',
 393                '{}/.'.format(self.res_dir)]
 394         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 395         self.__logger.info("%s\n%s", " ".join(cmd), output)
 396         self.__logger.info("Parsing the Test results...")
 397         res = (process_abot_test_result('{}/TestResults.json'.format(
 398             self.res_dir)))
 399         short_result = sig_test_format(res)
 400         self.__logger.info(short_result)
 401         self.details['test_vnf'].update(
 402             status='PASS', result=short_result, full_result=res,
 403             duration=duration)
 404         self.__logger.info(
 405             "Test VNF result: Passed: %d, Failed:%d, Skipped: %d",
 406             short_result['passed'],
 407             short_result['failures'], short_result['skipped'])
 408         return True
 409
 410     def clean(self):
 411         """Clean created objects/functions."""
 412         try:
 413             cmd = ['juju', 'debug-log', '--replay', '--no-tail']
 414             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 415             self.__logger.debug("%s\n%s", " ".join(cmd), output)
 416             if not self.orchestrator['requirements']['preserve_setup']:
 417                 self.__logger.info("Destroying Orchestrator...")
 418                 cmd = ['juju', 'destroy-controller', '-y', 'abot-controller',
 419                        '--destroy-all-models']
 420                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 421                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 422         except Exception:  # pylint: disable=broad-except
 423             self.__logger.exception("Some issue during the undeployment ..")
 424
 425         if not self.orchestrator['requirements']['preserve_setup']:
 426             self.__logger.info('Remove the Abot_epc OS object ..')
 427             super(JujuEpc, self).clean()
 428
 429         return True
 430
 431
 432 # ----------------------------------------------------------
 433 #
 434 #               YAML UTILS
 435 #
 436 # -----------------------------------------------------------
 437 def get_config(parameter, file_path):
 438     """
 439     Returns the value of a given parameter in file.yaml
 440     parameter must be given in string format with dots
 441     Example: general.openstack.image_name
 442     """
 443     with open(file_path) as config_file:
 444         file_yaml = yaml.safe_load(config_file)
 445     config_file.close()
 446     value = file_yaml
 447     for element in parameter.split("."):
 448         value = value.get(element)
 449         if value is None:
 450             raise ValueError("The parameter %s is not defined in"
 451                              " reporting.yaml" % parameter)
 452     return value
 453
 454
 455 def sig_test_format(sig_test):
 456     """
 457     Process the signaling result to have a short result
 458     """
 459     nb_passed = 0
 460     nb_failures = 0
 461     nb_skipped = 0
 462     for data_test in sig_test:
 463         if data_test['result'] == "passed":
 464             nb_passed += 1
 465         elif data_test['result'] == "failed":
 466             nb_failures += 1
 467         elif data_test['result'] == "skipped":
 468             nb_skipped += 1
 469     total_sig_test_result = {}
 470     total_sig_test_result['passed'] = nb_passed
 471     total_sig_test_result['failures'] = nb_failures
 472     total_sig_test_result['skipped'] = nb_skipped
 473     return total_sig_test_result
 474
 475
 476 def process_abot_test_result(file_path):
 477     """ Process ABoT Result """
 478     with open(file_path) as test_result:
 479         data = json.load(test_result)
 480         res = []
 481         for tests in data:
 482             tests = update_data(tests)
 483             try:
 484                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 485                 for steps in flatten_steps:
 486                     steps['result'] = steps['step_status']
 487                     res.append(steps)
 488             except:
 489                 logging.error("Could not post data to ElasticSearch host")
 490                 raise
 491         return res
 492
 493
 494 def update_data(obj):
 495     """ Update Result data"""
 496     try:
 497         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 498
 499         for element in obj['elements']:
 500             element['final_result'] = "passed"
 501             element['flatten_steps'] = []
 502
 503             for step in element['steps']:
 504                 temp_dict = {}
 505                 step['result'][step['result']['status']] = 1
 506                 if step['result']['status'].lower() in ['fail', 'failed']:
 507                     element['final_result'] = "failed"
 508
 509                 temp_dict['feature_file'] = obj['feature_file']
 510                 temp_dict['step_name'] = step['name']
 511                 temp_dict['step_status'] = step['result']['status']
 512                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 513                 temp_dict['step_' + step['result']['status']] = 1
 514                 element['flatten_steps'].append(deepcopy(temp_dict))
 515
 516             # Need to put the tag in OBJ and not ELEMENT
 517             if 'tags' in obj:
 518                 element['tags'] = deepcopy(obj['tags'])
 519                 for tag in obj['tags']:
 520                     element[tag['name']] = 1
 521             else:
 522                 for tag in element['tags']:
 523                     element[tag['name']] = 1
 524
 525     except Exception:  # pylint: disable=broad-except
 526         logging.error("Error in updating data, %s", sys.exc_info()[0])
 527         raise
 528
 529     return obj
 530
 531
 532 def get_instances(nova_client):
 533     """ To get all vm info of a project """
 534     try:
 535         instances = nova_client.servers.list()
 536         return instances
 537     except Exception as exc:  # pylint: disable=broad-except
 538         logging.error("Error [get_instances(nova_client)]: %s", exc)
 539         return None
 540
 541
 542 def get_instance_metadata(nova_client, instance):
 543     """ Get instance Metadata - Instance ID """
 544     try:
 545         instance = nova_client.servers.get(instance.id)
 546         return instance.metadata
 547     except Exception as exc:  # pylint: disable=broad-except
 548         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 549         return None