functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import re
  17 import subprocess
  18 import sys
  19 import uuid
  20 from copy import deepcopy
  21 from urlparse import urljoin
  22 import pkg_resources
  23 import yaml
  24
  25 from functest.core import vnf
  26 from functest.opnfv_tests.openstack.snaps import snaps_utils
  27 from functest.utils import config
  28 from functest.utils import env
  29
  30 from snaps.config.flavor import FlavorConfig
  31 from snaps.config.image import ImageConfig
  32 from snaps.config.network import NetworkConfig, SubnetConfig
  33 from snaps.config.router import RouterConfig
  34 from snaps.config.security_group import (
  35     Direction, Protocol, SecurityGroupConfig, SecurityGroupRuleConfig)
  36 from snaps.config.user import UserConfig
  37 from snaps.openstack.create_flavor import OpenStackFlavor
  38 from snaps.openstack.create_image import OpenStackImage
  39 from snaps.openstack.create_network import OpenStackNetwork
  40 from snaps.openstack.create_router import OpenStackRouter
  41 from snaps.openstack.create_security_group import OpenStackSecurityGroup
  42 from snaps.openstack.create_user import OpenStackUser
  43 from snaps.openstack.utils import keystone_utils
  44 from snaps.openstack.utils import nova_utils
  45
  46 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  47 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  48
  49 CLOUD_TEMPLATE = """clouds:
  50   abot-epc:
  51     type: openstack
  52     auth-types: [userpass]
  53     endpoint: {url}
  54     regions:
  55       {region}:
  56         endpoint: {url}"""
  57
  58 CREDS_TEMPLATE2 = """credentials:
  59   abot-epc:
  60     default-credential: abot-epc
  61     abot-epc:
  62       auth-type: userpass
  63       password: {pass}
  64       project-domain-name: {project_domain_n}
  65       tenant-name: {tenant_n}"""
  66
  67 CREDS_TEMPLATE3 = """credentials:
  68   abot-epc:
  69     default-credential: abot-epc
  70     abot-epc:
  71       auth-type: userpass
  72       password: {pass}
  73       project-domain-name: {project_domain_n}
  74       tenant-name: {tenant_n}
  75       user-domain-name: {user_domain_n}
  76       username: {user_n}"""
  77
  78
  79 class JujuEpc(vnf.VnfOnBoarding):
  80     # pylint:disable=too-many-instance-attributes
  81     """Abot EPC deployed with JUJU Orchestrator Case"""
  82
  83     __logger = logging.getLogger(__name__)
  84
  85     def __init__(self, **kwargs):
  86         if "case_name" not in kwargs:
  87             kwargs["case_name"] = "juju_epc"
  88         super(JujuEpc, self).__init__(**kwargs)
  89
  90         # Retrieve the configuration
  91         self.case_dir = pkg_resources.resource_filename(
  92             'functest', 'opnfv_tests/vnf/epc')
  93         try:
  94             self.config = getattr(
  95                 config.CONF, 'vnf_{}_config'.format(self.case_name))
  96         except Exception:
  97             raise Exception("VNF config file not found")
  98         self.config_file = os.path.join(self.case_dir, self.config)
  99         self.orchestrator = dict(requirements=get_config(
 100             "orchestrator.requirements", self.config_file))
 101
 102         self.created_object = []
 103         self.details['orchestrator'] = dict(
 104             name=get_config("orchestrator.name", self.config_file),
 105             version=get_config("orchestrator.version", self.config_file),
 106             status='ERROR',
 107             result=''
 108         )
 109
 110         self.vnf = dict(
 111             descriptor=get_config("vnf.descriptor", self.config_file),
 112             requirements=get_config("vnf.requirements", self.config_file)
 113         )
 114         self.details['vnf'] = dict(
 115             descriptor_version=self.vnf['descriptor']['version'],
 116             name=get_config("vnf.name", self.config_file),
 117             version=get_config("vnf.version", self.config_file),
 118         )
 119         self.__logger.debug("VNF configuration: %s", self.vnf)
 120
 121         self.details['test_vnf'] = dict(
 122             name=get_config("vnf_test_suite.name", self.config_file),
 123             version=get_config("vnf_test_suite.version", self.config_file),
 124             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 125         )
 126         self.public_auth_url = None
 127
 128         self.res_dir = os.path.join(
 129             getattr(config.CONF, 'dir_results'), self.case_name)
 130
 131     def _bypass_juju_network_discovery_bug(self, name):
 132         user_creator = OpenStackUser(
 133             self.snaps_creds,
 134             UserConfig(
 135                 name=name,
 136                 password=str(uuid.uuid4()),
 137                 project_name=self.tenant_name,
 138                 domain=self.snaps_creds.user_domain_name,
 139                 roles={'_member_': self.tenant_name}))
 140         user_creator.create()
 141         self.created_object.append(user_creator)
 142         return user_creator
 143
 144     def _register_cloud(self):
 145         self.__logger.info("Creating Cloud for Abot-epc .....")
 146         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 147         cloud_data = {
 148             'url': self.public_auth_url,
 149             'region': self.snaps_creds.region_name}
 150         with open(clouds_yaml, 'w') as yfile:
 151             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 152         cmd = ['juju', 'add-cloud', 'abot-epc', '-f', clouds_yaml, '--replace']
 153         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 154         self.__logger.info("%s\n%s", " ".join(cmd), output)
 155
 156     def _register_credentials_v2(self):
 157         self.__logger.info("Creating Credentials for Abot-epc .....")
 158         user_creator = self._bypass_juju_network_discovery_bug(
 159             'juju_network_discovery_bug')
 160         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 161         self.__logger.debug("snaps creds: %s", snaps_creds)
 162         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 163         creds_data = {
 164             'pass': snaps_creds.password,
 165             'tenant_n': snaps_creds.project_name,
 166             'user_n': snaps_creds.username}
 167         with open(credentials_yaml, 'w') as yfile:
 168             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 169         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 170                '--replace']
 171         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 172         self.__logger.info("%s\n%s", " ".join(cmd), output)
 173
 174     def _register_credentials_v3(self):
 175         self.__logger.info("Creating Credentials for Abot-epc .....")
 176         user_creator = self._bypass_juju_network_discovery_bug(
 177             'juju_network_discovery_bug')
 178         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 179         self.__logger.debug("snaps creds: %s", snaps_creds)
 180         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 181         creds_data = {
 182             'pass': snaps_creds.password,
 183             'tenant_n': snaps_creds.project_name,
 184             'user_n': snaps_creds.username,
 185             'project_domain_n': snaps_creds.project_domain_name,
 186             'user_domain_n': snaps_creds.user_domain_name}
 187         with open(credentials_yaml, 'w') as yfile:
 188             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 189         cmd = ['juju', 'add-credential', 'abot-epc', '-f', credentials_yaml,
 190                '--replace']
 191         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 192         self.__logger.info("%s\n%s", " ".join(cmd), output)
 193
 194     def _add_custom_rule(self, sec_grp_name):
 195         """ To add custom rule for SCTP Traffic """
 196         sec_grp_rules = list()
 197         sec_grp_rules.append(
 198             SecurityGroupRuleConfig(
 199                 sec_grp_name=sec_grp_name, direction=Direction.ingress,
 200                 protocol=Protocol.sctp))
 201         security_group = OpenStackSecurityGroup(
 202             self.snaps_creds,
 203             SecurityGroupConfig(
 204                 name=sec_grp_name,
 205                 rule_settings=sec_grp_rules))
 206         security_group.create()
 207         self.created_object.append(security_group)
 208
 209     def prepare(self):
 210         """Prepare testcase (Additional pre-configuration steps)."""
 211         self.__logger.info("Additional pre-configuration steps")
 212         super(JujuEpc, self).prepare()
 213         try:
 214             os.makedirs(self.res_dir)
 215         except OSError as ex:
 216             if ex.errno != errno.EEXIST:
 217                 self.__logger.exception("Cannot create %s", self.res_dir)
 218                 raise vnf.VnfPreparationException
 219         self.public_auth_url = keystone_utils.get_endpoint(
 220             self.snaps_creds, 'identity')
 221         # it enforces a versioned public identity endpoint as juju simply
 222         # adds /auth/tokens wich fails vs an unversioned endpoint.
 223         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 224             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 225         self._register_cloud()
 226         if self.snaps_creds.identity_api_version == 3:
 227             self._register_credentials_v3()
 228         else:
 229             self._register_credentials_v2()
 230
 231     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 232         """
 233         Create network, subnet, router
 234
 235         Bootstrap juju
 236         """
 237         self.__logger.info("Deployed Orchestrator")
 238         private_net_name = getattr(
 239             config.CONF, 'vnf_{}_private_net_name'.format(self.case_name))
 240         private_subnet_name = '{}-{}'.format(
 241             getattr(config.CONF,
 242                     'vnf_{}_private_subnet_name'.format(self.case_name)),
 243             self.uuid)
 244         private_subnet_cidr = getattr(
 245             config.CONF, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 246         abot_router = '{}-{}'.format(
 247             getattr(config.CONF,
 248                     'vnf_{}_external_router'.format(self.case_name)),
 249             self.uuid)
 250         self.__logger.info("Creating full network ...")
 251         subnet_settings = SubnetConfig(
 252             name=private_subnet_name,
 253             cidr=private_subnet_cidr,
 254             dns_nameservers=[env.get('NAMESERVER')])
 255         network_settings = NetworkConfig(
 256             name=private_net_name, subnet_settings=[subnet_settings])
 257         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 258         net_id = network_creator.create().id
 259         self.created_object.append(network_creator)
 260
 261         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 262         self.__logger.info("Creating network Router ....")
 263         router_creator = OpenStackRouter(
 264             self.snaps_creds, RouterConfig(
 265                 name=abot_router,
 266                 external_gateway=ext_net_name,
 267                 internal_subnets=[subnet_settings.name]))
 268         router_creator.create()
 269         self.created_object.append(router_creator)
 270         self.__logger.info("Creating Flavor ....")
 271         flavor_settings = FlavorConfig(
 272             name=self.orchestrator['requirements']['flavor']['name'],
 273             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 274             disk=10, vcpus=1)
 275         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 276         flavor_creator.create()
 277         self.created_object.append(flavor_creator)
 278         self.__logger.info("Upload some OS images if it doesn't exist")
 279         images = get_config("tenant_images", self.config_file)
 280         self.__logger.info("Images needed for vEPC: %s", images)
 281         for image_name, image_file in images.iteritems():
 282             self.__logger.info("image: %s, file: %s", image_name, image_file)
 283             if image_file and image_name:
 284                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 285                     name=image_name, image_user='cloud', img_format='qcow2',
 286                     image_file=image_file))
 287                 image_id = image_creator.create().id
 288                 cmd = ['juju', 'metadata', 'generate-image', '-d', '/root',
 289                        '-i', image_id, '-s', image_name,
 290                        '-r', self.snaps_creds.region_name,
 291                        '-u', self.public_auth_url]
 292                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 293                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 294                 self.created_object.append(image_creator)
 295         self.__logger.info("Network ID  : %s", net_id)
 296         cmd = ['juju', 'bootstrap', 'abot-epc', 'abot-controller',
 297                '--metadata-source', '/root',
 298                '--constraints', 'mem=2G',
 299                '--bootstrap-series', 'xenial',
 300                '--config', 'network={}'.format(net_id),
 301                '--config', 'ssl-hostname-verification=false',
 302                '--config', 'use-floating-ip=true',
 303                '--config', 'use-default-secgroup=true',
 304                '--debug']
 305         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 306         self.__logger.info("%s\n%s", " ".join(cmd), output)
 307         return True
 308
 309     def check_app(self, name='abot-epc-basic', status='active'):
 310         """Check application status."""
 311         cmd = ['juju', 'status', '--format', 'short', name]
 312         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 313         self.__logger.info("%s\n%s", " ".join(cmd), output)
 314         ret = re.search(r'(?=workload:({})\))'.format(status), output)
 315         if ret:
 316             self.__logger.info("%s workload is %s", name, status)
 317             return True
 318         self.__logger.error("%s workload differs from %s", name, status)
 319         return False
 320
 321     def deploy_vnf(self):
 322         """Deploy ABOT-OAI-EPC."""
 323         self.__logger.info("Upload VNFD")
 324         descriptor = self.vnf['descriptor']
 325         self.__logger.info("Get or create flavor for all Abot-EPC")
 326         flavor_settings = FlavorConfig(
 327             name=self.vnf['requirements']['flavor']['name'],
 328             ram=self.vnf['requirements']['flavor']['ram_min'],
 329             disk=10,
 330             vcpus=1)
 331         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 332         flavor_creator.create()
 333         self.created_object.append(flavor_creator)
 334         self.__logger.info("Deploying Abot-epc bundle file ...")
 335         cmd = ['juju', 'deploy', '{}'.format(descriptor.get('file_name'))]
 336         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 337         self.__logger.info("%s\n%s", " ".join(cmd), output)
 338         self.__logger.info("Waiting for instances .....")
 339         cmd = ['juju-wait']
 340         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 341         self.__logger.info("%s\n%s", " ".join(cmd), output)
 342         self.__logger.info("Deployed Abot-epc on Openstack")
 343         nova_client = nova_utils.nova_client(self.snaps_creds)
 344         instances = get_instances(nova_client)
 345         self.__logger.info("List of Instance: %s", instances)
 346         for items in instances:
 347             metadata = get_instance_metadata(nova_client, items)
 348             if 'juju-units-deployed' in metadata:
 349                 sec_group = 'juju-{}-{}'.format(
 350                     metadata['juju-controller-uuid'],
 351                     metadata['juju-model-uuid'])
 352                 self.__logger.info("Instance: %s", sec_group)
 353                 break
 354         self.__logger.info("Adding Security group rule....")
 355         # This will add sctp rule to a common Security Group Created
 356         # by juju and shared to all deployed units.
 357         self._add_custom_rule(sec_group)
 358         cmd = ['juju', 'status']
 359         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 360         self.__logger.debug("%s\n%s", " ".join(cmd), output)
 361         for app in ['abot-epc-basic', 'oai-epc', 'oai-hss']:
 362             if not self.check_app(app):
 363                 return False
 364         self.__logger.info("Copying the feature files to Abot_node ")
 365         cmd = ['juju', 'scp', '--', '-r', '-v',
 366                '{}/featureFiles'.format(self.case_dir), 'abot-epc-basic/0:~/']
 367         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 368         self.__logger.info("%s\n%s", " ".join(cmd), output)
 369         self.__logger.info("Copying the feature files in Abot_node ")
 370         cmd = ['juju', 'ssh', 'abot-epc-basic/0',
 371                'sudo', 'rsync', '-azvv', '~/featureFiles',
 372                '/etc/rebaca-test-suite/featureFiles']
 373         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 374         self.__logger.info("%s\n%s", " ".join(cmd), output)
 375         return True
 376
 377     def test_vnf(self):
 378         """Run test on ABoT."""
 379         start_time = time.time()
 380         self.__logger.info("Running VNF Test cases....")
 381         cmd = ['juju', 'run-action', 'abot-epc-basic/0', 'run',
 382                'tagnames={}'.format(self.details['test_vnf']['tag_name'])]
 383         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 384         self.__logger.info("%s\n%s", " ".join(cmd), output)
 385         cmd = ['juju-wait']
 386         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 387         self.__logger.info("%s\n%s", " ".join(cmd), output)
 388         duration = time.time() - start_time
 389         self.__logger.info("Getting results from Abot node....")
 390         cmd = ['juju', 'scp', '--', '-v',
 391                'abot-epc-basic/0:'
 392                '/var/lib/abot-epc-basic/artifacts/TestResults.json',
 393                '{}/.'.format(self.res_dir)]
 394         output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 395         self.__logger.info("%s\n%s", " ".join(cmd), output)
 396         self.__logger.info("Parsing the Test results...")
 397         res = (process_abot_test_result('{}/TestResults.json'.format(
 398             self.res_dir)))
 399         short_result = sig_test_format(res)
 400         self.__logger.info(short_result)
 401         self.details['test_vnf'].update(
 402             status='PASS', result=short_result, full_result=res,
 403             duration=duration)
 404         self.__logger.info(
 405             "Test VNF result: Passed: %d, Failed:%d, Skipped: %d",
 406             short_result['passed'],
 407             short_result['failures'], short_result['skipped'])
 408         return True
 409
 410     def clean(self):
 411         """Clean created objects/functions."""
 412         try:
 413             cmd = ['juju', 'debug-log', '--replay', '--no-tail']
 414             output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 415             self.__logger.debug("%s\n%s", " ".join(cmd), output)
 416             if not self.orchestrator['requirements']['preserve_setup']:
 417                 self.__logger.info("Destroying Orchestrator...")
 418                 cmd = ['juju', 'destroy-controller', '-y', 'abot-controller',
 419                        '--destroy-all-models']
 420                 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
 421                 self.__logger.info("%s\n%s", " ".join(cmd), output)
 422         except Exception:  # pylint: disable=broad-except
 423             self.__logger.exception("Some issue during the undeployment ..")
 424
 425         if not self.orchestrator['requirements']['preserve_setup']:
 426             self.__logger.info('Remove the Abot_epc OS object ..')
 427             super(JujuEpc, self).clean()
 428
 429         return True
 430
 431
 432 # ----------------------------------------------------------
 433 #
 434 #               YAML UTILS
 435 #
 436 # -----------------------------------------------------------
 437 def get_config(parameter, file_path):
 438     """
 439     Returns the value of a given parameter in file.yaml
 440     parameter must be given in string format with dots
 441     Example: general.openstack.image_name
 442     """
 443     with open(file_path) as config_file:
 444         file_yaml = yaml.safe_load(config_file)
 445     config_file.close()
 446     value = file_yaml
 447     for element in parameter.split("."):
 448         value = value.get(element)
 449         if value is None:
 450             raise ValueError("The parameter %s is not defined in"
 451                              " reporting.yaml" % parameter)
 452     return value
 453
 454
 455 def sig_test_format(sig_test):
 456     """
 457     Process the signaling result to have a short result
 458     """
 459     nb_passed = 0
 460     nb_failures = 0
 461     nb_skipped = 0
 462     for data_test in sig_test:
 463         if data_test['result'] == "passed":
 464             nb_passed += 1
 465         elif data_test['result'] == "failed":
 466             nb_failures += 1
 467         elif data_test['result'] == "skipped":
 468             nb_skipped += 1
 469     total_sig_test_result = {}
 470     total_sig_test_result['passed'] = nb_passed
 471     total_sig_test_result['failures'] = nb_failures
 472     total_sig_test_result['skipped'] = nb_skipped
 473     return total_sig_test_result
 474
 475
 476 def process_abot_test_result(file_path):
 477     """ Process ABoT Result """
 478     with open(file_path) as test_result:
 479         data = json.load(test_result)
 480         res = []
 481         for tests in data:
 482             tests = update_data(tests)
 483             try:
 484                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 485                 for steps in flatten_steps:
 486                     steps['result'] = steps['step_status']
 487                     res.append(steps)
 488             except:
 489                 logging.error("Could not post data to ElasticSearch host")
 490                 raise
 491         return res
 492
 493
 494 def update_data(obj):
 495     """ Update Result data"""
 496     try:
 497         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 498
 499         for element in obj['elements']:
 500             element['final_result'] = "passed"
 501             element['flatten_steps'] = []
 502
 503             for step in element['steps']:
 504                 temp_dict = {}
 505                 step['result'][step['result']['status']] = 1
 506                 if step['result']['status'].lower() in ['fail', 'failed']:
 507                     element['final_result'] = "failed"
 508
 509                 temp_dict['feature_file'] = obj['feature_file']
 510                 temp_dict['step_name'] = step['name']
 511                 temp_dict['step_status'] = step['result']['status']
 512                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 513                 temp_dict['step_' + step['result']['status']] = 1
 514                 element['flatten_steps'].append(deepcopy(temp_dict))
 515
 516             # Need to put the tag in OBJ and not ELEMENT
 517             if 'tags' in obj:
 518                 element['tags'] = deepcopy(obj['tags'])
 519                 for tag in obj['tags']:
 520                     element[tag['name']] = 1
 521             else:
 522                 for tag in element['tags']:
 523                     element[tag['name']] = 1
 524
 525     except Exception:  # pylint: disable=broad-except
 526         logging.error("Error in updating data, %s", sys.exc_info()[0])
 527         raise
 528
 529     return obj
 530
 531
 532 def get_instances(nova_client):
 533     """ To get all vm info of a project """
 534     try:
 535         instances = nova_client.servers.list()
 536         return instances
 537     except Exception as exc:  # pylint: disable=broad-except
 538         logging.error("Error [get_instances(nova_client)]: %s", exc)
 539         return None
 540
 541
 542 def get_instance_metadata(nova_client, instance):
 543     """ Get instance Metadata - Instance ID """
 544     try:
 545         instance = nova_client.servers.get(instance.id)
 546         return instance.metadata
 547     except Exception as exc:  # pylint: disable=broad-except
 548         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 549         return None