functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20
  21 from functest.core import vnf
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 from functest.utils.constants import CONST
  24 import functest.utils.openstack_utils as os_utils
  25
  26 import pkg_resources
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.user import UserConfig
  32 from snaps.openstack.create_flavor import OpenStackFlavor
  33 from snaps.openstack.create_image import OpenStackImage
  34 from snaps.openstack.create_network import OpenStackNetwork
  35 from snaps.openstack.create_router import OpenStackRouter
  36 from snaps.openstack.create_user import OpenStackUser
  37 from snaps.openstack.utils import keystone_utils
  38 from snaps.openstack.utils import neutron_utils
  39 from snaps.openstack.utils import nova_utils
  40 import yaml
  41
  42 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  43 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  44
  45 CLOUD_TEMPLATE = """clouds:
  46   abot-epc:
  47     type: openstack
  48     auth-types: [userpass]
  49     endpoint: {url}
  50     regions:
  51       {region}:
  52         endpoint: {url}"""
  53
  54 CREDS_TEMPLATE2 = """credentials:
  55   abot-epc:
  56     default-credential: abot-epc
  57     abot-epc:
  58       auth-type: userpass
  59       password: {pass}
  60       project-domain-name: {project_domain_n}
  61       tenant-name: {tenant_n}"""
  62
  63 CREDS_TEMPLATE3 = """credentials:
  64   abot-epc:
  65     default-credential: abot-epc
  66     abot-epc:
  67       auth-type: userpass
  68       password: {pass}
  69       project-domain-name: {project_domain_n}
  70       tenant-name: {tenant_n}
  71       user-domain-name: {user_domain_n}
  72       username: {user_n}"""
  73
  74
  75 class JujuEpc(vnf.VnfOnBoarding):
  76     # pylint:disable=too-many-instance-attributes
  77     """Abot EPC deployed with JUJU Orchestrator Case"""
  78
  79     __logger = logging.getLogger(__name__)
  80
  81     default_region_name = "RegionOne"
  82
  83     def __init__(self, **kwargs):
  84         if "case_name" not in kwargs:
  85             kwargs["case_name"] = "juju_epc"
  86         super(JujuEpc, self).__init__(**kwargs)
  87
  88         # Retrieve the configuration
  89         self.case_dir = pkg_resources.resource_filename(
  90             'functest', 'opnfv_tests/vnf/epc')
  91         try:
  92             self.config = getattr(
  93                 CONST, 'vnf_{}_config'.format(self.case_name))
  94         except Exception:
  95             raise Exception("VNF config file not found")
  96         self.config_file = os.path.join(self.case_dir, self.config)
  97         self.orchestrator = dict(requirements=get_config(
  98             "orchestrator.requirements", self.config_file))
  99
 100         self.created_object = []
 101         self.details['orchestrator'] = dict(
 102             name=get_config("orchestrator.name", self.config_file),
 103             version=get_config("orchestrator.version", self.config_file),
 104             status='ERROR',
 105             result=''
 106         )
 107
 108         self.vnf = dict(
 109             descriptor=get_config("vnf.descriptor", self.config_file),
 110             requirements=get_config("vnf.requirements", self.config_file)
 111         )
 112         self.details['vnf'] = dict(
 113             descriptor_version=self.vnf['descriptor']['version'],
 114             name=get_config("vnf.name", self.config_file),
 115             version=get_config("vnf.version", self.config_file),
 116         )
 117         self.__logger.debug("VNF configuration: %s", self.vnf)
 118
 119         self.details['test_vnf'] = dict(
 120             name=get_config("vnf_test_suite.name", self.config_file),
 121             version=get_config("vnf_test_suite.version", self.config_file),
 122             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 123         )
 124         self.public_auth_url = None
 125
 126         self.res_dir = os.path.join(
 127             getattr(CONST, 'dir_results'), self.case_name)
 128
 129     def _bypass_juju_network_discovery_bug(self, name):
 130         user_creator = OpenStackUser(
 131             self.snaps_creds,
 132             UserConfig(
 133                 name=name, password=str(uuid.uuid4()),
 134                 roles={'_member_': self.tenant_name}))
 135         user_creator.create()
 136         self.created_object.append(user_creator)
 137         return user_creator
 138
 139     def _register_cloud(self):
 140         self.__logger.info("Creating Cloud for Abot-epc .....")
 141         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 142         cloud_data = {
 143             'url': self.public_auth_url,
 144             'region': os.environ.get(
 145                 "OS_REGION_NAME", self.default_region_name)}
 146         with open(clouds_yaml, 'w') as yfile:
 147             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 148         if os.system(
 149                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 150             raise vnf.VnfPreparationException
 151
 152     def _register_credentials_v2(self):
 153         self.__logger.info("Creating Credentials for Abot-epc .....")
 154         user_creator = self._bypass_juju_network_discovery_bug(
 155             'juju_network_discovery_bug')
 156         snaps_creds = user_creator.get_os_creds('juju_network_discovery_bug')
 157         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 158         # 'tenant_n' should habe been equal to snaps_creds.project_name
 159         # user_creator.get_os_creds() must be checked
 160         creds_data = {
 161             'pass': snaps_creds.password,
 162             'tenant_n': self.snaps_creds.project_name,
 163             'user_n': snaps_creds.username}
 164         with open(credentials_yaml, 'w') as yfile:
 165             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 166         if os.system(
 167                 'juju add-credential abot-epc -f {} --replace'.format(
 168                     credentials_yaml)):
 169             raise vnf.VnfPreparationException
 170
 171     def _register_credentials_v3(self):
 172         self.__logger.info("Creating Credentials for Abot-epc .....")
 173         user_creator = self._bypass_juju_network_discovery_bug(
 174             'juju_network_discovery_bug')
 175         snaps_creds = user_creator.get_os_creds('juju_network_discovery_bug')
 176         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 177         # 'tenant_n' should habe been equal to snaps_creds.project_name
 178         # user_creator.get_os_creds() must be checked
 179         creds_data = {
 180             'pass': snaps_creds.password,
 181             'tenant_n': self.snaps_creds.project_name,
 182             'user_n': snaps_creds.username,
 183             'project_domain_n': snaps_creds.project_domain_name,
 184             'user_domain_n': snaps_creds.user_domain_name}
 185         with open(credentials_yaml, 'w') as yfile:
 186             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 187         if os.system(
 188                 'juju add-credential abot-epc -f {} --replace'.format(
 189                     credentials_yaml)):
 190             raise vnf.VnfPreparationException
 191
 192     def prepare(self):
 193         """Prepare testcase (Additional pre-configuration steps)."""
 194         self.__logger.info("Additional pre-configuration steps")
 195         super(JujuEpc, self).prepare()
 196         try:
 197             os.makedirs(self.res_dir)
 198         except OSError as ex:
 199             if ex.errno != errno.EEXIST:
 200                 self.__logger.exception("Cannot create %s", self.res_dir)
 201                 raise vnf.VnfPreparationException
 202         self.public_auth_url = keystone_utils.get_endpoint(
 203             self.snaps_creds, 'identity')
 204         # it enforces a versioned public identity endpoint as juju simply
 205         # adds /auth/tokens wich fails vs an unversioned endpoint.
 206         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 207             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 208         self._register_cloud()
 209         if self.snaps_creds.identity_api_version == 3:
 210             self._register_credentials_v3()
 211         else:
 212             self._register_credentials_v2()
 213
 214     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 215         """
 216         Create network, subnet, router
 217
 218         Bootstrap juju
 219         """
 220         self.__logger.info("Deployed Orchestrator")
 221         private_net_name = getattr(
 222             CONST, 'vnf_{}_private_net_name'.format(self.case_name))
 223         private_subnet_name = getattr(
 224             CONST, 'vnf_{}_private_subnet_name'.format(self.case_name))
 225         private_subnet_cidr = getattr(
 226             CONST, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 227         abot_router = getattr(
 228             CONST, 'vnf_{}_external_router'.format(self.case_name))
 229
 230         self.__logger.info("Creating full network ...")
 231         subnet_settings = SubnetConfig(
 232             name=private_subnet_name, cidr=private_subnet_cidr)
 233         network_settings = NetworkConfig(
 234             name=private_net_name, subnet_settings=[subnet_settings])
 235         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 236         net_id = network_creator.create().id
 237         self.created_object.append(network_creator)
 238
 239         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 240         self.__logger.info("Creating network Router ....")
 241         router_creator = OpenStackRouter(
 242             self.snaps_creds, RouterConfig(
 243                 name=abot_router,
 244                 external_gateway=ext_net_name,
 245                 internal_subnets=[subnet_settings.name]))
 246         router_creator.create()
 247         self.created_object.append(router_creator)
 248         self.__logger.info("Creating Flavor ....")
 249         flavor_settings = FlavorConfig(
 250             name=self.orchestrator['requirements']['flavor']['name'],
 251             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 252             disk=10, vcpus=1)
 253         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 254         flavor_creator.create()
 255         self.created_object.append(flavor_creator)
 256         self.__logger.info("Upload some OS images if it doesn't exist")
 257         images = get_config("tenant_images", self.config_file)
 258         self.__logger.info("Images needed for vEPC: %s", images)
 259         for image_name, image_file in images.iteritems():
 260             self.__logger.info("image: %s, file: %s", image_name, image_file)
 261             if image_file and image_name:
 262                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 263                     name=image_name, image_user='cloud', img_format='qcow2',
 264                     image_file=image_file))
 265                 image_id = image_creator.create().id
 266                 os.system(
 267                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 268                     '{} -u {}'.format(
 269                         image_id, image_name,
 270                         os.environ.get(
 271                             "OS_REGION_NAME", self.default_region_name),
 272                         self.public_auth_url))
 273                 self.created_object.append(image_creator)
 274         self.__logger.info("Credential information  : %s", net_id)
 275         juju_bootstrap_command = (
 276             'juju bootstrap abot-epc abot-controller --config network={} '
 277             '--metadata-source ~  --config ssl-hostname-verification=false '
 278             '--constraints mem=2G --bootstrap-series xenial '
 279             '--config use-floating-ip=true --debug '
 280             '--config use-default-secgroup=true'.format(net_id))
 281         os.system(juju_bootstrap_command)
 282         return True
 283
 284     def deploy_vnf(self):
 285         """Deploy ABOT-OAI-EPC."""
 286         self.__logger.info("Upload VNFD")
 287         descriptor = self.vnf['descriptor']
 288         self.__logger.info("Get or create flavor for all Abot-EPC")
 289         flavor_settings = FlavorConfig(
 290             name=self.vnf['requirements']['flavor']['name'],
 291             ram=self.vnf['requirements']['flavor']['ram_min'],
 292             disk=10,
 293             vcpus=1)
 294         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 295         flavor_creator.create()
 296         self.created_object.append(flavor_creator)
 297         self.__logger.info("Deploying Abot-epc bundle file ...")
 298         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 299         self.__logger.info("Waiting for instances .....")
 300         status = os.system('juju-wait')
 301         self.__logger.info("juju wait completed: %s", status)
 302         self.__logger.info("Deployed Abot-epc on Openstack")
 303         nova_client = nova_utils.nova_client(self.snaps_creds)
 304         neutron_client = neutron_utils.neutron_client(self.snaps_creds)
 305         if status == 0:
 306             instances = os_utils.get_instances(nova_client)
 307             for items in instances:
 308                 metadata = get_instance_metadata(nova_client, items)
 309                 if 'juju-units-deployed' in metadata:
 310                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 311                                  '-' + metadata['juju-model-uuid'])
 312                     self.sec_group_id = os_utils.get_security_group_id(
 313                         neutron_client, sec_group)
 314                     break
 315             self.__logger.info("Adding Security group rule....")
 316             os_utils.create_secgroup_rule(
 317                 neutron_client, self.sec_group_id, 'ingress', 132)
 318             self.__logger.info("Copying the feature files to Abot_node ")
 319             os.system('juju scp -- -r {}/featureFiles abot-'
 320                       'epc-basic/0:~/'.format(self.case_dir))
 321             self.__logger.info("Copying the feature files in Abot_node ")
 322             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 323                       "~/featureFiles /etc/rebaca-test-suite"
 324                       "/featureFiles'")
 325             count = 0
 326             while count < 10:
 327                 epcstatus = os.system('juju status oai-epc | '
 328                                       'grep {} | grep {} | grep {}'
 329                                       .format('EPC', 'is', 'running'))
 330                 if epcstatus == 0:
 331                     break
 332                 else:
 333                     time.sleep(60)
 334                     count = count + 1
 335             os.system('juju-wait')
 336             return True
 337         return False
 338
 339     def test_vnf(self):
 340         """Run test on ABoT."""
 341         start_time = time.time()
 342         self.__logger.info("Running VNF Test cases....")
 343         os.system('juju run-action abot-epc-basic/0 run '
 344                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 345         os.system('juju-wait')
 346         duration = time.time() - start_time
 347         self.__logger.info("Getting results from Abot node....")
 348         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 349                   'epc-basic/artifacts/TestResults.json {}/.'
 350                   .format(self.case_dir))
 351         self.__logger.info("Parsing the Test results...")
 352         res = (process_abot_test_result('{}/TestResults.'
 353                                         'json'.format(self.case_dir)))
 354         short_result = sig_test_format(res)
 355         self.__logger.info(short_result)
 356         self.details['test_vnf'].update(status='PASS',
 357                                         result=short_result,
 358                                         full_result=res,
 359                                         duration=duration)
 360
 361         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 362                            "%d, Skipped: %d", short_result['passed'],
 363                            short_result['failures'], short_result['skipped'])
 364         return True
 365
 366     def clean(self):
 367         """Clean created objects/functions."""
 368         try:
 369             if not self.orchestrator['requirements']['preserve_setup']:
 370                 self.__logger.info("Removing deployment files...")
 371                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 372                 if os.path.exists(testresult):
 373                     os.remove(testresult)
 374                 self.__logger.info("Destroying Orchestrator...")
 375                 os.system('juju destroy-controller -y abot-controller '
 376                           '--destroy-all-models')
 377         except Exception:  # pylint: disable=broad-except
 378             self.__logger.warn("Some issue during the undeployment ..")
 379             self.__logger.warn("Tenant clean continue ..")
 380
 381         if not self.orchestrator['requirements']['preserve_setup']:
 382             self.__logger.info('Remove the Abot_epc OS object ..')
 383             super(JujuEpc, self).clean()
 384
 385         return True
 386
 387
 388 # ----------------------------------------------------------
 389 #
 390 #               YAML UTILS
 391 #
 392 # -----------------------------------------------------------
 393 def get_config(parameter, file_path):
 394     """
 395     Returns the value of a given parameter in file.yaml
 396     parameter must be given in string format with dots
 397     Example: general.openstack.image_name
 398     """
 399     with open(file_path) as config_file:
 400         file_yaml = yaml.safe_load(config_file)
 401     config_file.close()
 402     value = file_yaml
 403     for element in parameter.split("."):
 404         value = value.get(element)
 405         if value is None:
 406             raise ValueError("The parameter %s is not defined in"
 407                              " reporting.yaml" % parameter)
 408     return value
 409
 410
 411 def sig_test_format(sig_test):
 412     """
 413     Process the signaling result to have a short result
 414     """
 415     nb_passed = 0
 416     nb_failures = 0
 417     nb_skipped = 0
 418     for data_test in sig_test:
 419         if data_test['result'] == "passed":
 420             nb_passed += 1
 421         elif data_test['result'] == "failed":
 422             nb_failures += 1
 423         elif data_test['result'] == "skipped":
 424             nb_skipped += 1
 425     total_sig_test_result = {}
 426     total_sig_test_result['passed'] = nb_passed
 427     total_sig_test_result['failures'] = nb_failures
 428     total_sig_test_result['skipped'] = nb_skipped
 429     return total_sig_test_result
 430
 431
 432 def process_abot_test_result(file_path):
 433     """ Process ABoT Result """
 434     with open(file_path) as test_result:
 435         data = json.load(test_result)
 436         res = []
 437         for tests in data:
 438             tests = update_data(tests)
 439             try:
 440                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 441                 for steps in flatten_steps:
 442                     steps['result'] = steps['step_status']
 443                     res.append(steps)
 444             except:
 445                 logging.error("Could not post data to ElasticSearch host")
 446                 raise
 447         return res
 448
 449
 450 def update_data(obj):
 451     """ Update Result data"""
 452     try:
 453         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 454
 455         for element in obj['elements']:
 456             element['final_result'] = "passed"
 457             element['flatten_steps'] = []
 458
 459             for step in element['steps']:
 460                 temp_dict = {}
 461                 step['result'][step['result']['status']] = 1
 462                 if step['result']['status'].lower() in ['fail', 'failed']:
 463                     element['final_result'] = "failed"
 464
 465                 temp_dict['feature_file'] = obj['feature_file']
 466                 temp_dict['step_name'] = step['name']
 467                 temp_dict['step_status'] = step['result']['status']
 468                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 469                 temp_dict['step_' + step['result']['status']] = 1
 470                 element['flatten_steps'].append(deepcopy(temp_dict))
 471
 472             # Need to put the tag in OBJ and not ELEMENT
 473             if 'tags' in obj:
 474                 element['tags'] = deepcopy(obj['tags'])
 475                 for tag in obj['tags']:
 476                     element[tag['name']] = 1
 477             else:
 478                 for tag in element['tags']:
 479                     element[tag['name']] = 1
 480
 481     except Exception:  # pylint: disable=broad-except
 482         logging.error("Error in updating data, %s", sys.exc_info()[0])
 483         raise
 484
 485     return obj
 486
 487
 488 def get_instance_metadata(nova_client, instance):
 489     """ Get instance Metadata - Instance ID """
 490     try:
 491         instance = nova_client.servers.get(instance.id)
 492         return instance.metadata
 493     except Exception as exc:  # pylint: disable=broad-except
 494         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 495         return None