functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20
  21 from functest.core import vnf
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 from functest.utils.constants import CONST
  24 import functest.utils.openstack_utils as os_utils
  25
  26 import pkg_resources
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.user import UserConfig
  32 from snaps.openstack.create_flavor import OpenStackFlavor
  33 from snaps.openstack.create_image import OpenStackImage
  34 from snaps.openstack.create_network import OpenStackNetwork
  35 from snaps.openstack.create_router import OpenStackRouter
  36 from snaps.openstack.create_user import OpenStackUser
  37 from snaps.openstack.utils import keystone_utils
  38 from snaps.openstack.utils import neutron_utils
  39 from snaps.openstack.utils import nova_utils
  40 import yaml
  41
  42 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  43 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  44
  45 CLOUD_TEMPLATE = """clouds:
  46   abot-epc:
  47     type: openstack
  48     auth-types: [userpass]
  49     endpoint: {url}
  50     regions:
  51       {region}:
  52         endpoint: {url}"""
  53
  54 CREDS_TEMPLATE2 = """credentials:
  55   abot-epc:
  56     default-credential: abot-epc
  57     abot-epc:
  58       auth-type: userpass
  59       password: {pass}
  60       project-domain-name: {project_domain_n}
  61       tenant-name: {tenant_n}"""
  62
  63 CREDS_TEMPLATE3 = """credentials:
  64   abot-epc:
  65     default-credential: abot-epc
  66     abot-epc:
  67       auth-type: userpass
  68       password: {pass}
  69       project-domain-name: {project_domain_n}
  70       tenant-name: {tenant_n}
  71       user-domain-name: {user_domain_n}
  72       username: {user_n}"""
  73
  74
  75 class JujuEpc(vnf.VnfOnBoarding):
  76     # pylint:disable=too-many-instance-attributes
  77     """Abot EPC deployed with JUJU Orchestrator Case"""
  78
  79     __logger = logging.getLogger(__name__)
  80
  81     default_region_name = "RegionOne"
  82
  83     def __init__(self, **kwargs):
  84         if "case_name" not in kwargs:
  85             kwargs["case_name"] = "juju_epc"
  86         super(JujuEpc, self).__init__(**kwargs)
  87
  88         # Retrieve the configuration
  89         self.case_dir = pkg_resources.resource_filename(
  90             'functest', 'opnfv_tests/vnf/epc')
  91         try:
  92             self.config = getattr(
  93                 CONST, 'vnf_{}_config'.format(self.case_name))
  94         except Exception:
  95             raise Exception("VNF config file not found")
  96         self.config_file = os.path.join(self.case_dir, self.config)
  97         self.orchestrator = dict(requirements=get_config(
  98             "orchestrator.requirements", self.config_file))
  99
 100         self.created_object = []
 101         self.details['orchestrator'] = dict(
 102             name=get_config("orchestrator.name", self.config_file),
 103             version=get_config("orchestrator.version", self.config_file),
 104             status='ERROR',
 105             result=''
 106         )
 107
 108         self.vnf = dict(
 109             descriptor=get_config("vnf.descriptor", self.config_file),
 110             requirements=get_config("vnf.requirements", self.config_file)
 111         )
 112         self.details['vnf'] = dict(
 113             descriptor_version=self.vnf['descriptor']['version'],
 114             name=get_config("vnf.name", self.config_file),
 115             version=get_config("vnf.version", self.config_file),
 116         )
 117         self.__logger.debug("VNF configuration: %s", self.vnf)
 118
 119         self.details['test_vnf'] = dict(
 120             name=get_config("vnf_test_suite.name", self.config_file),
 121             version=get_config("vnf_test_suite.version", self.config_file),
 122             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 123         )
 124         self.public_auth_url = None
 125
 126         self.res_dir = os.path.join(
 127             getattr(CONST, 'dir_results'), self.case_name)
 128
 129     def _bypass_juju_network_discovery_bug(self, name):
 130         user_creator = OpenStackUser(
 131             self.snaps_creds,
 132             UserConfig(
 133                 name=name, password=str(uuid.uuid4()),
 134                 roles={'_member_': self.tenant_name}))
 135         user_creator.create()
 136         self.created_object.append(user_creator)
 137         return user_creator
 138
 139     def _register_cloud(self):
 140         self.__logger.info("Creating Cloud for Abot-epc .....")
 141         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 142         cloud_data = {
 143             'url': self.public_auth_url,
 144             'region': os.environ.get(
 145                 "OS_REGION_NAME", self.default_region_name)}
 146         with open(clouds_yaml, 'w') as yfile:
 147             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 148         if os.system(
 149                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 150             raise vnf.VnfPreparationException
 151
 152     def _register_credentials_v2(self):
 153         self.__logger.info("Creating Credentials for Abot-epc .....")
 154         user_creator = self._bypass_juju_network_discovery_bug(
 155             'juju_network_discovery_bug')
 156         snaps_creds = user_creator.get_os_creds('juju_network_discovery_bug')
 157         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 158         # 'tenant_n' should habe been equal to snaps_creds.project_name
 159         # user_creator.get_os_creds() must be checked
 160         creds_data = {
 161             'pass': snaps_creds.password,
 162             'tenant_n': self.snaps_creds.project_name,
 163             'user_n': snaps_creds.username}
 164         with open(credentials_yaml, 'w') as yfile:
 165             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 166         if os.system(
 167                 'juju add-credential abot-epc -f {} --replace'.format(
 168                     credentials_yaml)):
 169             raise vnf.VnfPreparationException
 170
 171     def _register_credentials_v3(self):
 172         self.__logger.info("Creating Credentials for Abot-epc .....")
 173         user_creator = self._bypass_juju_network_discovery_bug(
 174             'juju_network_discovery_bug')
 175         snaps_creds = user_creator.get_os_creds('juju_network_discovery_bug')
 176         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 177         # 'tenant_n' should habe been equal to snaps_creds.project_name
 178         # user_creator.get_os_creds() must be checked
 179         creds_data = {
 180             'pass': snaps_creds.password,
 181             'tenant_n': self.snaps_creds.project_name,
 182             'user_n': snaps_creds.username,
 183             'project_domain_n': snaps_creds.project_domain_name,
 184             'user_domain_n': snaps_creds.user_domain_name}
 185         with open(credentials_yaml, 'w') as yfile:
 186             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 187         if os.system(
 188                 'juju add-credential abot-epc -f {} --replace'.format(
 189                     credentials_yaml)):
 190             raise vnf.VnfPreparationException
 191
 192     def prepare(self):
 193         """Prepare testcase (Additional pre-configuration steps)."""
 194         self.__logger.info("Additional pre-configuration steps")
 195         super(JujuEpc, self).prepare()
 196         try:
 197             os.makedirs(self.res_dir)
 198         except OSError as ex:
 199             if ex.errno != errno.EEXIST:
 200                 self.__logger.exception("Cannot create %s", self.res_dir)
 201                 raise vnf.VnfPreparationException
 202         self.public_auth_url = keystone_utils.get_endpoint(
 203             self.snaps_creds, 'identity')
 204         # it enforces a versioned public identity endpoint as juju simply
 205         # adds /auth/tokens wich fails vs an unversioned endpoint.
 206         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 207             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 208         self._register_cloud()
 209         if self.snaps_creds.identity_api_version == 3:
 210             self._register_credentials_v3()
 211         else:
 212             self._register_credentials_v2()
 213
 214     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 215         """
 216         Create network, subnet, router
 217
 218         Bootstrap juju
 219         """
 220         self.__logger.info("Deployed Orchestrator")
 221         private_net_name = getattr(
 222             CONST, 'vnf_{}_private_net_name'.format(self.case_name))
 223         private_subnet_name = getattr(
 224             CONST, 'vnf_{}_private_subnet_name'.format(self.case_name))
 225         private_subnet_cidr = getattr(
 226             CONST, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 227         abot_router = getattr(
 228             CONST, 'vnf_{}_external_router'.format(self.case_name))
 229         dns_nameserver = getattr(
 230             CONST, 'vnf_{}_dns_nameserver'.format(self.case_name))
 231
 232         self.__logger.info("Creating full network ...")
 233         subnet_settings = SubnetConfig(
 234             name=private_subnet_name, cidr=private_subnet_cidr,
 235             dns_nameservers=dns_nameserver)
 236         network_settings = NetworkConfig(
 237             name=private_net_name, subnet_settings=[subnet_settings])
 238         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 239         net_id = network_creator.create().id
 240         self.created_object.append(network_creator)
 241
 242         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 243         self.__logger.info("Creating network Router ....")
 244         router_creator = OpenStackRouter(
 245             self.snaps_creds, RouterConfig(
 246                 name=abot_router,
 247                 external_gateway=ext_net_name,
 248                 internal_subnets=[subnet_settings.name]))
 249         router_creator.create()
 250         self.created_object.append(router_creator)
 251         self.__logger.info("Creating Flavor ....")
 252         flavor_settings = FlavorConfig(
 253             name=self.orchestrator['requirements']['flavor']['name'],
 254             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 255             disk=10, vcpus=1)
 256         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 257         flavor_creator.create()
 258         self.created_object.append(flavor_creator)
 259         self.__logger.info("Upload some OS images if it doesn't exist")
 260         images = get_config("tenant_images", self.config_file)
 261         self.__logger.info("Images needed for vEPC: %s", images)
 262         for image_name, image_file in images.iteritems():
 263             self.__logger.info("image: %s, file: %s", image_name, image_file)
 264             if image_file and image_name:
 265                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 266                     name=image_name, image_user='cloud', img_format='qcow2',
 267                     image_file=image_file))
 268                 image_id = image_creator.create().id
 269                 os.system(
 270                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 271                     '{} -u {}'.format(
 272                         image_id, image_name,
 273                         os.environ.get(
 274                             "OS_REGION_NAME", self.default_region_name),
 275                         self.public_auth_url))
 276                 self.created_object.append(image_creator)
 277         self.__logger.info("Credential information  : %s", net_id)
 278         juju_bootstrap_command = (
 279             'juju bootstrap abot-epc abot-controller --config network={} '
 280             '--metadata-source ~  --config ssl-hostname-verification=false '
 281             '--constraints mem=2G --bootstrap-series xenial '
 282             '--config use-floating-ip=true --debug '
 283             '--config use-default-secgroup=true'.format(net_id))
 284         os.system(juju_bootstrap_command)
 285         return True
 286
 287     def deploy_vnf(self):
 288         """Deploy ABOT-OAI-EPC."""
 289         self.__logger.info("Upload VNFD")
 290         descriptor = self.vnf['descriptor']
 291         self.__logger.info("Get or create flavor for all Abot-EPC")
 292         flavor_settings = FlavorConfig(
 293             name=self.vnf['requirements']['flavor']['name'],
 294             ram=self.vnf['requirements']['flavor']['ram_min'],
 295             disk=10,
 296             vcpus=1)
 297         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 298         flavor_creator.create()
 299         self.created_object.append(flavor_creator)
 300         self.__logger.info("Deploying Abot-epc bundle file ...")
 301         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 302         self.__logger.info("Waiting for instances .....")
 303         status = os.system('juju-wait')
 304         self.__logger.info("juju wait completed: %s", status)
 305         self.__logger.info("Deployed Abot-epc on Openstack")
 306         nova_client = nova_utils.nova_client(self.snaps_creds)
 307         neutron_client = neutron_utils.neutron_client(self.snaps_creds)
 308         if status == 0:
 309             instances = os_utils.get_instances(nova_client)
 310             for items in instances:
 311                 metadata = get_instance_metadata(nova_client, items)
 312                 if 'juju-units-deployed' in metadata:
 313                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 314                                  '-' + metadata['juju-model-uuid'])
 315                     self.sec_group_id = os_utils.get_security_group_id(
 316                         neutron_client, sec_group)
 317                     break
 318             self.__logger.info("Adding Security group rule....")
 319             os_utils.create_secgroup_rule(
 320                 neutron_client, self.sec_group_id, 'ingress', 132)
 321             self.__logger.info("Copying the feature files to Abot_node ")
 322             os.system('juju scp -- -r {}/featureFiles abot-'
 323                       'epc-basic/0:~/'.format(self.case_dir))
 324             self.__logger.info("Copying the feature files in Abot_node ")
 325             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 326                       "~/featureFiles /etc/rebaca-test-suite"
 327                       "/featureFiles'")
 328             count = 0
 329             while count < 10:
 330                 epcstatus = os.system('juju status oai-epc | '
 331                                       'grep {} | grep {} | grep {}'
 332                                       .format('EPC', 'is', 'running'))
 333                 if epcstatus == 0:
 334                     break
 335                 else:
 336                     time.sleep(60)
 337                     count = count + 1
 338             os.system('juju-wait')
 339             return True
 340         return False
 341
 342     def test_vnf(self):
 343         """Run test on ABoT."""
 344         start_time = time.time()
 345         self.__logger.info("Running VNF Test cases....")
 346         os.system('juju run-action abot-epc-basic/0 run '
 347                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 348         os.system('juju-wait')
 349         duration = time.time() - start_time
 350         self.__logger.info("Getting results from Abot node....")
 351         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 352                   'epc-basic/artifacts/TestResults.json {}/.'
 353                   .format(self.case_dir))
 354         self.__logger.info("Parsing the Test results...")
 355         res = (process_abot_test_result('{}/TestResults.'
 356                                         'json'.format(self.case_dir)))
 357         short_result = sig_test_format(res)
 358         self.__logger.info(short_result)
 359         self.details['test_vnf'].update(status='PASS',
 360                                         result=short_result,
 361                                         full_result=res,
 362                                         duration=duration)
 363
 364         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 365                            "%d, Skipped: %d", short_result['passed'],
 366                            short_result['failures'], short_result['skipped'])
 367         return True
 368
 369     def clean(self):
 370         """Clean created objects/functions."""
 371         try:
 372             if not self.orchestrator['requirements']['preserve_setup']:
 373                 self.__logger.info("Removing deployment files...")
 374                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 375                 if os.path.exists(testresult):
 376                     os.remove(testresult)
 377                 self.__logger.info("Destroying Orchestrator...")
 378                 os.system('juju destroy-controller -y abot-controller '
 379                           '--destroy-all-models')
 380         except Exception:  # pylint: disable=broad-except
 381             self.__logger.warn("Some issue during the undeployment ..")
 382             self.__logger.warn("Tenant clean continue ..")
 383
 384         if not self.orchestrator['requirements']['preserve_setup']:
 385             self.__logger.info('Remove the Abot_epc OS object ..')
 386             super(JujuEpc, self).clean()
 387
 388         return True
 389
 390
 391 # ----------------------------------------------------------
 392 #
 393 #               YAML UTILS
 394 #
 395 # -----------------------------------------------------------
 396 def get_config(parameter, file_path):
 397     """
 398     Returns the value of a given parameter in file.yaml
 399     parameter must be given in string format with dots
 400     Example: general.openstack.image_name
 401     """
 402     with open(file_path) as config_file:
 403         file_yaml = yaml.safe_load(config_file)
 404     config_file.close()
 405     value = file_yaml
 406     for element in parameter.split("."):
 407         value = value.get(element)
 408         if value is None:
 409             raise ValueError("The parameter %s is not defined in"
 410                              " reporting.yaml" % parameter)
 411     return value
 412
 413
 414 def sig_test_format(sig_test):
 415     """
 416     Process the signaling result to have a short result
 417     """
 418     nb_passed = 0
 419     nb_failures = 0
 420     nb_skipped = 0
 421     for data_test in sig_test:
 422         if data_test['result'] == "passed":
 423             nb_passed += 1
 424         elif data_test['result'] == "failed":
 425             nb_failures += 1
 426         elif data_test['result'] == "skipped":
 427             nb_skipped += 1
 428     total_sig_test_result = {}
 429     total_sig_test_result['passed'] = nb_passed
 430     total_sig_test_result['failures'] = nb_failures
 431     total_sig_test_result['skipped'] = nb_skipped
 432     return total_sig_test_result
 433
 434
 435 def process_abot_test_result(file_path):
 436     """ Process ABoT Result """
 437     with open(file_path) as test_result:
 438         data = json.load(test_result)
 439         res = []
 440         for tests in data:
 441             tests = update_data(tests)
 442             try:
 443                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 444                 for steps in flatten_steps:
 445                     steps['result'] = steps['step_status']
 446                     res.append(steps)
 447             except:
 448                 logging.error("Could not post data to ElasticSearch host")
 449                 raise
 450         return res
 451
 452
 453 def update_data(obj):
 454     """ Update Result data"""
 455     try:
 456         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 457
 458         for element in obj['elements']:
 459             element['final_result'] = "passed"
 460             element['flatten_steps'] = []
 461
 462             for step in element['steps']:
 463                 temp_dict = {}
 464                 step['result'][step['result']['status']] = 1
 465                 if step['result']['status'].lower() in ['fail', 'failed']:
 466                     element['final_result'] = "failed"
 467
 468                 temp_dict['feature_file'] = obj['feature_file']
 469                 temp_dict['step_name'] = step['name']
 470                 temp_dict['step_status'] = step['result']['status']
 471                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 472                 temp_dict['step_' + step['result']['status']] = 1
 473                 element['flatten_steps'].append(deepcopy(temp_dict))
 474
 475             # Need to put the tag in OBJ and not ELEMENT
 476             if 'tags' in obj:
 477                 element['tags'] = deepcopy(obj['tags'])
 478                 for tag in obj['tags']:
 479                     element[tag['name']] = 1
 480             else:
 481                 for tag in element['tags']:
 482                     element[tag['name']] = 1
 483
 484     except Exception:  # pylint: disable=broad-except
 485         logging.error("Error in updating data, %s", sys.exc_info()[0])
 486         raise
 487
 488     return obj
 489
 490
 491 def get_instance_metadata(nova_client, instance):
 492     """ Get instance Metadata - Instance ID """
 493     try:
 494         instance = nova_client.servers.get(instance.id)
 495         return instance.metadata
 496     except Exception as exc:  # pylint: disable=broad-except
 497         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 498         return None