functest/opnfv_tests/vnf/epc/juju_epc.py

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2016 Rebaca and others.
   4 #
   5 # All rights reserved. This program and the accompanying materials
   6 # are made available under the terms of the Apache License, Version 2.0
   7 # which accompanies this distribution, and is available at
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 """Juju testcase implementation."""
  10
  11 import errno
  12 import logging
  13 import os
  14 import time
  15 import json
  16 import sys
  17 import uuid
  18 from copy import deepcopy
  19 from urlparse import urljoin
  20
  21 from functest.core import vnf
  22 from functest.opnfv_tests.openstack.snaps import snaps_utils
  23 from functest.utils.constants import CONST
  24 import functest.utils.openstack_utils as os_utils
  25
  26 import pkg_resources
  27 from snaps.config.flavor import FlavorConfig
  28 from snaps.config.image import ImageConfig
  29 from snaps.config.network import NetworkConfig, SubnetConfig
  30 from snaps.config.router import RouterConfig
  31 from snaps.config.user import UserConfig
  32 from snaps.openstack.create_flavor import OpenStackFlavor
  33 from snaps.openstack.create_image import OpenStackImage
  34 from snaps.openstack.create_network import OpenStackNetwork
  35 from snaps.openstack.create_router import OpenStackRouter
  36 from snaps.openstack.create_user import OpenStackUser
  37 from snaps.openstack.utils import keystone_utils
  38 from snaps.openstack.utils import neutron_utils
  39 from snaps.openstack.utils import nova_utils
  40 import yaml
  41
  42 __author__ = "Amarendra Meher <amarendra@rebaca.com>"
  43 __author__ = "Soumaya K Nayek <soumaya.nayek@rebaca.com>"
  44
  45 CLOUD_TEMPLATE = """clouds:
  46   abot-epc:
  47     type: openstack
  48     auth-types: [userpass]
  49     endpoint: {url}
  50     regions:
  51       {region}:
  52         endpoint: {url}"""
  53
  54 CREDS_TEMPLATE2 = """credentials:
  55   abot-epc:
  56     default-credential: abot-epc
  57     abot-epc:
  58       auth-type: userpass
  59       password: {pass}
  60       project-domain-name: {project_domain_n}
  61       tenant-name: {tenant_n}"""
  62
  63 CREDS_TEMPLATE3 = """credentials:
  64   abot-epc:
  65     default-credential: abot-epc
  66     abot-epc:
  67       auth-type: userpass
  68       password: {pass}
  69       project-domain-name: {project_domain_n}
  70       tenant-name: {tenant_n}
  71       user-domain-name: {user_domain_n}
  72       username: {user_n}"""
  73
  74
  75 class JujuEpc(vnf.VnfOnBoarding):
  76     # pylint:disable=too-many-instance-attributes
  77     """Abot EPC deployed with JUJU Orchestrator Case"""
  78
  79     __logger = logging.getLogger(__name__)
  80
  81     default_region_name = "RegionOne"
  82
  83     def __init__(self, **kwargs):
  84         if "case_name" not in kwargs:
  85             kwargs["case_name"] = "juju_epc"
  86         super(JujuEpc, self).__init__(**kwargs)
  87
  88         # Retrieve the configuration
  89         self.case_dir = pkg_resources.resource_filename(
  90             'functest', 'opnfv_tests/vnf/epc')
  91         try:
  92             self.config = getattr(
  93                 CONST, 'vnf_{}_config'.format(self.case_name))
  94         except Exception:
  95             raise Exception("VNF config file not found")
  96         self.config_file = os.path.join(self.case_dir, self.config)
  97         self.orchestrator = dict(requirements=get_config(
  98             "orchestrator.requirements", self.config_file))
  99
 100         self.created_object = []
 101         self.details['orchestrator'] = dict(
 102             name=get_config("orchestrator.name", self.config_file),
 103             version=get_config("orchestrator.version", self.config_file),
 104             status='ERROR',
 105             result=''
 106         )
 107
 108         self.vnf = dict(
 109             descriptor=get_config("vnf.descriptor", self.config_file),
 110             requirements=get_config("vnf.requirements", self.config_file)
 111         )
 112         self.details['vnf'] = dict(
 113             descriptor_version=self.vnf['descriptor']['version'],
 114             name=get_config("vnf.name", self.config_file),
 115             version=get_config("vnf.version", self.config_file),
 116         )
 117         self.__logger.debug("VNF configuration: %s", self.vnf)
 118
 119         self.details['test_vnf'] = dict(
 120             name=get_config("vnf_test_suite.name", self.config_file),
 121             version=get_config("vnf_test_suite.version", self.config_file),
 122             tag_name=get_config("vnf_test_suite.tag_name", self.config_file)
 123         )
 124         self.public_auth_url = None
 125
 126         self.res_dir = os.path.join(
 127             getattr(CONST, 'dir_results'), self.case_name)
 128
 129     def _bypass_juju_network_discovery_bug(self, name):
 130         user_creator = OpenStackUser(
 131             self.snaps_creds,
 132             UserConfig(
 133                 name=name, password=str(uuid.uuid4()),
 134                 roles={'_member_': self.tenant_name}))
 135         user_creator.create()
 136         self.created_object.append(user_creator)
 137         return user_creator
 138
 139     def _register_cloud(self):
 140         self.__logger.info("Creating Cloud for Abot-epc .....")
 141         clouds_yaml = os.path.join(self.res_dir, "clouds.yaml")
 142         cloud_data = {
 143             'url': self.public_auth_url,
 144             'region': os.environ.get(
 145                 "OS_REGION_NAME", self.default_region_name)}
 146         with open(clouds_yaml, 'w') as yfile:
 147             yfile.write(CLOUD_TEMPLATE.format(**cloud_data))
 148         if os.system(
 149                 'juju add-cloud abot-epc -f {} --replace'.format(clouds_yaml)):
 150             raise vnf.VnfPreparationException
 151
 152     def _register_credentials_v2(self):
 153         self.__logger.info("Creating Credentials for Abot-epc .....")
 154         user_creator = self._bypass_juju_network_discovery_bug(
 155             'juju_network_discovery_bug')
 156         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 157         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 158         creds_data = {
 159             'pass': snaps_creds.password,
 160             'tenant_n': snaps_creds.project_name,
 161             'user_n': snaps_creds.username}
 162         with open(credentials_yaml, 'w') as yfile:
 163             yfile.write(CREDS_TEMPLATE2.format(**creds_data))
 164         if os.system(
 165                 'juju add-credential abot-epc -f {} --replace'.format(
 166                     credentials_yaml)):
 167             raise vnf.VnfPreparationException
 168
 169     def _register_credentials_v3(self):
 170         self.__logger.info("Creating Credentials for Abot-epc .....")
 171         user_creator = self._bypass_juju_network_discovery_bug(
 172             'juju_network_discovery_bug')
 173         snaps_creds = user_creator.get_os_creds(self.snaps_creds.project_name)
 174         credentials_yaml = os.path.join(self.res_dir, "credentials.yaml")
 175         creds_data = {
 176             'pass': snaps_creds.password,
 177             'tenant_n': snaps_creds.project_name,
 178             'user_n': snaps_creds.username,
 179             'project_domain_n': snaps_creds.project_domain_name,
 180             'user_domain_n': snaps_creds.user_domain_name}
 181         with open(credentials_yaml, 'w') as yfile:
 182             yfile.write(CREDS_TEMPLATE3.format(**creds_data))
 183         if os.system(
 184                 'juju add-credential abot-epc -f {} --replace'.format(
 185                     credentials_yaml)):
 186             raise vnf.VnfPreparationException
 187
 188     def prepare(self):
 189         """Prepare testcase (Additional pre-configuration steps)."""
 190         self.__logger.info("Additional pre-configuration steps")
 191         super(JujuEpc, self).prepare()
 192         try:
 193             os.makedirs(self.res_dir)
 194         except OSError as ex:
 195             if ex.errno != errno.EEXIST:
 196                 self.__logger.exception("Cannot create %s", self.res_dir)
 197                 raise vnf.VnfPreparationException
 198         self.public_auth_url = keystone_utils.get_endpoint(
 199             self.snaps_creds, 'identity')
 200         # it enforces a versioned public identity endpoint as juju simply
 201         # adds /auth/tokens wich fails vs an unversioned endpoint.
 202         if not self.public_auth_url.endswith(('v3', 'v3/', 'v2.0', 'v2.0/')):
 203             self.public_auth_url = urljoin(self.public_auth_url, 'v3')
 204         self._register_cloud()
 205         if self.snaps_creds.identity_api_version == 3:
 206             self._register_credentials_v3()
 207         else:
 208             self._register_credentials_v2()
 209
 210     def deploy_orchestrator(self):  # pylint: disable=too-many-locals
 211         """
 212         Create network, subnet, router
 213
 214         Bootstrap juju
 215         """
 216         self.__logger.info("Deployed Orchestrator")
 217         private_net_name = getattr(
 218             CONST, 'vnf_{}_private_net_name'.format(self.case_name))
 219         private_subnet_name = getattr(
 220             CONST, 'vnf_{}_private_subnet_name'.format(self.case_name))
 221         private_subnet_cidr = getattr(
 222             CONST, 'vnf_{}_private_subnet_cidr'.format(self.case_name))
 223         abot_router = getattr(
 224             CONST, 'vnf_{}_external_router'.format(self.case_name))
 225
 226         self.__logger.info("Creating full network ...")
 227         subnet_settings = SubnetConfig(
 228             name=private_subnet_name, cidr=private_subnet_cidr)
 229         network_settings = NetworkConfig(
 230             name=private_net_name, subnet_settings=[subnet_settings])
 231         network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
 232         net_id = network_creator.create().id
 233         self.created_object.append(network_creator)
 234
 235         ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
 236         self.__logger.info("Creating network Router ....")
 237         router_creator = OpenStackRouter(
 238             self.snaps_creds, RouterConfig(
 239                 name=abot_router,
 240                 external_gateway=ext_net_name,
 241                 internal_subnets=[subnet_settings.name]))
 242         router_creator.create()
 243         self.created_object.append(router_creator)
 244         self.__logger.info("Creating Flavor ....")
 245         flavor_settings = FlavorConfig(
 246             name=self.orchestrator['requirements']['flavor']['name'],
 247             ram=self.orchestrator['requirements']['flavor']['ram_min'],
 248             disk=10, vcpus=1)
 249         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 250         flavor_creator.create()
 251         self.created_object.append(flavor_creator)
 252         self.__logger.info("Upload some OS images if it doesn't exist")
 253         images = get_config("tenant_images", self.config_file)
 254         self.__logger.info("Images needed for vEPC: %s", images)
 255         for image_name, image_file in images.iteritems():
 256             self.__logger.info("image: %s, file: %s", image_name, image_file)
 257             if image_file and image_name:
 258                 image_creator = OpenStackImage(self.snaps_creds, ImageConfig(
 259                     name=image_name, image_user='cloud', img_format='qcow2',
 260                     image_file=image_file))
 261                 image_id = image_creator.create().id
 262                 os.system(
 263                     'juju metadata generate-image -d ~ -i {} -s {} -r '
 264                     '{} -u {}'.format(
 265                         image_id, image_name,
 266                         os.environ.get(
 267                             "OS_REGION_NAME", self.default_region_name),
 268                         self.public_auth_url))
 269                 self.created_object.append(image_creator)
 270         self.__logger.info("Credential information  : %s", net_id)
 271         juju_bootstrap_command = (
 272             'juju bootstrap abot-epc abot-controller --config network={} '
 273             '--metadata-source ~  --config ssl-hostname-verification=false '
 274             '--constraints mem=2G --bootstrap-series xenial '
 275             '--config use-floating-ip=true --debug '
 276             '--config use-default-secgroup=true'.format(net_id))
 277         os.system(juju_bootstrap_command)
 278         return True
 279
 280     def deploy_vnf(self):
 281         """Deploy ABOT-OAI-EPC."""
 282         self.__logger.info("Upload VNFD")
 283         descriptor = self.vnf['descriptor']
 284         self.__logger.info("Get or create flavor for all Abot-EPC")
 285         flavor_settings = FlavorConfig(
 286             name=self.vnf['requirements']['flavor']['name'],
 287             ram=self.vnf['requirements']['flavor']['ram_min'],
 288             disk=10,
 289             vcpus=1)
 290         flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
 291         flavor_creator.create()
 292         self.created_object.append(flavor_creator)
 293         self.__logger.info("Deploying Abot-epc bundle file ...")
 294         os.system('juju deploy {}'.format('/' + descriptor.get('file_name')))
 295         self.__logger.info("Waiting for instances .....")
 296         status = os.system('juju-wait')
 297         self.__logger.info("juju wait completed: %s", status)
 298         self.__logger.info("Deployed Abot-epc on Openstack")
 299         nova_client = nova_utils.nova_client(self.snaps_creds)
 300         neutron_client = neutron_utils.neutron_client(self.snaps_creds)
 301         if status == 0:
 302             instances = os_utils.get_instances(nova_client)
 303             for items in instances:
 304                 metadata = get_instance_metadata(nova_client, items)
 305                 if 'juju-units-deployed' in metadata:
 306                     sec_group = ('juju-' + metadata['juju-controller-uuid'] +
 307                                  '-' + metadata['juju-model-uuid'])
 308                     self.sec_group_id = os_utils.get_security_group_id(
 309                         neutron_client, sec_group)
 310                     break
 311             self.__logger.info("Adding Security group rule....")
 312             os_utils.create_secgroup_rule(
 313                 neutron_client, self.sec_group_id, 'ingress', 132)
 314             self.__logger.info("Copying the feature files to Abot_node ")
 315             os.system('juju scp -- -r {}/featureFiles abot-'
 316                       'epc-basic/0:~/'.format(self.case_dir))
 317             self.__logger.info("Copying the feature files in Abot_node ")
 318             os.system("juju ssh abot-epc-basic/0 'sudo rsync -azvv "
 319                       "~/featureFiles /etc/rebaca-test-suite"
 320                       "/featureFiles'")
 321             count = 0
 322             while count < 10:
 323                 epcstatus = os.system('juju status oai-epc | '
 324                                       'grep {} | grep {} | grep {}'
 325                                       .format('EPC', 'is', 'running'))
 326                 if epcstatus == 0:
 327                     break
 328                 else:
 329                     time.sleep(60)
 330                     count = count + 1
 331             os.system('juju-wait')
 332             return True
 333         return False
 334
 335     def test_vnf(self):
 336         """Run test on ABoT."""
 337         start_time = time.time()
 338         self.__logger.info("Running VNF Test cases....")
 339         os.system('juju run-action abot-epc-basic/0 run '
 340                   'tagnames={}'.format(self.details['test_vnf']['tag_name']))
 341         os.system('juju-wait')
 342         duration = time.time() - start_time
 343         self.__logger.info("Getting results from Abot node....")
 344         os.system('juju scp abot-epc-basic/0:/var/lib/abot-'
 345                   'epc-basic/artifacts/TestResults.json {}/.'
 346                   .format(self.case_dir))
 347         self.__logger.info("Parsing the Test results...")
 348         res = (process_abot_test_result('{}/TestResults.'
 349                                         'json'.format(self.case_dir)))
 350         short_result = sig_test_format(res)
 351         self.__logger.info(short_result)
 352         self.details['test_vnf'].update(status='PASS',
 353                                         result=short_result,
 354                                         full_result=res,
 355                                         duration=duration)
 356
 357         self.__logger.info("Test VNF result: Passed: %d, Failed:"
 358                            "%d, Skipped: %d", short_result['passed'],
 359                            short_result['failures'], short_result['skipped'])
 360         return True
 361
 362     def clean(self):
 363         """Clean created objects/functions."""
 364         try:
 365             if not self.orchestrator['requirements']['preserve_setup']:
 366                 self.__logger.info("Removing deployment files...")
 367                 testresult = os.path.join(self.case_dir, 'TestResults.json')
 368                 if os.path.exists(testresult):
 369                     os.remove(testresult)
 370                 self.__logger.info("Destroying Orchestrator...")
 371                 os.system('juju destroy-controller -y abot-controller '
 372                           '--destroy-all-models')
 373         except Exception:  # pylint: disable=broad-except
 374             self.__logger.warn("Some issue during the undeployment ..")
 375             self.__logger.warn("Tenant clean continue ..")
 376
 377         if not self.orchestrator['requirements']['preserve_setup']:
 378             self.__logger.info('Remove the Abot_epc OS object ..')
 379             super(JujuEpc, self).clean()
 380
 381         return True
 382
 383
 384 # ----------------------------------------------------------
 385 #
 386 #               YAML UTILS
 387 #
 388 # -----------------------------------------------------------
 389 def get_config(parameter, file_path):
 390     """
 391     Returns the value of a given parameter in file.yaml
 392     parameter must be given in string format with dots
 393     Example: general.openstack.image_name
 394     """
 395     with open(file_path) as config_file:
 396         file_yaml = yaml.safe_load(config_file)
 397     config_file.close()
 398     value = file_yaml
 399     for element in parameter.split("."):
 400         value = value.get(element)
 401         if value is None:
 402             raise ValueError("The parameter %s is not defined in"
 403                              " reporting.yaml" % parameter)
 404     return value
 405
 406
 407 def sig_test_format(sig_test):
 408     """
 409     Process the signaling result to have a short result
 410     """
 411     nb_passed = 0
 412     nb_failures = 0
 413     nb_skipped = 0
 414     for data_test in sig_test:
 415         if data_test['result'] == "passed":
 416             nb_passed += 1
 417         elif data_test['result'] == "failed":
 418             nb_failures += 1
 419         elif data_test['result'] == "skipped":
 420             nb_skipped += 1
 421     total_sig_test_result = {}
 422     total_sig_test_result['passed'] = nb_passed
 423     total_sig_test_result['failures'] = nb_failures
 424     total_sig_test_result['skipped'] = nb_skipped
 425     return total_sig_test_result
 426
 427
 428 def process_abot_test_result(file_path):
 429     """ Process ABoT Result """
 430     with open(file_path) as test_result:
 431         data = json.load(test_result)
 432         res = []
 433         for tests in data:
 434             tests = update_data(tests)
 435             try:
 436                 flatten_steps = tests['elements'][0].pop('flatten_steps')
 437                 for steps in flatten_steps:
 438                     steps['result'] = steps['step_status']
 439                     res.append(steps)
 440             except:
 441                 logging.error("Could not post data to ElasticSearch host")
 442                 raise
 443         return res
 444
 445
 446 def update_data(obj):
 447     """ Update Result data"""
 448     try:
 449         obj['feature_file'] = os.path.splitext(os.path.basename(obj['uri']))[0]
 450
 451         for element in obj['elements']:
 452             element['final_result'] = "passed"
 453             element['flatten_steps'] = []
 454
 455             for step in element['steps']:
 456                 temp_dict = {}
 457                 step['result'][step['result']['status']] = 1
 458                 if step['result']['status'].lower() in ['fail', 'failed']:
 459                     element['final_result'] = "failed"
 460
 461                 temp_dict['feature_file'] = obj['feature_file']
 462                 temp_dict['step_name'] = step['name']
 463                 temp_dict['step_status'] = step['result']['status']
 464                 temp_dict['step_duration'] = step['result'].get('duration', 0)
 465                 temp_dict['step_' + step['result']['status']] = 1
 466                 element['flatten_steps'].append(deepcopy(temp_dict))
 467
 468             # Need to put the tag in OBJ and not ELEMENT
 469             if 'tags' in obj:
 470                 element['tags'] = deepcopy(obj['tags'])
 471                 for tag in obj['tags']:
 472                     element[tag['name']] = 1
 473             else:
 474                 for tag in element['tags']:
 475                     element[tag['name']] = 1
 476
 477     except Exception:  # pylint: disable=broad-except
 478         logging.error("Error in updating data, %s", sys.exc_info()[0])
 479         raise
 480
 481     return obj
 482
 483
 484 def get_instance_metadata(nova_client, instance):
 485     """ Get instance Metadata - Instance ID """
 486     try:
 487         instance = nova_client.servers.get(instance.id)
 488         return instance.metadata
 489     except Exception as exc:  # pylint: disable=broad-except
 490         logging.error("Error [get_instance_status(nova_client)]: %s", exc)
 491         return None