functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 import ConfigParser
  11 import logging
  12 import os
  13 import pkg_resources
  14 import re
  15 import shutil
  16 import subprocess
  17
  18 from functest.utils.constants import CONST
  19 import functest.utils.functest_utils as ft_utils
  20 import functest.utils.openstack_utils as os_utils
  21
  22
  23 IMAGE_ID_ALT = None
  24 FLAVOR_ID_ALT = None
  25 GLANCE_IMAGE_PATH = os.path.join(
  26     CONST.__getattribute__('dir_functest_images'),
  27     CONST.__getattribute__('openstack_image_file_name'))
  28 TEMPEST_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  29                                    'tempest')
  30 TEMPEST_CUSTOM = pkg_resources.resource_filename(
  31         'functest', 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  32 TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  33         'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.txt')
  34 TEMPEST_DEFCORE = pkg_resources.resource_filename(
  35         'functest',
  36         'opnfv_tests/openstack/tempest/custom_tests/defcore_req.txt')
  37 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  38 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  39 REFSTACK_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  40                                     'refstack')
  41
  42 CI_INSTALLER_TYPE = CONST.__getattribute__('INSTALLER_TYPE')
  43 CI_INSTALLER_IP = CONST.__getattribute__('INSTALLER_IP')
  44
  45 """ logging configuration """
  46 logger = logging.getLogger(__name__)
  47
  48
  49 def create_tempest_resources(use_custom_images=False,
  50                              use_custom_flavors=False):
  51     keystone_client = os_utils.get_keystone_client()
  52
  53     logger.debug("Creating tenant and user for Tempest suite")
  54     tenant_id = os_utils.create_tenant(
  55         keystone_client,
  56         CONST.__getattribute__('tempest_identity_tenant_name'),
  57         CONST.__getattribute__('tempest_identity_tenant_description'))
  58     if not tenant_id:
  59         logger.error("Failed to create %s tenant"
  60                      % CONST.__getattribute__('tempest_identity_tenant_name'))
  61
  62     user_id = os_utils.create_user(
  63         keystone_client,
  64         CONST.__getattribute__('tempest_identity_user_name'),
  65         CONST.__getattribute__('tempest_identity_user_password'),
  66         None, tenant_id)
  67     if not user_id:
  68         logger.error("Failed to create %s user" %
  69                      CONST.__getattribute__('tempest_identity_user_name'))
  70
  71     logger.debug("Creating private network for Tempest suite")
  72     network_dic = os_utils.create_shared_network_full(
  73         CONST.__getattribute__('tempest_private_net_name'),
  74         CONST.__getattribute__('tempest_private_subnet_name'),
  75         CONST.__getattribute__('tempest_router_name'),
  76         CONST.__getattribute__('tempest_private_subnet_cidr'))
  77     if network_dic is None:
  78         raise Exception('Failed to create private network')
  79
  80     image_id = ""
  81     image_id_alt = ""
  82     flavor_id = ""
  83     flavor_id_alt = ""
  84
  85     if (CONST.__getattribute__('tempest_use_custom_images') or
  86        use_custom_images):
  87         # adding alternative image should be trivial should we need it
  88         logger.debug("Creating image for Tempest suite")
  89         _, image_id = os_utils.get_or_create_image(
  90             CONST.__getattribute__('openstack_image_name'),
  91             GLANCE_IMAGE_PATH,
  92             CONST.__getattribute__('openstack_image_disk_format'))
  93         if image_id is None:
  94             raise Exception('Failed to create image')
  95
  96     if use_custom_images:
  97         logger.debug("Creating 2nd image for Tempest suite")
  98         _, image_id_alt = os_utils.get_or_create_image(
  99             CONST.__getattribute__('openstack_image_name_alt'),
 100             GLANCE_IMAGE_PATH,
 101             CONST.__getattribute__('openstack_image_disk_format'))
 102         if image_id_alt is None:
 103             raise Exception('Failed to create image')
 104
 105     if (CONST.__getattribute__('tempest_use_custom_flavors') or
 106        use_custom_flavors):
 107         # adding alternative flavor should be trivial should we need it
 108         logger.debug("Creating flavor for Tempest suite")
 109         _, flavor_id = os_utils.get_or_create_flavor(
 110             CONST.__getattribute__('openstack_flavor_name'),
 111             CONST.__getattribute__('openstack_flavor_ram'),
 112             CONST.__getattribute__('openstack_flavor_disk'),
 113             CONST.__getattribute__('openstack_flavor_vcpus'))
 114         if flavor_id is None:
 115             raise Exception('Failed to create flavor')
 116
 117     if use_custom_flavors:
 118         logger.debug("Creating 2nd flavor for tempest_defcore")
 119         _, flavor_id_alt = os_utils.get_or_create_flavor(
 120             CONST.__getattribute__('openstack_flavor_name_alt'),
 121             CONST.__getattribute__('openstack_flavor_ram'),
 122             CONST.__getattribute__('openstack_flavor_disk'),
 123             CONST.__getattribute__('openstack_flavor_vcpus'))
 124         if flavor_id_alt is None:
 125             raise Exception('Failed to create flavor')
 126
 127     img_flavor_dict = {}
 128     img_flavor_dict['image_id'] = image_id
 129     img_flavor_dict['image_id_alt'] = image_id_alt
 130     img_flavor_dict['flavor_id'] = flavor_id
 131     img_flavor_dict['flavor_id_alt'] = flavor_id_alt
 132
 133     return img_flavor_dict
 134
 135
 136 def get_verifier_id():
 137     """
 138     Returns verifer id for current Tempest
 139     """
 140     cmd = ("rally verify list-verifiers | awk '/" +
 141            CONST.__getattribute__('tempest_deployment_name') +
 142            "/ {print $2}'")
 143     p = subprocess.Popen(cmd, shell=True,
 144                          stdout=subprocess.PIPE,
 145                          stderr=subprocess.STDOUT)
 146     deployment_uuid = p.stdout.readline().rstrip()
 147     if deployment_uuid == "":
 148         logger.error("Tempest verifier not found.")
 149         raise Exception('Error with command:%s' % cmd)
 150     return deployment_uuid
 151
 152
 153 def get_verifier_deployment_id():
 154     """
 155     Returns deployment id for active Rally deployment
 156     """
 157     cmd = ("rally deployment list | awk '/" +
 158            CONST.__getattribute__('rally_deployment_name') +
 159            "/ {print $2}'")
 160     p = subprocess.Popen(cmd, shell=True,
 161                          stdout=subprocess.PIPE,
 162                          stderr=subprocess.STDOUT)
 163     deployment_uuid = p.stdout.readline().rstrip()
 164     if deployment_uuid == "":
 165         logger.error("Rally deployment not found.")
 166         raise Exception('Error with command:%s' % cmd)
 167     return deployment_uuid
 168
 169
 170 def get_verifier_repo_dir(verifier_id):
 171     """
 172     Returns installed verfier repo directory for Tempest
 173     """
 174     if not verifier_id:
 175         verifier_id = get_verifier_id()
 176
 177     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 178                         'verification',
 179                         'verifier-{}'.format(verifier_id),
 180                         'repo')
 181
 182
 183 def get_verifier_deployment_dir(verifier_id, deployment_id):
 184     """
 185     Returns Rally deployment directory for current verifier
 186     """
 187     if not verifier_id:
 188         verifier_id = get_verifier_id()
 189
 190     if not deployment_id:
 191         deployment_id = get_verifier_deployment_id()
 192
 193     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 194                         'verification',
 195                         'verifier-{}'.format(verifier_id),
 196                         'for-deployment-{}'.format(deployment_id))
 197
 198
 199 def get_repo_tag(repo):
 200     """
 201     Returns last tag of current branch
 202     """
 203     cmd = ("git -C {0} describe --abbrev=0 HEAD".format(repo))
 204     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 205     tag = p.stdout.readline().rstrip()
 206
 207     return str(tag)
 208
 209
 210 def backup_tempest_config(conf_file):
 211     """
 212     Copy config file to tempest results directory
 213     """
 214     if not os.path.exists(TEMPEST_RESULTS_DIR):
 215         os.makedirs(TEMPEST_RESULTS_DIR)
 216
 217     shutil.copyfile(conf_file,
 218                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 219
 220
 221 def configure_tempest(deployment_dir, IMAGE_ID=None, FLAVOR_ID=None,
 222                       MODE=None):
 223     """
 224     Calls rally verify and updates the generated tempest.conf with
 225     given parameters
 226     """
 227     conf_file = configure_verifier(deployment_dir)
 228     configure_tempest_update_params(conf_file,
 229                                     IMAGE_ID, FLAVOR_ID)
 230     if MODE == 'feature_multisite':
 231         configure_tempest_multisite_params(conf_file)
 232
 233
 234 def configure_tempest_defcore(deployment_dir, img_flavor_dict):
 235     """
 236     Add/update needed parameters into tempest.conf file
 237     """
 238     conf_file = configure_verifier(deployment_dir)
 239     configure_tempest_update_params(conf_file,
 240                                     img_flavor_dict.get("image_id"),
 241                                     img_flavor_dict.get("flavor_id"))
 242
 243     logger.debug("Updating selected tempest.conf parameters for defcore...")
 244     config = ConfigParser.RawConfigParser()
 245     config.read(conf_file)
 246     config.set('compute', 'image_ref', img_flavor_dict.get("image_id"))
 247     config.set('compute', 'image_ref_alt',
 248                img_flavor_dict['image_id_alt'])
 249     config.set('compute', 'flavor_ref', img_flavor_dict.get("flavor_id"))
 250     config.set('compute', 'flavor_ref_alt',
 251                img_flavor_dict['flavor_id_alt'])
 252
 253     with open(conf_file, 'wb') as config_file:
 254         config.write(config_file)
 255
 256     confpath = pkg_resources.resource_filename(
 257         'functest',
 258         'opnfv_tests/openstack/refstack_client/refstack_tempest.conf')
 259     shutil.copyfile(conf_file, confpath)
 260
 261
 262 def configure_tempest_update_params(tempest_conf_file,
 263                                     IMAGE_ID=None, FLAVOR_ID=None):
 264     """
 265     Add/update needed parameters into tempest.conf file
 266     """
 267     logger.debug("Updating selected tempest.conf parameters...")
 268     config = ConfigParser.RawConfigParser()
 269     config.read(tempest_conf_file)
 270     config.set(
 271         'compute',
 272         'fixed_network_name',
 273         CONST.__getattribute__('tempest_private_net_name'))
 274     config.set('compute', 'volume_device_name',
 275                CONST.__getattribute__('tempest_volume_device_name'))
 276     if CONST.__getattribute__('tempest_use_custom_images'):
 277         if IMAGE_ID is not None:
 278             config.set('compute', 'image_ref', IMAGE_ID)
 279         if IMAGE_ID_ALT is not None:
 280             config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 281     if CONST.__getattribute__('tempest_use_custom_flavors'):
 282         if FLAVOR_ID is not None:
 283             config.set('compute', 'flavor_ref', FLAVOR_ID)
 284         if FLAVOR_ID_ALT is not None:
 285             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 286     config.set('identity', 'tenant_name',
 287                CONST.__getattribute__('tempest_identity_tenant_name'))
 288     config.set('identity', 'username',
 289                CONST.__getattribute__('tempest_identity_user_name'))
 290     config.set('identity', 'password',
 291                CONST.__getattribute__('tempest_identity_user_password'))
 292     config.set('identity', 'region', 'RegionOne')
 293     if os_utils.is_keystone_v3():
 294         auth_version = 'v3'
 295     else:
 296         auth_version = 'v2'
 297     config.set('identity', 'auth_version', auth_version)
 298     config.set(
 299         'validation', 'ssh_timeout',
 300         CONST.__getattribute__('tempest_validation_ssh_timeout'))
 301     config.set('object-storage', 'operator_role',
 302                CONST.__getattribute__('tempest_object_storage_operator_role'))
 303
 304     if CONST.__getattribute__('OS_ENDPOINT_TYPE') is not None:
 305         sections = config.sections()
 306         if os_utils.is_keystone_v3():
 307             config.set('identity', 'v3_endpoint_type',
 308                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 309             if 'identity-feature-enabled' not in sections:
 310                 config.add_section('identity-feature-enabled')
 311                 config.set('identity-feature-enabled', 'api_v2', False)
 312                 config.set('identity-feature-enabled', 'api_v2_admin', False)
 313         services_list = ['compute',
 314                          'volume',
 315                          'image',
 316                          'network',
 317                          'data-processing',
 318                          'object-storage',
 319                          'orchestration']
 320         for service in services_list:
 321             if service not in sections:
 322                 config.add_section(service)
 323             config.set(service, 'endpoint_type',
 324                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 325
 326     with open(tempest_conf_file, 'wb') as config_file:
 327         config.write(config_file)
 328
 329     backup_tempest_config(tempest_conf_file)
 330
 331
 332 def configure_verifier(deployment_dir):
 333     """
 334     Execute rally verify configure-verifier, which generates tempest.conf
 335     """
 336     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 337     if os.path.isfile(tempest_conf_file):
 338         logger.debug("Verifier is already configured.")
 339         logger.debug("Reconfiguring the current verifier...")
 340         cmd = "rally verify configure-verifier --reconfigure"
 341     else:
 342         logger.info("Configuring the verifier...")
 343         cmd = "rally verify configure-verifier"
 344     ft_utils.execute_command(cmd)
 345
 346     logger.debug("Looking for tempest.conf file...")
 347     if not os.path.isfile(tempest_conf_file):
 348         logger.error("Tempest configuration file %s NOT found."
 349                      % tempest_conf_file)
 350         raise Exception("Tempest configuration file %s NOT found."
 351                         % tempest_conf_file)
 352     else:
 353         return tempest_conf_file
 354
 355
 356 def configure_tempest_multisite_params(tempest_conf_file):
 357     """
 358     Add/update multisite parameters into tempest.conf file generated by Rally
 359     """
 360     logger.debug("Updating multisite tempest.conf parameters...")
 361     config = ConfigParser.RawConfigParser()
 362     config.read(tempest_conf_file)
 363
 364     config.set('service_available', 'kingbird', 'true')
 365     # cmd = ("openstack endpoint show kingbird | grep publicurl |"
 366     #       "awk '{print $4}' | awk -F '/' '{print $4}'")
 367     # kingbird_api_version = os.popen(cmd).read()
 368     # kingbird_api_version = os_utils.get_endpoint(service_type='multisite')
 369
 370     if CI_INSTALLER_TYPE == 'fuel':
 371         # For MOS based setup, the service is accessible
 372         # via bind host
 373         kingbird_conf_path = "/etc/kingbird/kingbird.conf"
 374         installer_type = CI_INSTALLER_TYPE
 375         installer_ip = CI_INSTALLER_IP
 376         installer_username = CONST.__getattribute__(
 377             'multisite_{}_installer_username'.format(installer_type))
 378         installer_password = CONST.__getattribute__(
 379             'multisite_{}_installer_password'.format(installer_type))
 380
 381         ssh_options = ("-o UserKnownHostsFile=/dev/null -o "
 382                        "StrictHostKeyChecking=no")
 383
 384         # Get the controller IP from the fuel node
 385         cmd = ('sshpass -p %s ssh 2>/dev/null %s %s@%s '
 386                '\'fuel node --env 1| grep controller | grep "True\|  1" '
 387                '| awk -F\| "{print \$5}"\'' % (installer_password,
 388                                                ssh_options,
 389                                                installer_username,
 390                                                installer_ip))
 391         multisite_controller_ip = "".join(os.popen(cmd).read().split())
 392
 393         # Login to controller and get bind host details
 394         cmd = ('sshpass -p %s ssh 2>/dev/null  %s %s@%s "ssh %s \\" '
 395                'grep -e "^bind_" %s  \\""' % (installer_password,
 396                                               ssh_options,
 397                                               installer_username,
 398                                               installer_ip,
 399                                               multisite_controller_ip,
 400                                               kingbird_conf_path))
 401         bind_details = os.popen(cmd).read()
 402         bind_details = "".join(bind_details.split())
 403         # Extract port number from the bind details
 404         bind_port = re.findall(r"\D(\d{4})", bind_details)[0]
 405         # Extract ip address from the bind details
 406         bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
 407                                bind_details)[0]
 408         kingbird_endpoint_url = "http://%s:%s/" % (bind_host, bind_port)
 409     else:
 410         # cmd = "openstack endpoint show kingbird | grep publicurl |\
 411         #       awk '{print $4}' | awk -F '/' '{print $3}'"
 412         # kingbird_endpoint_url = os.popen(cmd).read()
 413         kingbird_endpoint_url = os_utils.get_endpoint(service_type='kingbird')
 414
 415     try:
 416         config.add_section("kingbird")
 417     except Exception:
 418         logger.info('kingbird section exist')
 419
 420     # set the domain id
 421     config.set('auth', 'admin_domain_name', 'default')
 422
 423     config.set('kingbird', 'endpoint_type', 'publicURL')
 424     config.set('kingbird', 'TIME_TO_SYNC', '120')
 425     config.set('kingbird', 'endpoint_url', kingbird_endpoint_url)
 426     config.set('kingbird', 'api_version', 'v1.0')
 427     with open(tempest_conf_file, 'wb') as config_file:
 428         config.write(config_file)
 429
 430     backup_tempest_config(tempest_conf_file)
 431
 432
 433 def install_verifier_ext(path):
 434     """
 435     Install extension to active verifier
 436     """
 437     logger.info("Installing verifier from existing repo...")
 438     tag = get_repo_tag(path)
 439     cmd = ("rally verify add-verifier-ext --source {0} "
 440            "--version {1}"
 441            .format(path, tag))
 442     error_msg = ("Problem while adding verifier extension from %s" % path)
 443     ft_utils.execute_command_raise(cmd, error_msg=error_msg)