functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 import ConfigParser
  11 import logging
  12 import os
  13 import pkg_resources
  14 import re
  15 import shutil
  16 import subprocess
  17
  18 import yaml
  19
  20 from functest.utils.constants import CONST
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23
  24
  25 IMAGE_ID_ALT = None
  26 FLAVOR_ID_ALT = None
  27 GLANCE_IMAGE_PATH = os.path.join(
  28     CONST.__getattribute__('dir_functest_images'),
  29     CONST.__getattribute__('openstack_image_file_name'))
  30 TEMPEST_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  31                                    'tempest')
  32 TEMPEST_CUSTOM = pkg_resources.resource_filename(
  33     'functest', 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  34 TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  35     'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.txt')
  36 TEMPEST_DEFCORE = pkg_resources.resource_filename(
  37     'functest',
  38     'opnfv_tests/openstack/tempest/custom_tests/defcore_req.txt')
  39 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  40 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  41 REFSTACK_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  42                                     'refstack')
  43 TEMPEST_CONF_YAML = pkg_resources.resource_filename(
  44     'functest', 'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
  45
  46 CI_INSTALLER_TYPE = CONST.__getattribute__('INSTALLER_TYPE')
  47 CI_INSTALLER_IP = CONST.__getattribute__('INSTALLER_IP')
  48
  49 """ logging configuration """
  50 logger = logging.getLogger(__name__)
  51
  52
  53 def create_tempest_resources(use_custom_images=False,
  54                              use_custom_flavors=False):
  55     keystone_client = os_utils.get_keystone_client()
  56
  57     logger.debug("Creating tenant and user for Tempest suite")
  58     tenant_id = os_utils.create_tenant(
  59         keystone_client,
  60         CONST.__getattribute__('tempest_identity_tenant_name'),
  61         CONST.__getattribute__('tempest_identity_tenant_description'))
  62     if not tenant_id:
  63         logger.error("Failed to create %s tenant"
  64                      % CONST.__getattribute__('tempest_identity_tenant_name'))
  65
  66     user_id = os_utils.create_user(
  67         keystone_client,
  68         CONST.__getattribute__('tempest_identity_user_name'),
  69         CONST.__getattribute__('tempest_identity_user_password'),
  70         None, tenant_id)
  71     if not user_id:
  72         logger.error("Failed to create %s user" %
  73                      CONST.__getattribute__('tempest_identity_user_name'))
  74
  75     logger.debug("Creating private network for Tempest suite")
  76     network_dic = os_utils.create_shared_network_full(
  77         CONST.__getattribute__('tempest_private_net_name'),
  78         CONST.__getattribute__('tempest_private_subnet_name'),
  79         CONST.__getattribute__('tempest_router_name'),
  80         CONST.__getattribute__('tempest_private_subnet_cidr'))
  81     if network_dic is None:
  82         raise Exception('Failed to create private network')
  83
  84     image_id = ""
  85     image_id_alt = ""
  86     flavor_id = ""
  87     flavor_id_alt = ""
  88
  89     if (CONST.__getattribute__('tempest_use_custom_images') or
  90        use_custom_images):
  91         # adding alternative image should be trivial should we need it
  92         logger.debug("Creating image for Tempest suite")
  93         _, image_id = os_utils.get_or_create_image(
  94             CONST.__getattribute__('openstack_image_name'),
  95             GLANCE_IMAGE_PATH,
  96             CONST.__getattribute__('openstack_image_disk_format'))
  97         if image_id is None:
  98             raise Exception('Failed to create image')
  99
 100     if use_custom_images:
 101         logger.debug("Creating 2nd image for Tempest suite")
 102         _, image_id_alt = os_utils.get_or_create_image(
 103             CONST.__getattribute__('openstack_image_name_alt'),
 104             GLANCE_IMAGE_PATH,
 105             CONST.__getattribute__('openstack_image_disk_format'))
 106         if image_id_alt is None:
 107             raise Exception('Failed to create image')
 108
 109     if (CONST.__getattribute__('tempest_use_custom_flavors') or
 110        use_custom_flavors):
 111         # adding alternative flavor should be trivial should we need it
 112         logger.debug("Creating flavor for Tempest suite")
 113         _, flavor_id = os_utils.get_or_create_flavor(
 114             CONST.__getattribute__('openstack_flavor_name'),
 115             CONST.__getattribute__('openstack_flavor_ram'),
 116             CONST.__getattribute__('openstack_flavor_disk'),
 117             CONST.__getattribute__('openstack_flavor_vcpus'))
 118         if flavor_id is None:
 119             raise Exception('Failed to create flavor')
 120
 121     if use_custom_flavors:
 122         logger.debug("Creating 2nd flavor for tempest_defcore")
 123         _, flavor_id_alt = os_utils.get_or_create_flavor(
 124             CONST.__getattribute__('openstack_flavor_name_alt'),
 125             CONST.__getattribute__('openstack_flavor_ram'),
 126             CONST.__getattribute__('openstack_flavor_disk'),
 127             CONST.__getattribute__('openstack_flavor_vcpus'))
 128         if flavor_id_alt is None:
 129             raise Exception('Failed to create flavor')
 130
 131     img_flavor_dict = {}
 132     img_flavor_dict['image_id'] = image_id
 133     img_flavor_dict['image_id_alt'] = image_id_alt
 134     img_flavor_dict['flavor_id'] = flavor_id
 135     img_flavor_dict['flavor_id_alt'] = flavor_id_alt
 136
 137     return img_flavor_dict
 138
 139
 140 def get_verifier_id():
 141     """
 142     Returns verifer id for current Tempest
 143     """
 144     cmd = ("rally verify list-verifiers | awk '/" +
 145            CONST.__getattribute__('tempest_deployment_name') +
 146            "/ {print $2}'")
 147     p = subprocess.Popen(cmd, shell=True,
 148                          stdout=subprocess.PIPE,
 149                          stderr=subprocess.STDOUT)
 150     deployment_uuid = p.stdout.readline().rstrip()
 151     if deployment_uuid == "":
 152         logger.error("Tempest verifier not found.")
 153         raise Exception('Error with command:%s' % cmd)
 154     return deployment_uuid
 155
 156
 157 def get_verifier_deployment_id():
 158     """
 159     Returns deployment id for active Rally deployment
 160     """
 161     cmd = ("rally deployment list | awk '/" +
 162            CONST.__getattribute__('rally_deployment_name') +
 163            "/ {print $2}'")
 164     p = subprocess.Popen(cmd, shell=True,
 165                          stdout=subprocess.PIPE,
 166                          stderr=subprocess.STDOUT)
 167     deployment_uuid = p.stdout.readline().rstrip()
 168     if deployment_uuid == "":
 169         logger.error("Rally deployment not found.")
 170         raise Exception('Error with command:%s' % cmd)
 171     return deployment_uuid
 172
 173
 174 def get_verifier_repo_dir(verifier_id):
 175     """
 176     Returns installed verfier repo directory for Tempest
 177     """
 178     if not verifier_id:
 179         verifier_id = get_verifier_id()
 180
 181     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 182                         'verification',
 183                         'verifier-{}'.format(verifier_id),
 184                         'repo')
 185
 186
 187 def get_verifier_deployment_dir(verifier_id, deployment_id):
 188     """
 189     Returns Rally deployment directory for current verifier
 190     """
 191     if not verifier_id:
 192         verifier_id = get_verifier_id()
 193
 194     if not deployment_id:
 195         deployment_id = get_verifier_deployment_id()
 196
 197     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 198                         'verification',
 199                         'verifier-{}'.format(verifier_id),
 200                         'for-deployment-{}'.format(deployment_id))
 201
 202
 203 def get_repo_tag(repo):
 204     """
 205     Returns last tag of current branch
 206     """
 207     cmd = ("git -C {0} describe --abbrev=0 HEAD".format(repo))
 208     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 209     tag = p.stdout.readline().rstrip()
 210
 211     return str(tag)
 212
 213
 214 def backup_tempest_config(conf_file):
 215     """
 216     Copy config file to tempest results directory
 217     """
 218     if not os.path.exists(TEMPEST_RESULTS_DIR):
 219         os.makedirs(TEMPEST_RESULTS_DIR)
 220
 221     shutil.copyfile(conf_file,
 222                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 223
 224
 225 def configure_tempest(deployment_dir, IMAGE_ID=None, FLAVOR_ID=None,
 226                       MODE=None):
 227     """
 228     Calls rally verify and updates the generated tempest.conf with
 229     given parameters
 230     """
 231     conf_file = configure_verifier(deployment_dir)
 232     configure_tempest_update_params(conf_file,
 233                                     IMAGE_ID, FLAVOR_ID)
 234     if MODE == 'feature_multisite':
 235         configure_tempest_multisite_params(conf_file)
 236
 237
 238 def configure_tempest_defcore(deployment_dir, img_flavor_dict):
 239     """
 240     Add/update needed parameters into tempest.conf file
 241     """
 242     conf_file = configure_verifier(deployment_dir)
 243     configure_tempest_update_params(conf_file,
 244                                     img_flavor_dict.get("image_id"),
 245                                     img_flavor_dict.get("flavor_id"))
 246
 247     logger.debug("Updating selected tempest.conf parameters for defcore...")
 248     config = ConfigParser.RawConfigParser()
 249     config.read(conf_file)
 250     config.set('DEFAULT', 'log_file', '{}/tempest.log'.format(deployment_dir))
 251     config.set('oslo_concurrency', 'lock_path',
 252                '{}/lock_files'.format(deployment_dir))
 253     config.set('scenario', 'img_dir', '{}'.format(deployment_dir))
 254     config.set('scenario', 'img_file', 'tempest-image')
 255     config.set('compute', 'image_ref', img_flavor_dict.get("image_id"))
 256     config.set('compute', 'image_ref_alt',
 257                img_flavor_dict['image_id_alt'])
 258     config.set('compute', 'flavor_ref', img_flavor_dict.get("flavor_id"))
 259     config.set('compute', 'flavor_ref_alt',
 260                img_flavor_dict['flavor_id_alt'])
 261
 262     with open(conf_file, 'wb') as config_file:
 263         config.write(config_file)
 264
 265     confpath = pkg_resources.resource_filename(
 266         'functest',
 267         'opnfv_tests/openstack/refstack_client/refstack_tempest.conf')
 268     shutil.copyfile(conf_file, confpath)
 269
 270
 271 def configure_tempest_update_params(tempest_conf_file,
 272                                     IMAGE_ID=None, FLAVOR_ID=None):
 273     """
 274     Add/update needed parameters into tempest.conf file
 275     """
 276     logger.debug("Updating selected tempest.conf parameters...")
 277     config = ConfigParser.RawConfigParser()
 278     config.read(tempest_conf_file)
 279     config.set(
 280         'compute',
 281         'fixed_network_name',
 282         CONST.__getattribute__('tempest_private_net_name'))
 283     config.set('compute', 'volume_device_name',
 284                CONST.__getattribute__('tempest_volume_device_name'))
 285     if CONST.__getattribute__('tempest_use_custom_images'):
 286         if IMAGE_ID is not None:
 287             config.set('compute', 'image_ref', IMAGE_ID)
 288         if IMAGE_ID_ALT is not None:
 289             config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 290     if CONST.__getattribute__('tempest_use_custom_flavors'):
 291         if FLAVOR_ID is not None:
 292             config.set('compute', 'flavor_ref', FLAVOR_ID)
 293         if FLAVOR_ID_ALT is not None:
 294             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 295     config.set('identity', 'tenant_name',
 296                CONST.__getattribute__('tempest_identity_tenant_name'))
 297     config.set('identity', 'username',
 298                CONST.__getattribute__('tempest_identity_user_name'))
 299     config.set('identity', 'password',
 300                CONST.__getattribute__('tempest_identity_user_password'))
 301     config.set('identity', 'region', 'RegionOne')
 302     if os_utils.is_keystone_v3():
 303         auth_version = 'v3'
 304     else:
 305         auth_version = 'v2'
 306     config.set('identity', 'auth_version', auth_version)
 307     config.set(
 308         'validation', 'ssh_timeout',
 309         CONST.__getattribute__('tempest_validation_ssh_timeout'))
 310     config.set('object-storage', 'operator_role',
 311                CONST.__getattribute__('tempest_object_storage_operator_role'))
 312
 313     if CONST.__getattribute__('OS_ENDPOINT_TYPE') is not None:
 314         sections = config.sections()
 315         if os_utils.is_keystone_v3():
 316             config.set('identity', 'v3_endpoint_type',
 317                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 318             if 'identity-feature-enabled' not in sections:
 319                 config.add_section('identity-feature-enabled')
 320                 config.set('identity-feature-enabled', 'api_v2', False)
 321                 config.set('identity-feature-enabled', 'api_v2_admin', False)
 322         services_list = ['compute',
 323                          'volume',
 324                          'image',
 325                          'network',
 326                          'data-processing',
 327                          'object-storage',
 328                          'orchestration']
 329         for service in services_list:
 330             if service not in sections:
 331                 config.add_section(service)
 332             config.set(service, 'endpoint_type',
 333                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 334
 335     logger.debug('Add/Update required params defined in tempest_conf.yaml '
 336                  'into tempest.conf file')
 337     with open(TEMPEST_CONF_YAML) as f:
 338         conf_yaml = yaml.safe_load(f)
 339     if conf_yaml:
 340         sections = config.sections()
 341         for section in conf_yaml:
 342             if section not in sections:
 343                 config.add_section(section)
 344             sub_conf = conf_yaml.get(section)
 345             for key, value in sub_conf.items():
 346                 config.set(section, key, value)
 347
 348     with open(tempest_conf_file, 'wb') as config_file:
 349         config.write(config_file)
 350
 351     backup_tempest_config(tempest_conf_file)
 352
 353
 354 def configure_verifier(deployment_dir):
 355     """
 356     Execute rally verify configure-verifier, which generates tempest.conf
 357     """
 358     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 359     if os.path.isfile(tempest_conf_file):
 360         logger.debug("Verifier is already configured.")
 361         logger.debug("Reconfiguring the current verifier...")
 362         cmd = "rally verify configure-verifier --reconfigure"
 363     else:
 364         logger.info("Configuring the verifier...")
 365         cmd = "rally verify configure-verifier"
 366     ft_utils.execute_command(cmd)
 367
 368     logger.debug("Looking for tempest.conf file...")
 369     if not os.path.isfile(tempest_conf_file):
 370         logger.error("Tempest configuration file %s NOT found."
 371                      % tempest_conf_file)
 372         raise Exception("Tempest configuration file %s NOT found."
 373                         % tempest_conf_file)
 374     else:
 375         return tempest_conf_file
 376
 377
 378 def configure_tempest_multisite_params(tempest_conf_file):
 379     """
 380     Add/update multisite parameters into tempest.conf file generated by Rally
 381     """
 382     logger.debug("Updating multisite tempest.conf parameters...")
 383     config = ConfigParser.RawConfigParser()
 384     config.read(tempest_conf_file)
 385
 386     config.set('service_available', 'kingbird', 'true')
 387     # cmd = ("openstack endpoint show kingbird | grep publicurl |"
 388     #       "awk '{print $4}' | awk -F '/' '{print $4}'")
 389     # kingbird_api_version = os.popen(cmd).read()
 390     # kingbird_api_version = os_utils.get_endpoint(service_type='multisite')
 391
 392     if CI_INSTALLER_TYPE == 'fuel':
 393         # For MOS based setup, the service is accessible
 394         # via bind host
 395         kingbird_conf_path = "/etc/kingbird/kingbird.conf"
 396         installer_type = CI_INSTALLER_TYPE
 397         installer_ip = CI_INSTALLER_IP
 398         installer_username = CONST.__getattribute__(
 399             'multisite_{}_installer_username'.format(installer_type))
 400         installer_password = CONST.__getattribute__(
 401             'multisite_{}_installer_password'.format(installer_type))
 402
 403         ssh_options = ("-o UserKnownHostsFile=/dev/null -o "
 404                        "StrictHostKeyChecking=no")
 405
 406         # Get the controller IP from the fuel node
 407         cmd = ('sshpass -p %s ssh 2>/dev/null %s %s@%s '
 408                '\'fuel node --env 1| grep controller | grep "True\|  1" '
 409                '| awk -F\| "{print \$5}"\'' % (installer_password,
 410                                                ssh_options,
 411                                                installer_username,
 412                                                installer_ip))
 413         multisite_controller_ip = "".join(os.popen(cmd).read().split())
 414
 415         # Login to controller and get bind host details
 416         cmd = ('sshpass -p %s ssh 2>/dev/null  %s %s@%s "ssh %s \\" '
 417                'grep -e "^bind_" %s  \\""' % (installer_password,
 418                                               ssh_options,
 419                                               installer_username,
 420                                               installer_ip,
 421                                               multisite_controller_ip,
 422                                               kingbird_conf_path))
 423         bind_details = os.popen(cmd).read()
 424         bind_details = "".join(bind_details.split())
 425         # Extract port number from the bind details
 426         bind_port = re.findall(r"\D(\d{4})", bind_details)[0]
 427         # Extract ip address from the bind details
 428         bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
 429                                bind_details)[0]
 430         kingbird_endpoint_url = "http://%s:%s/" % (bind_host, bind_port)
 431     else:
 432         # cmd = "openstack endpoint show kingbird | grep publicurl |\
 433         #       awk '{print $4}' | awk -F '/' '{print $3}'"
 434         # kingbird_endpoint_url = os.popen(cmd).read()
 435         kingbird_endpoint_url = os_utils.get_endpoint(service_type='kingbird')
 436
 437     try:
 438         config.add_section("kingbird")
 439     except Exception:
 440         logger.info('kingbird section exist')
 441
 442     # set the domain id
 443     config.set('auth', 'admin_domain_name', 'default')
 444
 445     config.set('kingbird', 'endpoint_type', 'publicURL')
 446     config.set('kingbird', 'TIME_TO_SYNC', '120')
 447     config.set('kingbird', 'endpoint_url', kingbird_endpoint_url)
 448     config.set('kingbird', 'api_version', 'v1.0')
 449     with open(tempest_conf_file, 'wb') as config_file:
 450         config.write(config_file)
 451
 452     backup_tempest_config(tempest_conf_file)
 453
 454
 455 def install_verifier_ext(path):
 456     """
 457     Install extension to active verifier
 458     """
 459     logger.info("Installing verifier from existing repo...")
 460     tag = get_repo_tag(path)
 461     cmd = ("rally verify add-verifier-ext --source {0} "
 462            "--version {1}"
 463            .format(path, tag))
 464     error_msg = ("Problem while adding verifier extension from %s" % path)
 465     ft_utils.execute_command_raise(cmd, error_msg=error_msg)