functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 import ConfigParser
  11 import logging
  12 import os
  13 import pkg_resources
  14 import re
  15 import shutil
  16 import subprocess
  17
  18 from functest.utils.constants import CONST
  19 import functest.utils.functest_utils as ft_utils
  20 import functest.utils.openstack_utils as os_utils
  21
  22
  23 IMAGE_ID_ALT = None
  24 FLAVOR_ID_ALT = None
  25 GLANCE_IMAGE_PATH = os.path.join(
  26     CONST.__getattribute__('dir_functest_images'),
  27     CONST.__getattribute__('openstack_image_file_name'))
  28 TEMPEST_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  29                                    'tempest')
  30 TEMPEST_CUSTOM = pkg_resources.resource_filename(
  31     'functest', 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  32 TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  33     'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.txt')
  34 TEMPEST_DEFCORE = pkg_resources.resource_filename(
  35     'functest',
  36     'opnfv_tests/openstack/tempest/custom_tests/defcore_req.txt')
  37 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  38 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  39 REFSTACK_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  40                                     'refstack')
  41
  42 CI_INSTALLER_TYPE = CONST.__getattribute__('INSTALLER_TYPE')
  43 CI_INSTALLER_IP = CONST.__getattribute__('INSTALLER_IP')
  44
  45 """ logging configuration """
  46 logger = logging.getLogger(__name__)
  47
  48
  49 def create_tempest_resources(use_custom_images=False,
  50                              use_custom_flavors=False):
  51     keystone_client = os_utils.get_keystone_client()
  52
  53     logger.debug("Creating tenant and user for Tempest suite")
  54     tenant_id = os_utils.create_tenant(
  55         keystone_client,
  56         CONST.__getattribute__('tempest_identity_tenant_name'),
  57         CONST.__getattribute__('tempest_identity_tenant_description'))
  58     if not tenant_id:
  59         logger.error("Failed to create %s tenant"
  60                      % CONST.__getattribute__('tempest_identity_tenant_name'))
  61
  62     user_id = os_utils.create_user(
  63         keystone_client,
  64         CONST.__getattribute__('tempest_identity_user_name'),
  65         CONST.__getattribute__('tempest_identity_user_password'),
  66         None, tenant_id)
  67     if not user_id:
  68         logger.error("Failed to create %s user" %
  69                      CONST.__getattribute__('tempest_identity_user_name'))
  70
  71     logger.debug("Creating private network for Tempest suite")
  72     network_dic = os_utils.create_shared_network_full(
  73         CONST.__getattribute__('tempest_private_net_name'),
  74         CONST.__getattribute__('tempest_private_subnet_name'),
  75         CONST.__getattribute__('tempest_router_name'),
  76         CONST.__getattribute__('tempest_private_subnet_cidr'))
  77     if network_dic is None:
  78         raise Exception('Failed to create private network')
  79
  80     image_id = ""
  81     image_id_alt = ""
  82     flavor_id = ""
  83     flavor_id_alt = ""
  84
  85     if (CONST.__getattribute__('tempest_use_custom_images') or
  86        use_custom_images):
  87         # adding alternative image should be trivial should we need it
  88         logger.debug("Creating image for Tempest suite")
  89         _, image_id = os_utils.get_or_create_image(
  90             CONST.__getattribute__('openstack_image_name'),
  91             GLANCE_IMAGE_PATH,
  92             CONST.__getattribute__('openstack_image_disk_format'))
  93         if image_id is None:
  94             raise Exception('Failed to create image')
  95
  96     if use_custom_images:
  97         logger.debug("Creating 2nd image for Tempest suite")
  98         _, image_id_alt = os_utils.get_or_create_image(
  99             CONST.__getattribute__('openstack_image_name_alt'),
 100             GLANCE_IMAGE_PATH,
 101             CONST.__getattribute__('openstack_image_disk_format'))
 102         if image_id_alt is None:
 103             raise Exception('Failed to create image')
 104
 105     if (CONST.__getattribute__('tempest_use_custom_flavors') or
 106        use_custom_flavors):
 107         # adding alternative flavor should be trivial should we need it
 108         logger.debug("Creating flavor for Tempest suite")
 109         _, flavor_id = os_utils.get_or_create_flavor(
 110             CONST.__getattribute__('openstack_flavor_name'),
 111             CONST.__getattribute__('openstack_flavor_ram'),
 112             CONST.__getattribute__('openstack_flavor_disk'),
 113             CONST.__getattribute__('openstack_flavor_vcpus'))
 114         if flavor_id is None:
 115             raise Exception('Failed to create flavor')
 116
 117     if use_custom_flavors:
 118         logger.debug("Creating 2nd flavor for tempest_defcore")
 119         _, flavor_id_alt = os_utils.get_or_create_flavor(
 120             CONST.__getattribute__('openstack_flavor_name_alt'),
 121             CONST.__getattribute__('openstack_flavor_ram'),
 122             CONST.__getattribute__('openstack_flavor_disk'),
 123             CONST.__getattribute__('openstack_flavor_vcpus'))
 124         if flavor_id_alt is None:
 125             raise Exception('Failed to create flavor')
 126
 127     img_flavor_dict = {}
 128     img_flavor_dict['image_id'] = image_id
 129     img_flavor_dict['image_id_alt'] = image_id_alt
 130     img_flavor_dict['flavor_id'] = flavor_id
 131     img_flavor_dict['flavor_id_alt'] = flavor_id_alt
 132
 133     return img_flavor_dict
 134
 135
 136 def get_verifier_id():
 137     """
 138     Returns verifer id for current Tempest
 139     """
 140     cmd = ("rally verify list-verifiers | awk '/" +
 141            CONST.__getattribute__('tempest_deployment_name') +
 142            "/ {print $2}'")
 143     p = subprocess.Popen(cmd, shell=True,
 144                          stdout=subprocess.PIPE,
 145                          stderr=subprocess.STDOUT)
 146     deployment_uuid = p.stdout.readline().rstrip()
 147     if deployment_uuid == "":
 148         logger.error("Tempest verifier not found.")
 149         raise Exception('Error with command:%s' % cmd)
 150     return deployment_uuid
 151
 152
 153 def get_verifier_deployment_id():
 154     """
 155     Returns deployment id for active Rally deployment
 156     """
 157     cmd = ("rally deployment list | awk '/" +
 158            CONST.__getattribute__('rally_deployment_name') +
 159            "/ {print $2}'")
 160     p = subprocess.Popen(cmd, shell=True,
 161                          stdout=subprocess.PIPE,
 162                          stderr=subprocess.STDOUT)
 163     deployment_uuid = p.stdout.readline().rstrip()
 164     if deployment_uuid == "":
 165         logger.error("Rally deployment not found.")
 166         raise Exception('Error with command:%s' % cmd)
 167     return deployment_uuid
 168
 169
 170 def get_verifier_repo_dir(verifier_id):
 171     """
 172     Returns installed verfier repo directory for Tempest
 173     """
 174     if not verifier_id:
 175         verifier_id = get_verifier_id()
 176
 177     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 178                         'verification',
 179                         'verifier-{}'.format(verifier_id),
 180                         'repo')
 181
 182
 183 def get_verifier_deployment_dir(verifier_id, deployment_id):
 184     """
 185     Returns Rally deployment directory for current verifier
 186     """
 187     if not verifier_id:
 188         verifier_id = get_verifier_id()
 189
 190     if not deployment_id:
 191         deployment_id = get_verifier_deployment_id()
 192
 193     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 194                         'verification',
 195                         'verifier-{}'.format(verifier_id),
 196                         'for-deployment-{}'.format(deployment_id))
 197
 198
 199 def get_repo_tag(repo):
 200     """
 201     Returns last tag of current branch
 202     """
 203     cmd = ("git -C {0} describe --abbrev=0 HEAD".format(repo))
 204     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 205     tag = p.stdout.readline().rstrip()
 206
 207     return str(tag)
 208
 209
 210 def backup_tempest_config(conf_file):
 211     """
 212     Copy config file to tempest results directory
 213     """
 214     if not os.path.exists(TEMPEST_RESULTS_DIR):
 215         os.makedirs(TEMPEST_RESULTS_DIR)
 216
 217     shutil.copyfile(conf_file,
 218                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 219
 220
 221 def configure_tempest(deployment_dir, IMAGE_ID=None, FLAVOR_ID=None,
 222                       MODE=None):
 223     """
 224     Calls rally verify and updates the generated tempest.conf with
 225     given parameters
 226     """
 227     conf_file = configure_verifier(deployment_dir)
 228     configure_tempest_update_params(conf_file,
 229                                     IMAGE_ID, FLAVOR_ID)
 230     if MODE == 'feature_multisite':
 231         configure_tempest_multisite_params(conf_file)
 232
 233
 234 def configure_tempest_defcore(deployment_dir, img_flavor_dict):
 235     """
 236     Add/update needed parameters into tempest.conf file
 237     """
 238     conf_file = configure_verifier(deployment_dir)
 239     configure_tempest_update_params(conf_file,
 240                                     img_flavor_dict.get("image_id"),
 241                                     img_flavor_dict.get("flavor_id"))
 242
 243     logger.debug("Updating selected tempest.conf parameters for defcore...")
 244     config = ConfigParser.RawConfigParser()
 245     config.read(conf_file)
 246     config.set('DEFAULT', 'log_file', '{}/tempest.log'.format(deployment_dir))
 247     config.set('oslo_concurrency', 'lock_path',
 248                '{}/lock_files'.format(deployment_dir))
 249     config.set('scenario', 'img_dir', '{}'.format(deployment_dir))
 250     config.set('scenario', 'img_file', 'tempest-image')
 251     config.set('compute', 'image_ref', img_flavor_dict.get("image_id"))
 252     config.set('compute', 'image_ref_alt',
 253                img_flavor_dict['image_id_alt'])
 254     config.set('compute', 'flavor_ref', img_flavor_dict.get("flavor_id"))
 255     config.set('compute', 'flavor_ref_alt',
 256                img_flavor_dict['flavor_id_alt'])
 257
 258     with open(conf_file, 'wb') as config_file:
 259         config.write(config_file)
 260
 261     confpath = pkg_resources.resource_filename(
 262         'functest',
 263         'opnfv_tests/openstack/refstack_client/refstack_tempest.conf')
 264     shutil.copyfile(conf_file, confpath)
 265
 266
 267 def configure_tempest_update_params(tempest_conf_file,
 268                                     IMAGE_ID=None, FLAVOR_ID=None):
 269     """
 270     Add/update needed parameters into tempest.conf file
 271     """
 272     logger.debug("Updating selected tempest.conf parameters...")
 273     config = ConfigParser.RawConfigParser()
 274     config.read(tempest_conf_file)
 275     config.set(
 276         'compute',
 277         'fixed_network_name',
 278         CONST.__getattribute__('tempest_private_net_name'))
 279     config.set('compute', 'volume_device_name',
 280                CONST.__getattribute__('tempest_volume_device_name'))
 281     if CONST.__getattribute__('tempest_use_custom_images'):
 282         if IMAGE_ID is not None:
 283             config.set('compute', 'image_ref', IMAGE_ID)
 284         if IMAGE_ID_ALT is not None:
 285             config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 286     if CONST.__getattribute__('tempest_use_custom_flavors'):
 287         if FLAVOR_ID is not None:
 288             config.set('compute', 'flavor_ref', FLAVOR_ID)
 289         if FLAVOR_ID_ALT is not None:
 290             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 291     config.set('identity', 'tenant_name',
 292                CONST.__getattribute__('tempest_identity_tenant_name'))
 293     config.set('identity', 'username',
 294                CONST.__getattribute__('tempest_identity_user_name'))
 295     config.set('identity', 'password',
 296                CONST.__getattribute__('tempest_identity_user_password'))
 297     config.set('identity', 'region', 'RegionOne')
 298     if os_utils.is_keystone_v3():
 299         auth_version = 'v3'
 300     else:
 301         auth_version = 'v2'
 302     config.set('identity', 'auth_version', auth_version)
 303     config.set(
 304         'validation', 'ssh_timeout',
 305         CONST.__getattribute__('tempest_validation_ssh_timeout'))
 306     config.set('object-storage', 'operator_role',
 307                CONST.__getattribute__('tempest_object_storage_operator_role'))
 308
 309     if CONST.__getattribute__('OS_ENDPOINT_TYPE') is not None:
 310         sections = config.sections()
 311         if os_utils.is_keystone_v3():
 312             config.set('identity', 'v3_endpoint_type',
 313                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 314             if 'identity-feature-enabled' not in sections:
 315                 config.add_section('identity-feature-enabled')
 316                 config.set('identity-feature-enabled', 'api_v2', False)
 317                 config.set('identity-feature-enabled', 'api_v2_admin', False)
 318         services_list = ['compute',
 319                          'volume',
 320                          'image',
 321                          'network',
 322                          'data-processing',
 323                          'object-storage',
 324                          'orchestration']
 325         for service in services_list:
 326             if service not in sections:
 327                 config.add_section(service)
 328             config.set(service, 'endpoint_type',
 329                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 330
 331     with open(tempest_conf_file, 'wb') as config_file:
 332         config.write(config_file)
 333
 334     backup_tempest_config(tempest_conf_file)
 335
 336
 337 def configure_verifier(deployment_dir):
 338     """
 339     Execute rally verify configure-verifier, which generates tempest.conf
 340     """
 341     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 342     if os.path.isfile(tempest_conf_file):
 343         logger.debug("Verifier is already configured.")
 344         logger.debug("Reconfiguring the current verifier...")
 345         cmd = "rally verify configure-verifier --reconfigure"
 346     else:
 347         logger.info("Configuring the verifier...")
 348         cmd = "rally verify configure-verifier"
 349     ft_utils.execute_command(cmd)
 350
 351     logger.debug("Looking for tempest.conf file...")
 352     if not os.path.isfile(tempest_conf_file):
 353         logger.error("Tempest configuration file %s NOT found."
 354                      % tempest_conf_file)
 355         raise Exception("Tempest configuration file %s NOT found."
 356                         % tempest_conf_file)
 357     else:
 358         return tempest_conf_file
 359
 360
 361 def configure_tempest_multisite_params(tempest_conf_file):
 362     """
 363     Add/update multisite parameters into tempest.conf file generated by Rally
 364     """
 365     logger.debug("Updating multisite tempest.conf parameters...")
 366     config = ConfigParser.RawConfigParser()
 367     config.read(tempest_conf_file)
 368
 369     config.set('service_available', 'kingbird', 'true')
 370     # cmd = ("openstack endpoint show kingbird | grep publicurl |"
 371     #       "awk '{print $4}' | awk -F '/' '{print $4}'")
 372     # kingbird_api_version = os.popen(cmd).read()
 373     # kingbird_api_version = os_utils.get_endpoint(service_type='multisite')
 374
 375     if CI_INSTALLER_TYPE == 'fuel':
 376         # For MOS based setup, the service is accessible
 377         # via bind host
 378         kingbird_conf_path = "/etc/kingbird/kingbird.conf"
 379         installer_type = CI_INSTALLER_TYPE
 380         installer_ip = CI_INSTALLER_IP
 381         installer_username = CONST.__getattribute__(
 382             'multisite_{}_installer_username'.format(installer_type))
 383         installer_password = CONST.__getattribute__(
 384             'multisite_{}_installer_password'.format(installer_type))
 385
 386         ssh_options = ("-o UserKnownHostsFile=/dev/null -o "
 387                        "StrictHostKeyChecking=no")
 388
 389         # Get the controller IP from the fuel node
 390         cmd = ('sshpass -p %s ssh 2>/dev/null %s %s@%s '
 391                '\'fuel node --env 1| grep controller | grep "True\|  1" '
 392                '| awk -F\| "{print \$5}"\'' % (installer_password,
 393                                                ssh_options,
 394                                                installer_username,
 395                                                installer_ip))
 396         multisite_controller_ip = "".join(os.popen(cmd).read().split())
 397
 398         # Login to controller and get bind host details
 399         cmd = ('sshpass -p %s ssh 2>/dev/null  %s %s@%s "ssh %s \\" '
 400                'grep -e "^bind_" %s  \\""' % (installer_password,
 401                                               ssh_options,
 402                                               installer_username,
 403                                               installer_ip,
 404                                               multisite_controller_ip,
 405                                               kingbird_conf_path))
 406         bind_details = os.popen(cmd).read()
 407         bind_details = "".join(bind_details.split())
 408         # Extract port number from the bind details
 409         bind_port = re.findall(r"\D(\d{4})", bind_details)[0]
 410         # Extract ip address from the bind details
 411         bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
 412                                bind_details)[0]
 413         kingbird_endpoint_url = "http://%s:%s/" % (bind_host, bind_port)
 414     else:
 415         # cmd = "openstack endpoint show kingbird | grep publicurl |\
 416         #       awk '{print $4}' | awk -F '/' '{print $3}'"
 417         # kingbird_endpoint_url = os.popen(cmd).read()
 418         kingbird_endpoint_url = os_utils.get_endpoint(service_type='kingbird')
 419
 420     try:
 421         config.add_section("kingbird")
 422     except Exception:
 423         logger.info('kingbird section exist')
 424
 425     # set the domain id
 426     config.set('auth', 'admin_domain_name', 'default')
 427
 428     config.set('kingbird', 'endpoint_type', 'publicURL')
 429     config.set('kingbird', 'TIME_TO_SYNC', '120')
 430     config.set('kingbird', 'endpoint_url', kingbird_endpoint_url)
 431     config.set('kingbird', 'api_version', 'v1.0')
 432     with open(tempest_conf_file, 'wb') as config_file:
 433         config.write(config_file)
 434
 435     backup_tempest_config(tempest_conf_file)
 436
 437
 438 def install_verifier_ext(path):
 439     """
 440     Install extension to active verifier
 441     """
 442     logger.info("Installing verifier from existing repo...")
 443     tag = get_repo_tag(path)
 444     cmd = ("rally verify add-verifier-ext --source {0} "
 445            "--version {1}"
 446            .format(path, tag))
 447     error_msg = ("Problem while adding verifier extension from %s" % path)
 448     ft_utils.execute_command_raise(cmd, error_msg=error_msg)