functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 import ConfigParser
  11 import logging
  12 import os
  13 import re
  14 import shutil
  15 import subprocess
  16
  17 from functest.utils.constants import CONST
  18 import functest.utils.functest_utils as ft_utils
  19 import functest.utils.openstack_utils as os_utils
  20
  21
  22 IMAGE_ID_ALT = None
  23 FLAVOR_ID_ALT = None
  24 REPO_PATH = CONST.__getattribute__('dir_repo_functest')
  25 GLANCE_IMAGE_PATH = os.path.join(
  26     CONST.__getattribute__('dir_functest_images'),
  27     CONST.__getattribute__('openstack_image_file_name'))
  28 TEMPEST_TEST_LIST_DIR = CONST.__getattribute__('dir_tempest_cases')
  29 TEMPEST_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  30                                    'tempest')
  31 TEMPEST_CUSTOM = os.path.join(REPO_PATH, TEMPEST_TEST_LIST_DIR,
  32                               'test_list.txt')
  33 TEMPEST_BLACKLIST = os.path.join(REPO_PATH, TEMPEST_TEST_LIST_DIR,
  34                                  'blacklist.txt')
  35 TEMPEST_DEFCORE = os.path.join(REPO_PATH, TEMPEST_TEST_LIST_DIR,
  36                                'defcore_req.txt')
  37 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  38 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  39 REFSTACK_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  40                                     'refstack')
  41
  42 CI_INSTALLER_TYPE = CONST.__getattribute__('INSTALLER_TYPE')
  43 CI_INSTALLER_IP = CONST.__getattribute__('INSTALLER_IP')
  44
  45 """ logging configuration """
  46 logger = logging.getLogger(__name__)
  47
  48
  49 def create_tempest_resources(use_custom_images=False,
  50                              use_custom_flavors=False):
  51     keystone_client = os_utils.get_keystone_client()
  52
  53     logger.debug("Creating tenant and user for Tempest suite")
  54     tenant_id = os_utils.create_tenant(
  55         keystone_client,
  56         CONST.__getattribute__('tempest_identity_tenant_name'),
  57         CONST.__getattribute__('tempest_identity_tenant_description'))
  58     if not tenant_id:
  59         logger.error("Failed to create %s tenant"
  60                      % CONST.__getattribute__('tempest_identity_tenant_name'))
  61
  62     user_id = os_utils.create_user(
  63         keystone_client,
  64         CONST.__getattribute__('tempest_identity_user_name'),
  65         CONST.__getattribute__('tempest_identity_user_password'),
  66         None, tenant_id)
  67     if not user_id:
  68         logger.error("Failed to create %s user" %
  69                      CONST.__getattribute__('tempest_identity_user_name'))
  70
  71     logger.debug("Creating private network for Tempest suite")
  72     network_dic = os_utils.create_shared_network_full(
  73         CONST.__getattribute__('tempest_private_net_name'),
  74         CONST.__getattribute__('tempest_private_subnet_name'),
  75         CONST.__getattribute__('tempest_router_name'),
  76         CONST.__getattribute__('tempest_private_subnet_cidr'))
  77     if network_dic is None:
  78         raise Exception('Failed to create private network')
  79
  80     image_id = ""
  81     image_id_alt = ""
  82     flavor_id = ""
  83     flavor_id_alt = ""
  84
  85     if (CONST.__getattribute__('tempest_use_custom_images') or
  86        use_custom_images):
  87         # adding alternative image should be trivial should we need it
  88         logger.debug("Creating image for Tempest suite")
  89         _, image_id = os_utils.get_or_create_image(
  90             CONST.__getattribute__('openstack_image_name'),
  91             GLANCE_IMAGE_PATH,
  92             CONST.__getattribute__('openstack_image_disk_format'))
  93         if image_id is None:
  94             raise Exception('Failed to create image')
  95
  96     if use_custom_images:
  97         logger.debug("Creating 2nd image for Tempest suite")
  98         _, image_id_alt = os_utils.get_or_create_image(
  99             CONST.__getattribute__('openstack_image_name_alt'),
 100             GLANCE_IMAGE_PATH,
 101             CONST.__getattribute__('openstack_image_disk_format'))
 102         if image_id_alt is None:
 103             raise Exception('Failed to create image')
 104
 105     if (CONST.__getattribute__('tempest_use_custom_flavors') or
 106        use_custom_flavors):
 107         # adding alternative flavor should be trivial should we need it
 108         logger.debug("Creating flavor for Tempest suite")
 109         _, flavor_id = os_utils.get_or_create_flavor(
 110             CONST.__getattribute__('openstack_flavor_name'),
 111             CONST.__getattribute__('openstack_flavor_ram'),
 112             CONST.__getattribute__('openstack_flavor_disk'),
 113             CONST.__getattribute__('openstack_flavor_vcpus'))
 114         if flavor_id is None:
 115             raise Exception('Failed to create flavor')
 116
 117     if use_custom_flavors:
 118         logger.debug("Creating 2nd flavor for tempest_defcore")
 119         _, flavor_id_alt = os_utils.get_or_create_flavor(
 120             CONST.__getattribute__('openstack_flavor_name_alt'),
 121             CONST.__getattribute__('openstack_flavor_ram'),
 122             CONST.__getattribute__('openstack_flavor_disk'),
 123             CONST.__getattribute__('openstack_flavor_vcpus'))
 124         if flavor_id_alt is None:
 125             raise Exception('Failed to create flavor')
 126
 127     img_flavor_dict = {}
 128     img_flavor_dict['image_id'] = image_id
 129     img_flavor_dict['image_id_alt'] = image_id_alt
 130     img_flavor_dict['flavor_id'] = flavor_id
 131     img_flavor_dict['flavor_id_alt'] = flavor_id_alt
 132
 133     return img_flavor_dict
 134
 135
 136 def get_verifier_id():
 137     """
 138     Returns verifer id for current Tempest
 139     """
 140     cmd = ("rally verify list-verifiers | awk '/" +
 141            CONST.__getattribute__('tempest_deployment_name') +
 142            "/ {print $2}'")
 143     p = subprocess.Popen(cmd, shell=True,
 144                          stdout=subprocess.PIPE,
 145                          stderr=subprocess.STDOUT)
 146     deployment_uuid = p.stdout.readline().rstrip()
 147     if deployment_uuid == "":
 148         logger.error("Tempest verifier not found.")
 149         raise Exception('Error with command:%s' % cmd)
 150     return deployment_uuid
 151
 152
 153 def get_verifier_deployment_id():
 154     """
 155     Returns deployment id for active Rally deployment
 156     """
 157     cmd = ("rally deployment list | awk '/" +
 158            CONST.__getattribute__('rally_deployment_name') +
 159            "/ {print $2}'")
 160     p = subprocess.Popen(cmd, shell=True,
 161                          stdout=subprocess.PIPE,
 162                          stderr=subprocess.STDOUT)
 163     deployment_uuid = p.stdout.readline().rstrip()
 164     if deployment_uuid == "":
 165         logger.error("Rally deployment not found.")
 166         raise Exception('Error with command:%s' % cmd)
 167     return deployment_uuid
 168
 169
 170 def get_verifier_repo_dir(verifier_id):
 171     """
 172     Returns installed verfier repo directory for Tempest
 173     """
 174     if not verifier_id:
 175         verifier_id = get_verifier_id()
 176
 177     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 178                         'verification',
 179                         'verifier-{}'.format(verifier_id),
 180                         'repo')
 181
 182
 183 def get_verifier_deployment_dir(verifier_id, deployment_id):
 184     """
 185     Returns Rally deployment directory for current verifier
 186     """
 187     if not verifier_id:
 188         verifier_id = get_verifier_id()
 189
 190     if not deployment_id:
 191         deployment_id = get_verifier_deployment_id()
 192
 193     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 194                         'verification',
 195                         'verifier-{}'.format(verifier_id),
 196                         'for-deployment-{}'.format(deployment_id))
 197
 198
 199 def get_repo_tag(repo):
 200     """
 201     Returns last tag of current branch
 202     """
 203     cmd = ("git -C {0} describe --abbrev=0 HEAD".format(repo))
 204     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 205     tag = p.stdout.readline().rstrip()
 206
 207     return str(tag)
 208
 209
 210 def backup_tempest_config(conf_file):
 211     """
 212     Copy config file to tempest results directory
 213     """
 214     if not os.path.exists(TEMPEST_RESULTS_DIR):
 215         os.makedirs(TEMPEST_RESULTS_DIR)
 216
 217     shutil.copyfile(conf_file,
 218                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 219
 220
 221 def configure_tempest(deployment_dir, IMAGE_ID=None, FLAVOR_ID=None,
 222                       MODE=None):
 223     """
 224     Calls rally verify and updates the generated tempest.conf with
 225     given parameters
 226     """
 227     conf_file = configure_verifier(deployment_dir)
 228     configure_tempest_update_params(conf_file,
 229                                     IMAGE_ID, FLAVOR_ID)
 230     if MODE == 'feature_multisite':
 231         configure_tempest_multisite_params(conf_file)
 232
 233
 234 def configure_tempest_defcore(deployment_dir, img_flavor_dict):
 235     """
 236     Add/update needed parameters into tempest.conf file
 237     """
 238     conf_file = configure_verifier(deployment_dir)
 239     configure_tempest_update_params(conf_file,
 240                                     img_flavor_dict.get("image_id"),
 241                                     img_flavor_dict.get("flavor_id"))
 242
 243     logger.debug("Updating selected tempest.conf parameters for defcore...")
 244     config = ConfigParser.RawConfigParser()
 245     config.read(conf_file)
 246     config.set('compute', 'image_ref', img_flavor_dict.get("image_id"))
 247     config.set('compute', 'image_ref_alt',
 248                img_flavor_dict['image_id_alt'])
 249     config.set('compute', 'flavor_ref', img_flavor_dict.get("flavor_id"))
 250     config.set('compute', 'flavor_ref_alt',
 251                img_flavor_dict['flavor_id_alt'])
 252
 253     with open(conf_file, 'wb') as config_file:
 254         config.write(config_file)
 255
 256     confpath = os.path.join(
 257         CONST.__getattribute__('dir_functest_test'),
 258         CONST.__getattribute__('refstack_tempest_conf_path'))
 259     shutil.copyfile(conf_file, confpath)
 260
 261
 262 def configure_tempest_update_params(tempest_conf_file,
 263                                     IMAGE_ID=None, FLAVOR_ID=None):
 264     """
 265     Add/update needed parameters into tempest.conf file
 266     """
 267     logger.debug("Updating selected tempest.conf parameters...")
 268     config = ConfigParser.RawConfigParser()
 269     config.read(tempest_conf_file)
 270     config.set(
 271         'compute',
 272         'fixed_network_name',
 273         CONST.__getattribute__('tempest_private_net_name'))
 274     config.set('compute', 'volume_device_name',
 275                CONST.__getattribute__('tempest_volume_device_name'))
 276     if CONST.__getattribute__('tempest_use_custom_images'):
 277         if IMAGE_ID is not None:
 278             config.set('compute', 'image_ref', IMAGE_ID)
 279         if IMAGE_ID_ALT is not None:
 280             config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 281     if CONST.__getattribute__('tempest_use_custom_flavors'):
 282         if FLAVOR_ID is not None:
 283             config.set('compute', 'flavor_ref', FLAVOR_ID)
 284         if FLAVOR_ID_ALT is not None:
 285             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 286     config.set('identity', 'tenant_name',
 287                CONST.__getattribute__('tempest_identity_tenant_name'))
 288     config.set('identity', 'username',
 289                CONST.__getattribute__('tempest_identity_user_name'))
 290     config.set('identity', 'password',
 291                CONST.__getattribute__('tempest_identity_user_password'))
 292     config.set('identity', 'region', 'RegionOne')
 293     config.set(
 294         'validation', 'ssh_timeout',
 295         CONST.__getattribute__('tempest_validation_ssh_timeout'))
 296     config.set('object-storage', 'operator_role',
 297                CONST.__getattribute__('tempest_object_storage_operator_role'))
 298
 299     if CONST.__getattribute__('OS_ENDPOINT_TYPE') is not None:
 300         sections = config.sections()
 301         if os_utils.is_keystone_v3():
 302             config.set('identity', 'v3_endpoint_type',
 303                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 304             if 'identity-feature-enabled' not in sections:
 305                 config.add_section('identity-feature-enabled')
 306                 config.set('identity-feature-enabled', 'api_v2', False)
 307                 config.set('identity-feature-enabled', 'api_v2_admin', False)
 308         services_list = ['compute',
 309                          'volume',
 310                          'image',
 311                          'network',
 312                          'data-processing',
 313                          'object-storage',
 314                          'orchestration']
 315         for service in services_list:
 316             if service not in sections:
 317                 config.add_section(service)
 318             config.set(service, 'endpoint_type',
 319                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 320
 321     with open(tempest_conf_file, 'wb') as config_file:
 322         config.write(config_file)
 323
 324     backup_tempest_config(tempest_conf_file)
 325
 326
 327 def configure_verifier(deployment_dir):
 328     """
 329     Execute rally verify configure-verifier, which generates tempest.conf
 330     """
 331     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 332     if os.path.isfile(tempest_conf_file):
 333         logger.debug("Verifier is already configured.")
 334         logger.debug("Reconfiguring the current verifier...")
 335         cmd = "rally verify configure-verifier --reconfigure"
 336     else:
 337         logger.info("Configuring the verifier...")
 338         cmd = "rally verify configure-verifier"
 339     ft_utils.execute_command(cmd)
 340
 341     logger.debug("Looking for tempest.conf file...")
 342     if not os.path.isfile(tempest_conf_file):
 343         logger.error("Tempest configuration file %s NOT found."
 344                      % tempest_conf_file)
 345         raise Exception("Tempest configuration file %s NOT found."
 346                         % tempest_conf_file)
 347     else:
 348         return tempest_conf_file
 349
 350
 351 def configure_tempest_multisite_params(tempest_conf_file):
 352     """
 353     Add/update multisite parameters into tempest.conf file generated by Rally
 354     """
 355     logger.debug("Updating multisite tempest.conf parameters...")
 356     config = ConfigParser.RawConfigParser()
 357     config.read(tempest_conf_file)
 358
 359     config.set('service_available', 'kingbird', 'true')
 360     # cmd = ("openstack endpoint show kingbird | grep publicurl |"
 361     #       "awk '{print $4}' | awk -F '/' '{print $4}'")
 362     # kingbird_api_version = os.popen(cmd).read()
 363     # kingbird_api_version = os_utils.get_endpoint(service_type='multisite')
 364
 365     if CI_INSTALLER_TYPE == 'fuel':
 366         # For MOS based setup, the service is accessible
 367         # via bind host
 368         kingbird_conf_path = "/etc/kingbird/kingbird.conf"
 369         installer_type = CI_INSTALLER_TYPE
 370         installer_ip = CI_INSTALLER_IP
 371         installer_username = CONST.__getattribute__(
 372             'multisite_{}_installer_username'.format(installer_type))
 373         installer_password = CONST.__getattribute__(
 374             'multisite_{}_installer_password'.format(installer_type))
 375
 376         ssh_options = ("-o UserKnownHostsFile=/dev/null -o "
 377                        "StrictHostKeyChecking=no")
 378
 379         # Get the controller IP from the fuel node
 380         cmd = ('sshpass -p %s ssh 2>/dev/null %s %s@%s '
 381                '\'fuel node --env 1| grep controller | grep "True\|  1" '
 382                '| awk -F\| "{print \$5}"\'' % (installer_password,
 383                                                ssh_options,
 384                                                installer_username,
 385                                                installer_ip))
 386         multisite_controller_ip = "".join(os.popen(cmd).read().split())
 387
 388         # Login to controller and get bind host details
 389         cmd = ('sshpass -p %s ssh 2>/dev/null  %s %s@%s "ssh %s \\" '
 390                'grep -e "^bind_" %s  \\""' % (installer_password,
 391                                               ssh_options,
 392                                               installer_username,
 393                                               installer_ip,
 394                                               multisite_controller_ip,
 395                                               kingbird_conf_path))
 396         bind_details = os.popen(cmd).read()
 397         bind_details = "".join(bind_details.split())
 398         # Extract port number from the bind details
 399         bind_port = re.findall(r"\D(\d{4})", bind_details)[0]
 400         # Extract ip address from the bind details
 401         bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",
 402                                bind_details)[0]
 403         kingbird_endpoint_url = "http://%s:%s/" % (bind_host, bind_port)
 404     else:
 405         # cmd = "openstack endpoint show kingbird | grep publicurl |\
 406         #       awk '{print $4}' | awk -F '/' '{print $3}'"
 407         # kingbird_endpoint_url = os.popen(cmd).read()
 408         kingbird_endpoint_url = os_utils.get_endpoint(service_type='kingbird')
 409
 410     try:
 411         config.add_section("kingbird")
 412     except Exception:
 413         logger.info('kingbird section exist')
 414
 415     # set the domain id
 416     config.set('auth', 'admin_domain_name', 'default')
 417
 418     config.set('kingbird', 'endpoint_type', 'publicURL')
 419     config.set('kingbird', 'TIME_TO_SYNC', '120')
 420     config.set('kingbird', 'endpoint_url', kingbird_endpoint_url)
 421     config.set('kingbird', 'api_version', 'v1.0')
 422     with open(tempest_conf_file, 'wb') as config_file:
 423         config.write(config_file)
 424
 425     backup_tempest_config(tempest_conf_file)
 426
 427
 428 def install_verifier_ext(path):
 429     """
 430     Install extension to active verifier
 431     """
 432     logger.info("Installing verifier from existing repo...")
 433     tag = get_repo_tag(path)
 434     cmd = ("rally verify add-verifier-ext --source {0} "
 435            "--version {1}"
 436            .format(path, tag))
 437     error_msg = ("Problem while adding verifier extension from %s" % path)
 438     ft_utils.execute_command_raise(cmd, error_msg=error_msg)