functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 import ConfigParser
  11 import logging
  12 import os
  13 import pkg_resources
  14 import shutil
  15 import subprocess
  16
  17 import yaml
  18
  19 from functest.utils.constants import CONST
  20 import functest.utils.functest_utils as ft_utils
  21 import functest.utils.openstack_utils as os_utils
  22
  23
  24 IMAGE_ID_ALT = None
  25 FLAVOR_ID_ALT = None
  26 GLANCE_IMAGE_PATH = os.path.join(
  27     CONST.__getattribute__('dir_functest_images'),
  28     CONST.__getattribute__('openstack_image_file_name'))
  29 TEMPEST_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  30                                    'tempest')
  31 TEMPEST_CUSTOM = pkg_resources.resource_filename(
  32     'functest', 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  33 TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  34     'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.txt')
  35 TEMPEST_DEFCORE = pkg_resources.resource_filename(
  36     'functest',
  37     'opnfv_tests/openstack/tempest/custom_tests/defcore_req.txt')
  38 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  39 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  40 REFSTACK_RESULTS_DIR = os.path.join(CONST.__getattribute__('dir_results'),
  41                                     'refstack')
  42 TEMPEST_CONF_YAML = pkg_resources.resource_filename(
  43     'functest', 'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
  44 TEST_ACCOUNTS_FILE = pkg_resources.resource_filename(
  45     'functest',
  46     'opnfv_tests/openstack/tempest/custom_tests/test_accounts.yaml')
  47
  48 CI_INSTALLER_TYPE = CONST.__getattribute__('INSTALLER_TYPE')
  49 CI_INSTALLER_IP = CONST.__getattribute__('INSTALLER_IP')
  50
  51 """ logging configuration """
  52 logger = logging.getLogger(__name__)
  53
  54
  55 def create_tempest_resources(use_custom_images=False,
  56                              use_custom_flavors=False):
  57
  58     logger.debug("Creating private network for Tempest suite")
  59     network_dic = os_utils.create_shared_network_full(
  60         CONST.__getattribute__('tempest_private_net_name'),
  61         CONST.__getattribute__('tempest_private_subnet_name'),
  62         CONST.__getattribute__('tempest_router_name'),
  63         CONST.__getattribute__('tempest_private_subnet_cidr'))
  64     if network_dic is None:
  65         raise Exception('Failed to create private network')
  66
  67     image_id = ""
  68     image_id_alt = ""
  69     flavor_id = ""
  70     flavor_id_alt = ""
  71
  72     if (CONST.__getattribute__('tempest_use_custom_images') or
  73        use_custom_images):
  74         # adding alternative image should be trivial should we need it
  75         logger.debug("Creating image for Tempest suite")
  76         _, image_id = os_utils.get_or_create_image(
  77             CONST.__getattribute__('openstack_image_name'),
  78             GLANCE_IMAGE_PATH,
  79             CONST.__getattribute__('openstack_image_disk_format'))
  80         if image_id is None:
  81             raise Exception('Failed to create image')
  82
  83     if use_custom_images:
  84         logger.debug("Creating 2nd image for Tempest suite")
  85         _, image_id_alt = os_utils.get_or_create_image(
  86             CONST.__getattribute__('openstack_image_name_alt'),
  87             GLANCE_IMAGE_PATH,
  88             CONST.__getattribute__('openstack_image_disk_format'))
  89         if image_id_alt is None:
  90             raise Exception('Failed to create image')
  91
  92     if (CONST.__getattribute__('tempest_use_custom_flavors') or
  93        use_custom_flavors):
  94         # adding alternative flavor should be trivial should we need it
  95         logger.debug("Creating flavor for Tempest suite")
  96         _, flavor_id = os_utils.get_or_create_flavor(
  97             CONST.__getattribute__('openstack_flavor_name'),
  98             CONST.__getattribute__('openstack_flavor_ram'),
  99             CONST.__getattribute__('openstack_flavor_disk'),
 100             CONST.__getattribute__('openstack_flavor_vcpus'))
 101         if flavor_id is None:
 102             raise Exception('Failed to create flavor')
 103
 104     if use_custom_flavors:
 105         logger.debug("Creating 2nd flavor for tempest_defcore")
 106         _, flavor_id_alt = os_utils.get_or_create_flavor(
 107             CONST.__getattribute__('openstack_flavor_name_alt'),
 108             CONST.__getattribute__('openstack_flavor_ram'),
 109             CONST.__getattribute__('openstack_flavor_disk'),
 110             CONST.__getattribute__('openstack_flavor_vcpus'))
 111         if flavor_id_alt is None:
 112             raise Exception('Failed to create flavor')
 113
 114     img_flavor_dict = {}
 115     img_flavor_dict['image_id'] = image_id
 116     img_flavor_dict['image_id_alt'] = image_id_alt
 117     img_flavor_dict['flavor_id'] = flavor_id
 118     img_flavor_dict['flavor_id_alt'] = flavor_id_alt
 119
 120     return img_flavor_dict
 121
 122
 123 def create_tenant_user():
 124     keystone_client = os_utils.get_keystone_client()
 125
 126     logger.debug("Creating tenant and user for Tempest suite")
 127     tenant_id = os_utils.create_tenant(
 128         keystone_client,
 129         CONST.__getattribute__('tempest_identity_tenant_name'),
 130         CONST.__getattribute__('tempest_identity_tenant_description'))
 131     if not tenant_id:
 132         logger.error("Failed to create %s tenant"
 133                      % CONST.__getattribute__('tempest_identity_tenant_name'))
 134
 135     user_id = os_utils.create_user(
 136         keystone_client,
 137         CONST.__getattribute__('tempest_identity_user_name'),
 138         CONST.__getattribute__('tempest_identity_user_password'),
 139         None, tenant_id)
 140     if not user_id:
 141         logger.error("Failed to create %s user" %
 142                      CONST.__getattribute__('tempest_identity_user_name'))
 143
 144     return tenant_id
 145
 146
 147 def get_verifier_id():
 148     """
 149     Returns verifer id for current Tempest
 150     """
 151     cmd = ("rally verify list-verifiers | awk '/" +
 152            CONST.__getattribute__('tempest_deployment_name') +
 153            "/ {print $2}'")
 154     p = subprocess.Popen(cmd, shell=True,
 155                          stdout=subprocess.PIPE,
 156                          stderr=subprocess.STDOUT)
 157     deployment_uuid = p.stdout.readline().rstrip()
 158     if deployment_uuid == "":
 159         logger.error("Tempest verifier not found.")
 160         raise Exception('Error with command:%s' % cmd)
 161     return deployment_uuid
 162
 163
 164 def get_verifier_deployment_id():
 165     """
 166     Returns deployment id for active Rally deployment
 167     """
 168     cmd = ("rally deployment list | awk '/" +
 169            CONST.__getattribute__('rally_deployment_name') +
 170            "/ {print $2}'")
 171     p = subprocess.Popen(cmd, shell=True,
 172                          stdout=subprocess.PIPE,
 173                          stderr=subprocess.STDOUT)
 174     deployment_uuid = p.stdout.readline().rstrip()
 175     if deployment_uuid == "":
 176         logger.error("Rally deployment not found.")
 177         raise Exception('Error with command:%s' % cmd)
 178     return deployment_uuid
 179
 180
 181 def get_verifier_repo_dir(verifier_id):
 182     """
 183     Returns installed verfier repo directory for Tempest
 184     """
 185     if not verifier_id:
 186         verifier_id = get_verifier_id()
 187
 188     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 189                         'verification',
 190                         'verifier-{}'.format(verifier_id),
 191                         'repo')
 192
 193
 194 def get_verifier_deployment_dir(verifier_id, deployment_id):
 195     """
 196     Returns Rally deployment directory for current verifier
 197     """
 198     if not verifier_id:
 199         verifier_id = get_verifier_id()
 200
 201     if not deployment_id:
 202         deployment_id = get_verifier_deployment_id()
 203
 204     return os.path.join(CONST.__getattribute__('dir_rally_inst'),
 205                         'verification',
 206                         'verifier-{}'.format(verifier_id),
 207                         'for-deployment-{}'.format(deployment_id))
 208
 209
 210 def get_repo_tag(repo):
 211     """
 212     Returns last tag of current branch
 213     """
 214     cmd = ("git -C {0} describe --abbrev=0 HEAD".format(repo))
 215     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 216     tag = p.stdout.readline().rstrip()
 217
 218     return str(tag)
 219
 220
 221 def backup_tempest_config(conf_file):
 222     """
 223     Copy config file to tempest results directory
 224     """
 225     if not os.path.exists(TEMPEST_RESULTS_DIR):
 226         os.makedirs(TEMPEST_RESULTS_DIR)
 227
 228     shutil.copyfile(conf_file,
 229                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 230
 231
 232 def configure_tempest(deployment_dir, IMAGE_ID=None, FLAVOR_ID=None,
 233                       MODE=None):
 234     """
 235     Calls rally verify and updates the generated tempest.conf with
 236     given parameters
 237     """
 238     conf_file = configure_verifier(deployment_dir)
 239     configure_tempest_update_params(conf_file,
 240                                     IMAGE_ID, FLAVOR_ID)
 241
 242
 243 def configure_tempest_defcore(deployment_dir, img_flavor_dict):
 244     """
 245     Add/update needed parameters into tempest.conf file
 246     """
 247     conf_file = configure_verifier(deployment_dir)
 248     configure_tempest_update_params(conf_file,
 249                                     img_flavor_dict.get("image_id"),
 250                                     img_flavor_dict.get("flavor_id"))
 251
 252     logger.debug("Updating selected tempest.conf parameters for defcore...")
 253     config = ConfigParser.RawConfigParser()
 254     config.read(conf_file)
 255     config.set('DEFAULT', 'log_file', '{}/tempest.log'.format(deployment_dir))
 256     config.set('oslo_concurrency', 'lock_path',
 257                '{}/lock_files'.format(deployment_dir))
 258     generate_test_accounts_file()
 259     config.set('auth', 'test_accounts_file', TEST_ACCOUNTS_FILE)
 260     config.set('scenario', 'img_dir', '{}'.format(deployment_dir))
 261     config.set('scenario', 'img_file', 'tempest-image')
 262     config.set('compute', 'image_ref', img_flavor_dict.get("image_id"))
 263     config.set('compute', 'image_ref_alt',
 264                img_flavor_dict['image_id_alt'])
 265     config.set('compute', 'flavor_ref', img_flavor_dict.get("flavor_id"))
 266     config.set('compute', 'flavor_ref_alt',
 267                img_flavor_dict['flavor_id_alt'])
 268
 269     with open(conf_file, 'wb') as config_file:
 270         config.write(config_file)
 271
 272     confpath = pkg_resources.resource_filename(
 273         'functest',
 274         'opnfv_tests/openstack/refstack_client/refstack_tempest.conf')
 275     shutil.copyfile(conf_file, confpath)
 276
 277
 278 def generate_test_accounts_file():
 279     """
 280     Add needed tenant and user params into test_accounts.yaml
 281     """
 282
 283     logger.debug("Add needed params into test_accounts.yaml...")
 284     tenant_id = create_tenant_user()
 285     accounts_list = [
 286         {
 287             'tenant_name':
 288                 CONST.__getattribute__('tempest_identity_tenant_name'),
 289             'tenant_id': str(tenant_id),
 290             'username': CONST.__getattribute__('tempest_identity_user_name'),
 291             'password':
 292                 CONST.__getattribute__('tempest_identity_user_password')
 293         }
 294     ]
 295
 296     with open(TEST_ACCOUNTS_FILE, "w") as f:
 297         yaml.dump(accounts_list, f, default_flow_style=False)
 298
 299
 300 def configure_tempest_update_params(tempest_conf_file,
 301                                     IMAGE_ID=None, FLAVOR_ID=None):
 302     """
 303     Add/update needed parameters into tempest.conf file
 304     """
 305     logger.debug("Updating selected tempest.conf parameters...")
 306     config = ConfigParser.RawConfigParser()
 307     config.read(tempest_conf_file)
 308     config.set(
 309         'compute',
 310         'fixed_network_name',
 311         CONST.__getattribute__('tempest_private_net_name'))
 312     config.set('compute', 'volume_device_name',
 313                CONST.__getattribute__('tempest_volume_device_name'))
 314     if CONST.__getattribute__('tempest_use_custom_images'):
 315         if IMAGE_ID is not None:
 316             config.set('compute', 'image_ref', IMAGE_ID)
 317         if IMAGE_ID_ALT is not None:
 318             config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 319     if CONST.__getattribute__('tempest_use_custom_flavors'):
 320         if FLAVOR_ID is not None:
 321             config.set('compute', 'flavor_ref', FLAVOR_ID)
 322         if FLAVOR_ID_ALT is not None:
 323             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 324     config.set('identity', 'region', 'RegionOne')
 325     if os_utils.is_keystone_v3():
 326         auth_version = 'v3'
 327     else:
 328         auth_version = 'v2'
 329     config.set('identity', 'auth_version', auth_version)
 330     config.set(
 331         'validation', 'ssh_timeout',
 332         CONST.__getattribute__('tempest_validation_ssh_timeout'))
 333     config.set('object-storage', 'operator_role',
 334                CONST.__getattribute__('tempest_object_storage_operator_role'))
 335
 336     if CONST.__getattribute__('OS_ENDPOINT_TYPE') is not None:
 337         sections = config.sections()
 338         if os_utils.is_keystone_v3():
 339             config.set('identity', 'v3_endpoint_type',
 340                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 341             if 'identity-feature-enabled' not in sections:
 342                 config.add_section('identity-feature-enabled')
 343                 config.set('identity-feature-enabled', 'api_v2', False)
 344                 config.set('identity-feature-enabled', 'api_v2_admin', False)
 345         services_list = ['compute',
 346                          'volume',
 347                          'image',
 348                          'network',
 349                          'data-processing',
 350                          'object-storage',
 351                          'orchestration']
 352         for service in services_list:
 353             if service not in sections:
 354                 config.add_section(service)
 355             config.set(service, 'endpoint_type',
 356                        CONST.__getattribute__('OS_ENDPOINT_TYPE'))
 357
 358     logger.debug('Add/Update required params defined in tempest_conf.yaml '
 359                  'into tempest.conf file')
 360     with open(TEMPEST_CONF_YAML) as f:
 361         conf_yaml = yaml.safe_load(f)
 362     if conf_yaml:
 363         sections = config.sections()
 364         for section in conf_yaml:
 365             if section not in sections:
 366                 config.add_section(section)
 367             sub_conf = conf_yaml.get(section)
 368             for key, value in sub_conf.items():
 369                 config.set(section, key, value)
 370
 371     with open(tempest_conf_file, 'wb') as config_file:
 372         config.write(config_file)
 373
 374     backup_tempest_config(tempest_conf_file)
 375
 376
 377 def configure_verifier(deployment_dir):
 378     """
 379     Execute rally verify configure-verifier, which generates tempest.conf
 380     """
 381     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 382     if os.path.isfile(tempest_conf_file):
 383         logger.debug("Verifier is already configured.")
 384         logger.debug("Reconfiguring the current verifier...")
 385         cmd = "rally verify configure-verifier --reconfigure"
 386     else:
 387         logger.info("Configuring the verifier...")
 388         cmd = "rally verify configure-verifier"
 389     ft_utils.execute_command(cmd)
 390
 391     logger.debug("Looking for tempest.conf file...")
 392     if not os.path.isfile(tempest_conf_file):
 393         logger.error("Tempest configuration file %s NOT found."
 394                      % tempest_conf_file)
 395         raise Exception("Tempest configuration file %s NOT found."
 396                         % tempest_conf_file)
 397     else:
 398         return tempest_conf_file