functest/opnfv_tests/openstack/tempest/conf_utils.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2015 All rights reserved
   4 # This program and the accompanying materials
   5 # are made available under the terms of the Apache License, Version 2.0
   6 # which accompanies this distribution, and is available at
   7 #
   8 # http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10
  11 """Tempest configuration utilities."""
  12
  13 import ConfigParser
  14 import logging
  15 import fileinput
  16 import os
  17 import shutil
  18 import subprocess
  19
  20 import pkg_resources
  21 import yaml
  22
  23 from functest.utils.constants import CONST
  24 from functest.utils import env
  25 import functest.utils.functest_utils as ft_utils
  26
  27
  28 IMAGE_ID_ALT = None
  29 FLAVOR_ID_ALT = None
  30 RALLY_CONF_PATH = "/etc/rally/rally.conf"
  31 RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  32     'functest', 'ci/rally_aarch64_patch.conf')
  33 GLANCE_IMAGE_PATH = os.path.join(
  34     getattr(CONST, 'dir_functest_images'),
  35     getattr(CONST, 'openstack_image_file_name'))
  36 TEMPEST_RESULTS_DIR = os.path.join(getattr(CONST, 'dir_results'), 'tempest')
  37 TEMPEST_CUSTOM = pkg_resources.resource_filename(
  38     'functest', 'opnfv_tests/openstack/tempest/custom_tests/test_list.txt')
  39 TEMPEST_BLACKLIST = pkg_resources.resource_filename(
  40     'functest', 'opnfv_tests/openstack/tempest/custom_tests/blacklist.txt')
  41 TEMPEST_DEFCORE = pkg_resources.resource_filename(
  42     'functest',
  43     'opnfv_tests/openstack/tempest/custom_tests/defcore_req.txt')
  44 TEMPEST_RAW_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_raw_list.txt')
  45 TEMPEST_LIST = os.path.join(TEMPEST_RESULTS_DIR, 'test_list.txt')
  46 REFSTACK_RESULTS_DIR = os.path.join(getattr(CONST, 'dir_results'), 'refstack')
  47 TEMPEST_CONF_YAML = pkg_resources.resource_filename(
  48     'functest', 'opnfv_tests/openstack/tempest/custom_tests/tempest_conf.yaml')
  49 TEST_ACCOUNTS_FILE = pkg_resources.resource_filename(
  50     'functest',
  51     'opnfv_tests/openstack/tempest/custom_tests/test_accounts.yaml')
  52
  53 CI_INSTALLER_TYPE = env.get('INSTALLER_TYPE')
  54
  55 """ logging configuration """
  56 LOGGER = logging.getLogger(__name__)
  57
  58
  59 def create_rally_deployment():
  60     """Create new rally deployment"""
  61     # set the architecture to default
  62     pod_arch = env.get("POD_ARCH")
  63     arch_filter = ['aarch64']
  64
  65     if pod_arch and pod_arch in arch_filter:
  66         LOGGER.info("Apply aarch64 specific to rally config...")
  67         with open(RALLY_AARCH64_PATCH_PATH, "r") as pfile:
  68             rally_patch_conf = pfile.read()
  69
  70         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
  71             print line,
  72             if "cirros|testvm" in line:
  73                 print rally_patch_conf
  74
  75     LOGGER.info("Creating Rally environment...")
  76
  77     cmd = "rally deployment destroy opnfv-rally"
  78     ft_utils.execute_command(cmd, error_msg=(
  79         "Deployment %s does not exist."
  80         % getattr(CONST, 'rally_deployment_name')),
  81                              verbose=False)
  82
  83     cmd = ("rally deployment create --fromenv --name={0}"
  84            .format(getattr(CONST, 'rally_deployment_name')))
  85     error_msg = "Problem while creating Rally deployment"
  86     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
  87
  88     cmd = "rally deployment check"
  89     error_msg = "OpenStack not responding or faulty Rally deployment."
  90     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
  91
  92
  93 def create_verifier():
  94     """Create new verifier"""
  95     LOGGER.info("Create verifier from existing repo...")
  96     cmd = ("rally verify delete-verifier --id '{0}' --force").format(
  97         getattr(CONST, 'tempest_verifier_name'))
  98     ft_utils.execute_command(cmd, error_msg=(
  99         "Verifier %s does not exist."
 100         % getattr(CONST, 'tempest_verifier_name')),
 101                              verbose=False)
 102     cmd = ("rally verify create-verifier --source {0} "
 103            "--name {1} --type tempest --system-wide"
 104            .format(getattr(CONST, 'dir_repo_tempest'),
 105                    getattr(CONST, 'tempest_verifier_name')))
 106     ft_utils.execute_command_raise(cmd,
 107                                    error_msg='Problem while creating verifier')
 108
 109
 110 def get_verifier_id():
 111     """
 112     Returns verifier id for current Tempest
 113     """
 114     create_rally_deployment()
 115     create_verifier()
 116     cmd = ("rally verify list-verifiers | awk '/" +
 117            getattr(CONST, 'tempest_verifier_name') +
 118            "/ {print $2}'")
 119     proc = subprocess.Popen(cmd, shell=True,
 120                             stdout=subprocess.PIPE,
 121                             stderr=subprocess.STDOUT)
 122     deployment_uuid = proc.stdout.readline().rstrip()
 123     if deployment_uuid == "":
 124         LOGGER.error("Tempest verifier not found.")
 125         raise Exception('Error with command:%s' % cmd)
 126     return deployment_uuid
 127
 128
 129 def get_verifier_deployment_id():
 130     """
 131     Returns deployment id for active Rally deployment
 132     """
 133     cmd = ("rally deployment list | awk '/" +
 134            getattr(CONST, 'rally_deployment_name') +
 135            "/ {print $2}'")
 136     proc = subprocess.Popen(cmd, shell=True,
 137                             stdout=subprocess.PIPE,
 138                             stderr=subprocess.STDOUT)
 139     deployment_uuid = proc.stdout.readline().rstrip()
 140     if deployment_uuid == "":
 141         LOGGER.error("Rally deployment not found.")
 142         raise Exception('Error with command:%s' % cmd)
 143     return deployment_uuid
 144
 145
 146 def get_verifier_repo_dir(verifier_id):
 147     """
 148     Returns installed verifier repo directory for Tempest
 149     """
 150     if not verifier_id:
 151         verifier_id = get_verifier_id()
 152
 153     return os.path.join(getattr(CONST, 'dir_rally_inst'),
 154                         'verification',
 155                         'verifier-{}'.format(verifier_id),
 156                         'repo')
 157
 158
 159 def get_verifier_deployment_dir(verifier_id, deployment_id):
 160     """
 161     Returns Rally deployment directory for current verifier
 162     """
 163     if not verifier_id:
 164         verifier_id = get_verifier_id()
 165
 166     if not deployment_id:
 167         deployment_id = get_verifier_deployment_id()
 168
 169     return os.path.join(getattr(CONST, 'dir_rally_inst'),
 170                         'verification',
 171                         'verifier-{}'.format(verifier_id),
 172                         'for-deployment-{}'.format(deployment_id))
 173
 174
 175 def backup_tempest_config(conf_file):
 176     """
 177     Copy config file to tempest results directory
 178     """
 179     if not os.path.exists(TEMPEST_RESULTS_DIR):
 180         os.makedirs(TEMPEST_RESULTS_DIR)
 181     shutil.copyfile(conf_file,
 182                     os.path.join(TEMPEST_RESULTS_DIR, 'tempest.conf'))
 183
 184
 185 def configure_tempest(deployment_dir, network_name=None, image_id=None,
 186                       flavor_id=None, compute_cnt=None):
 187     """
 188     Calls rally verify and updates the generated tempest.conf with
 189     given parameters
 190     """
 191     conf_file = configure_verifier(deployment_dir)
 192     configure_tempest_update_params(conf_file, network_name, image_id,
 193                                     flavor_id, compute_cnt)
 194
 195
 196 def configure_tempest_defcore(deployment_dir, network_name, image_id,
 197                               flavor_id, image_id_alt, flavor_id_alt,
 198                               tenant_id):
 199     # pylint: disable=too-many-arguments
 200     """
 201     Add/update needed parameters into tempest.conf file
 202     """
 203     conf_file = configure_verifier(deployment_dir)
 204     configure_tempest_update_params(conf_file, network_name, image_id,
 205                                     flavor_id)
 206
 207     LOGGER.debug("Updating selected tempest.conf parameters for defcore...")
 208     config = ConfigParser.RawConfigParser()
 209     config.read(conf_file)
 210     config.set('DEFAULT', 'log_file', '{}/tempest.log'.format(deployment_dir))
 211     config.set('oslo_concurrency', 'lock_path',
 212                '{}/lock_files'.format(deployment_dir))
 213     generate_test_accounts_file(tenant_id=tenant_id)
 214     config.set('auth', 'test_accounts_file', TEST_ACCOUNTS_FILE)
 215     config.set('scenario', 'img_dir', '{}'.format(deployment_dir))
 216     config.set('scenario', 'img_file', 'tempest-image')
 217     config.set('compute', 'image_ref', image_id)
 218     config.set('compute', 'image_ref_alt', image_id_alt)
 219     config.set('compute', 'flavor_ref', flavor_id)
 220     config.set('compute', 'flavor_ref_alt', flavor_id_alt)
 221
 222     with open(conf_file, 'wb') as config_file:
 223         config.write(config_file)
 224
 225     confpath = pkg_resources.resource_filename(
 226         'functest',
 227         'opnfv_tests/openstack/refstack_client/refstack_tempest.conf')
 228     shutil.copyfile(conf_file, confpath)
 229
 230
 231 def generate_test_accounts_file(tenant_id):
 232     """
 233     Add needed tenant and user params into test_accounts.yaml
 234     """
 235
 236     LOGGER.debug("Add needed params into test_accounts.yaml...")
 237     accounts_list = [
 238         {
 239             'tenant_name': getattr(CONST, 'tempest_identity_tenant_name'),
 240             'tenant_id': str(tenant_id),
 241             'username': getattr(CONST, 'tempest_identity_user_name'),
 242             'password': getattr(CONST, 'tempest_identity_user_password')
 243         }
 244     ]
 245
 246     with open(TEST_ACCOUNTS_FILE, "w") as tfile:
 247         yaml.dump(accounts_list, tfile, default_flow_style=False)
 248
 249
 250 def update_tempest_conf_file(conf_file, config):
 251     """Update defined paramters into tempest config file"""
 252     with open(TEMPEST_CONF_YAML) as yfile:
 253         conf_yaml = yaml.safe_load(yfile)
 254     if conf_yaml:
 255         sections = config.sections()
 256         for section in conf_yaml:
 257             if section not in sections:
 258                 config.add_section(section)
 259             sub_conf = conf_yaml.get(section)
 260             for key, value in sub_conf.items():
 261                 config.set(section, key, value)
 262
 263     with open(conf_file, 'wb') as config_file:
 264         config.write(config_file)
 265
 266
 267 def configure_tempest_update_params(tempest_conf_file, network_name=None,
 268                                     image_id=None, flavor_id=None,
 269                                     compute_cnt=1):
 270     """
 271     Add/update needed parameters into tempest.conf file
 272     """
 273     LOGGER.debug("Updating selected tempest.conf parameters...")
 274     config = ConfigParser.RawConfigParser()
 275     config.read(tempest_conf_file)
 276     config.set('compute', 'fixed_network_name', network_name)
 277     config.set('compute', 'volume_device_name',
 278                getattr(CONST, 'tempest_volume_device_name'))
 279
 280     if image_id is not None:
 281         config.set('compute', 'image_ref', image_id)
 282     if IMAGE_ID_ALT is not None:
 283         config.set('compute', 'image_ref_alt', IMAGE_ID_ALT)
 284     if getattr(CONST, 'tempest_use_custom_flavors'):
 285         if flavor_id is not None:
 286             config.set('compute', 'flavor_ref', flavor_id)
 287         if FLAVOR_ID_ALT is not None:
 288             config.set('compute', 'flavor_ref_alt', FLAVOR_ID_ALT)
 289     if compute_cnt > 1:
 290         # enable multinode tests
 291         config.set('compute', 'min_compute_nodes', compute_cnt)
 292         config.set('compute-feature-enabled', 'live_migration', True)
 293
 294     config.set('identity', 'region', os.environ.get('OS_REGION_NAME'))
 295     identity_api_version = os.environ.get(
 296         "OS_IDENTITY_API_VERSION", os.environ.get("IDENTITY_API_VERSION"))
 297     if identity_api_version == '3':
 298         auth_version = 'v3'
 299         config.set('identity-feature-enabled', 'api_v2', False)
 300     else:
 301         auth_version = 'v2'
 302     config.set('identity', 'auth_version', auth_version)
 303     config.set(
 304         'validation', 'ssh_timeout',
 305         getattr(CONST, 'tempest_validation_ssh_timeout'))
 306     config.set('object-storage', 'operator_role',
 307                getattr(CONST, 'tempest_object_storage_operator_role'))
 308
 309     if os.environ.get('OS_ENDPOINT_TYPE') is not None:
 310         config.set('identity', 'v3_endpoint_type',
 311                    os.environ.get('OS_ENDPOINT_TYPE'))
 312
 313     if os.environ.get('OS_ENDPOINT_TYPE') is not None:
 314         sections = config.sections()
 315         services_list = ['compute',
 316                          'volume',
 317                          'image',
 318                          'network',
 319                          'data-processing',
 320                          'object-storage',
 321                          'orchestration']
 322         for service in services_list:
 323             if service not in sections:
 324                 config.add_section(service)
 325             config.set(service, 'endpoint_type',
 326                        os.environ.get('OS_ENDPOINT_TYPE'))
 327
 328     LOGGER.debug('Add/Update required params defined in tempest_conf.yaml '
 329                  'into tempest.conf file')
 330     update_tempest_conf_file(tempest_conf_file, config)
 331
 332     backup_tempest_config(tempest_conf_file)
 333
 334
 335 def configure_verifier(deployment_dir):
 336     """
 337     Execute rally verify configure-verifier, which generates tempest.conf
 338     """
 339     tempest_conf_file = os.path.join(deployment_dir, "tempest.conf")
 340     if os.path.isfile(tempest_conf_file):
 341         LOGGER.debug("Verifier is already configured.")
 342         LOGGER.debug("Reconfiguring the current verifier...")
 343         cmd = "rally verify configure-verifier --reconfigure"
 344     else:
 345         LOGGER.info("Configuring the verifier...")
 346         cmd = "rally verify configure-verifier"
 347     ft_utils.execute_command(cmd)
 348
 349     LOGGER.debug("Looking for tempest.conf file...")
 350     if not os.path.isfile(tempest_conf_file):
 351         LOGGER.error("Tempest configuration file %s NOT found.",
 352                      tempest_conf_file)
 353         raise Exception("Tempest configuration file %s NOT found."
 354                         % tempest_conf_file)
 355     else:
 356         return tempest_conf_file