functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import pkg_resources
  15 import re
  16 import subprocess
  17 import sys
  18 import fileinput
  19
  20 import yaml
  21
  22 from functest.ci import check_deployment
  23 import functest.utils.functest_utils as ft_utils
  24 import functest.utils.openstack_utils as os_utils
  25 from functest.utils.constants import CONST
  26
  27 from opnfv.utils import constants as opnfv_constants
  28 from opnfv.deployment import factory
  29
  30 actions = ['start', 'check']
  31
  32 """ logging configuration """
  33 logger = logging.getLogger('functest.ci.prepare_env')
  34 handler = None
  35 # set the architecture to default
  36 pod_arch = None
  37 arch_filter = ['aarch64']
  38
  39 CONFIG_FUNCTEST_PATH = pkg_resources.resource_filename(
  40             'functest', 'ci/config_functest.yaml')
  41 CONFIG_PATCH_PATH = pkg_resources.resource_filename(
  42             'functest', 'ci/config_patch.yaml')
  43 CONFIG_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  44             'functest', 'ci/config_aarch64_patch.yaml')
  45 RALLY_CONF_PATH = "/etc/rally/rally.conf"
  46 RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename(
  47             'functest', 'ci/rally_aarch64_patch.conf')
  48
  49
  50 class PrepareEnvParser(object):
  51
  52     def __init__(self):
  53         self.parser = argparse.ArgumentParser()
  54         self.parser.add_argument("action", help="Possible actions are: "
  55                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  56                                  choices=actions)
  57         self.parser.add_argument("-d", "--debug", help="Debug mode",
  58                                  action="store_true")
  59
  60     def parse_args(self, argv=[]):
  61         return vars(self.parser.parse_args(argv))
  62
  63
  64 def print_separator():
  65     logger.info("==============================================")
  66
  67
  68 def check_env_variables():
  69     print_separator()
  70     logger.info("Checking environment variables...")
  71
  72     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  73         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  74         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  75     else:
  76         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  77                 opnfv_constants.INSTALLERS):
  78             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  79                            "Available OPNFV Installers are : %s. "
  80                            "Setting INSTALLER_TYPE=undefined."
  81                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  82                               opnfv_constants.INSTALLERS))
  83             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  84         else:
  85             logger.info("    INSTALLER_TYPE=%s"
  86                         % CONST.__getattribute__('INSTALLER_TYPE'))
  87
  88     if CONST.__getattribute__('INSTALLER_IP') is None:
  89         logger.warning(
  90             "The env variable 'INSTALLER_IP' is not defined. It is recommended"
  91             " to extract some information from the deployment")
  92     else:
  93         logger.info("    INSTALLER_IP=%s" %
  94                     CONST.__getattribute__('INSTALLER_IP'))
  95
  96     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  97         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  98                        "Setting CI_SCENARIO=undefined.")
  99         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
 100     else:
 101         logger.info("    DEPLOY_SCENARIO=%s"
 102                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 103     if CONST.__getattribute__('CI_DEBUG'):
 104         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 105
 106     if CONST.__getattribute__('NODE_NAME'):
 107         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 108
 109     if CONST.__getattribute__('BUILD_TAG'):
 110         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 111
 112     if CONST.__getattribute__('IS_CI_RUN'):
 113         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 114
 115
 116 def get_deployment_handler():
 117     global handler
 118     global pod_arch
 119
 120     installer_params_yaml = pkg_resources.resource_filename(
 121             'functest', 'ci/installer_params.yaml')
 122     if (CONST.__getattribute__('INSTALLER_IP') and
 123         CONST.__getattribute__('INSTALLER_TYPE') and
 124             CONST.__getattribute__('INSTALLER_TYPE') in
 125             opnfv_constants.INSTALLERS):
 126         try:
 127             installer_params = ft_utils.get_parameter_from_yaml(
 128                 CONST.__getattribute__('INSTALLER_TYPE'),
 129                 installer_params_yaml)
 130         except ValueError as e:
 131             logger.debug('Printing deployment info is not supported for %s' %
 132                          CONST.__getattribute__('INSTALLER_TYPE'))
 133             logger.debug(e)
 134         else:
 135             user = installer_params.get('user', None)
 136             password = installer_params.get('password', None)
 137             pkey = installer_params.get('pkey', None)
 138             try:
 139                 handler = factory.Factory.get_handler(
 140                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 141                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 142                     installer_user=user,
 143                     installer_pwd=password,
 144                     pkey_file=pkey)
 145                 if handler:
 146                     pod_arch = handler.get_arch()
 147             except Exception as e:
 148                 logger.debug("Cannot get deployment information. %s" % e)
 149
 150
 151 def create_directories():
 152     print_separator()
 153     logger.info("Creating needed directories...")
 154     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 155         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 156         logger.info("    %s created." %
 157                     CONST.__getattribute__('dir_functest_conf'))
 158     else:
 159         logger.debug("   %s already exists." %
 160                      CONST.__getattribute__('dir_functest_conf'))
 161
 162     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 163         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 164         logger.info("    %s created." %
 165                     CONST.__getattribute__('dir_functest_data'))
 166     else:
 167         logger.debug("   %s already exists." %
 168                      CONST.__getattribute__('dir_functest_data'))
 169     if not os.path.exists(CONST.__getattribute__('dir_functest_images')):
 170         os.makedirs(CONST.__getattribute__('dir_functest_images'))
 171         logger.info("    %s created." %
 172                     CONST.__getattribute__('dir_functest_images'))
 173     else:
 174         logger.debug("   %s already exists." %
 175                      CONST.__getattribute__('dir_functest_images'))
 176
 177
 178 def source_rc_file():
 179     print_separator()
 180
 181     logger.info("Sourcing the OpenStack RC file...")
 182     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 183     for key, value in os.environ.iteritems():
 184         if re.search("OS_", key):
 185             if key == 'OS_AUTH_URL':
 186                 CONST.__setattr__('OS_AUTH_URL', value)
 187             elif key == 'OS_USERNAME':
 188                 CONST.__setattr__('OS_USERNAME', value)
 189             elif key == 'OS_TENANT_NAME':
 190                 CONST.__setattr__('OS_TENANT_NAME', value)
 191             elif key == 'OS_PASSWORD':
 192                 CONST.__setattr__('OS_PASSWORD', value)
 193
 194
 195 def update_config_file():
 196     patch_file(CONFIG_PATCH_PATH)
 197
 198     if pod_arch and pod_arch in arch_filter:
 199         patch_file(CONFIG_AARCH64_PATCH_PATH)
 200
 201     if "TEST_DB_URL" in os.environ:
 202         update_db_url()
 203
 204
 205 def patch_file(patch_file_path):
 206     logger.debug('Updating file: %s', patch_file_path)
 207     with open(patch_file_path) as f:
 208         patch_file = yaml.safe_load(f)
 209
 210     updated = False
 211     for key in patch_file:
 212         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 213             new_functest_yaml = dict(ft_utils.merge_dicts(
 214                 ft_utils.get_functest_yaml(), patch_file[key]))
 215             updated = True
 216
 217     if updated:
 218         os.remove(CONFIG_FUNCTEST_PATH)
 219         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 220             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 221
 222
 223 def update_db_url():
 224     with open(CONFIG_FUNCTEST_PATH) as f:
 225         functest_yaml = yaml.safe_load(f)
 226
 227     with open(CONFIG_FUNCTEST_PATH, "w") as f:
 228         functest_yaml["results"]["test_db_url"] = os.environ.get('TEST_DB_URL')
 229         f.write(yaml.dump(functest_yaml, default_style='"'))
 230
 231
 232 def verify_deployment():
 233     print_separator()
 234     logger.info("Verifying OpenStack deployment...")
 235     deployment = check_deployment.CheckDeployment()
 236     deployment.check_all()
 237
 238
 239 def install_rally():
 240     print_separator()
 241
 242     if pod_arch and pod_arch in arch_filter:
 243         logger.info("Apply aarch64 specific to rally config...")
 244         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 245             rally_patch_conf = f.read()
 246
 247         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 248             print line,
 249             if "cirros|testvm" in line:
 250                 print rally_patch_conf
 251
 252     logger.info("Creating Rally environment...")
 253
 254     cmd = "rally deployment destroy opnfv-rally"
 255     ft_utils.execute_command(cmd, error_msg=(
 256         "Deployment %s does not exist."
 257         % CONST.__getattribute__('rally_deployment_name')),
 258         verbose=False)
 259
 260     rally_conf = os_utils.get_credentials_for_rally()
 261     with open('rally_conf.json', 'w') as fp:
 262         json.dump(rally_conf, fp)
 263     cmd = ("rally deployment create "
 264            "--file=rally_conf.json --name={0}"
 265            .format(CONST.__getattribute__('rally_deployment_name')))
 266     error_msg = "Problem while creating Rally deployment"
 267     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 268
 269     cmd = "rally deployment check"
 270     error_msg = "OpenStack not responding or faulty Rally deployment."
 271     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 272
 273     cmd = "rally deployment list"
 274     ft_utils.execute_command(cmd,
 275                              error_msg=("Problem while listing "
 276                                         "Rally deployment."))
 277
 278     cmd = "rally plugin list | head -5"
 279     ft_utils.execute_command(cmd,
 280                              error_msg=("Problem while showing "
 281                                         "Rally plugins."))
 282
 283
 284 def install_tempest():
 285     logger.info("Installing tempest from existing repo...")
 286     cmd = ("rally verify list-verifiers | "
 287            "grep '{0}' | wc -l".format(
 288                CONST.__getattribute__('tempest_deployment_name')))
 289     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 290     while p.poll() is None:
 291         line = p.stdout.readline().rstrip()
 292         if str(line) == '0':
 293             logger.debug("Tempest %s does not exist" %
 294                          CONST.__getattribute__('tempest_deployment_name'))
 295             cmd = ("rally verify create-verifier --source {0} "
 296                    "--name {1} --type tempest --system-wide"
 297                    .format(CONST.__getattribute__('dir_repo_tempest'),
 298                            CONST.__getattribute__('tempest_deployment_name')))
 299             error_msg = "Problem while installing Tempest."
 300             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 301
 302
 303 def create_flavor():
 304     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 305                                                  '512',
 306                                                  '1',
 307                                                  '1',
 308                                                  public=True)
 309     if flavor_id is None:
 310         raise Exception('Failed to create flavor')
 311
 312
 313 def check_environment():
 314     msg_not_active = "The Functest environment is not installed."
 315     if not os.path.isfile(CONST.__getattribute__('env_active')):
 316         raise Exception(msg_not_active)
 317
 318     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 319         s = env_file.read()
 320         if not re.search("1", s):
 321             raise Exception(msg_not_active)
 322
 323     logger.info("Functest environment is installed.")
 324
 325
 326 def print_deployment_info():
 327     if handler:
 328         logger.info('\n\nDeployment information:\n%s' %
 329                     handler.get_deployment_info())
 330
 331
 332 def prepare_env(**kwargs):
 333     try:
 334         if not (kwargs['action'] in actions):
 335             logger.error('Argument not valid.')
 336             return -1
 337         elif kwargs['action'] == "start":
 338             logger.info("######### Preparing Functest environment #########\n")
 339             verify_deployment()
 340             check_env_variables()
 341             create_directories()
 342             source_rc_file()
 343             update_config_file()
 344             install_rally()
 345             install_tempest()
 346             create_flavor()
 347             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 348                 env_file.write("1")
 349             check_environment()
 350         elif kwargs['action'] == "check":
 351             check_environment()
 352     except Exception as e:
 353         logger.error(e)
 354         return -1
 355     return 0
 356
 357
 358 def main():
 359     logging.config.fileConfig(pkg_resources.resource_filename(
 360         'functest', 'ci/logging.ini'))
 361     parser = PrepareEnvParser()
 362     args = parser.parse_args(sys.argv[1:])
 363     return prepare_env(**args)