functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  72     else:
  73         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  74                 opnfv_constants.INSTALLERS):
  75             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  76                            "Available OPNFV Installers are : %s. "
  77                            "Setting INSTALLER_TYPE=undefined."
  78                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  79                               opnfv_constants.INSTALLERS))
  80             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  81         else:
  82             logger.info("    INSTALLER_TYPE=%s"
  83                         % CONST.__getattribute__('INSTALLER_TYPE'))
  84
  85     if CONST.__getattribute__('INSTALLER_IP') is None:
  86         logger.warning(
  87             "The env variable 'INSTALLER_IP' is not defined. It is recommended"
  88             " to extract some information from the deployment")
  89     else:
  90         logger.info("    INSTALLER_IP=%s" %
  91                     CONST.__getattribute__('INSTALLER_IP'))
  92
  93     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  94         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  95                        "Setting CI_SCENARIO=undefined.")
  96         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
  97     else:
  98         logger.info("    DEPLOY_SCENARIO=%s"
  99                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 100     if CONST.__getattribute__('CI_DEBUG'):
 101         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 102
 103     if CONST.__getattribute__('NODE_NAME'):
 104         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 105
 106     if CONST.__getattribute__('BUILD_TAG'):
 107         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 108
 109     if CONST.__getattribute__('IS_CI_RUN'):
 110         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 111
 112
 113 def get_deployment_handler():
 114     global handler
 115     global pod_arch
 116
 117     installer_params_yaml = os.path.join(
 118         CONST.__getattribute__('dir_repo_functest'),
 119         'functest/ci/installer_params.yaml')
 120     if (CONST.__getattribute__('INSTALLER_IP') and
 121         CONST.__getattribute__('INSTALLER_TYPE') and
 122             CONST.__getattribute__('INSTALLER_TYPE') in
 123             opnfv_constants.INSTALLERS):
 124         try:
 125             installer_params = ft_utils.get_parameter_from_yaml(
 126                 CONST.__getattribute__('INSTALLER_TYPE'),
 127                 installer_params_yaml)
 128         except ValueError as e:
 129             logger.debug('Printing deployment info is not supported for %s' %
 130                          CONST.__getattribute__('INSTALLER_TYPE'))
 131             logger.debug(e)
 132         else:
 133             user = installer_params.get('user', None)
 134             password = installer_params.get('password', None)
 135             pkey = installer_params.get('pkey', None)
 136             try:
 137                 handler = factory.Factory.get_handler(
 138                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 139                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 140                     installer_user=user,
 141                     installer_pwd=password,
 142                     pkey_file=pkey)
 143                 if handler:
 144                     pod_arch = handler.get_arch()
 145             except Exception as e:
 146                 logger.debug("Cannot get deployment information. %s" % e)
 147
 148
 149 def create_directories():
 150     print_separator()
 151     logger.info("Creating needed directories...")
 152     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 153         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 154         logger.info("    %s created." %
 155                     CONST.__getattribute__('dir_functest_conf'))
 156     else:
 157         logger.debug("   %s already exists." %
 158                      CONST.__getattribute__('dir_functest_conf'))
 159
 160     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 161         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 162         logger.info("    %s created." %
 163                     CONST.__getattribute__('dir_functest_data'))
 164     else:
 165         logger.debug("   %s already exists." %
 166                      CONST.__getattribute__('dir_functest_data'))
 167     if not os.path.exists(CONST.__getattribute__('dir_functest_images')):
 168         os.makedirs(CONST.__getattribute__('dir_functest_images'))
 169         logger.info("    %s created." %
 170                     CONST.__getattribute__('dir_functest_images'))
 171     else:
 172         logger.debug("   %s already exists." %
 173                      CONST.__getattribute__('dir_functest_images'))
 174
 175
 176 def source_rc_file():
 177     print_separator()
 178
 179     if CONST.__getattribute__('openstack_creds') is None:
 180         logger.warning("The environment variable 'creds' must be set and"
 181                        "pointing to the local RC file. Using default: "
 182                        "/home/opnfv/functest/conf/openstack.creds ...")
 183         os.path.join(
 184             CONST.__getattribute__('dir_functest_conf'), 'openstack.creds')
 185
 186     if not os.path.isfile(CONST.__getattribute__('openstack_creds')):
 187         raise Exception(
 188             "OpenStack credentials file not provided. "
 189             "The OpenStack credentials must be in {}"
 190             .format(CONST.__getattribute__('openstack_creds')))
 191     else:
 192         logger.info("RC file provided in %s."
 193                     % CONST.__getattribute__('openstack_creds'))
 194         if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0:
 195             raise Exception(
 196                 "The OpenStack RC file {} is empty."
 197                 .format(CONST.__getattribute__('openstack_creds')))
 198
 199     logger.info("Sourcing the OpenStack RC file...")
 200     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 201     for key, value in os.environ.iteritems():
 202         if re.search("OS_", key):
 203             if key == 'OS_AUTH_URL':
 204                 CONST.__setattr__('OS_AUTH_URL', value)
 205             elif key == 'OS_USERNAME':
 206                 CONST.__setattr__('OS_USERNAME', value)
 207             elif key == 'OS_TENANT_NAME':
 208                 CONST.__setattr__('OS_TENANT_NAME', value)
 209             elif key == 'OS_PASSWORD':
 210                 CONST.__setattr__('OS_PASSWORD', value)
 211
 212
 213 def update_config_file():
 214     patch_file(CONFIG_PATCH_PATH)
 215
 216     if pod_arch and pod_arch in arch_filter:
 217         patch_file(CONFIG_AARCH64_PATCH_PATH)
 218
 219     if "TEST_DB_URL" in os.environ:
 220         update_db_url()
 221
 222
 223 def patch_file(patch_file_path):
 224     logger.debug('Updating file: %s', patch_file_path)
 225     with open(patch_file_path) as f:
 226         patch_file = yaml.safe_load(f)
 227
 228     updated = False
 229     for key in patch_file:
 230         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 231             new_functest_yaml = dict(ft_utils.merge_dicts(
 232                 ft_utils.get_functest_yaml(), patch_file[key]))
 233             updated = True
 234
 235     if updated:
 236         os.remove(CONFIG_FUNCTEST_PATH)
 237         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 238             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 239
 240
 241 def update_db_url():
 242     with open(CONFIG_FUNCTEST_PATH) as f:
 243         functest_yaml = yaml.safe_load(f)
 244
 245     with open(CONFIG_FUNCTEST_PATH, "w") as f:
 246         functest_yaml["results"]["test_db_url"] = os.environ.get('TEST_DB_URL')
 247         f.write(yaml.dump(functest_yaml, default_style='"'))
 248
 249
 250 def verify_deployment():
 251     print_separator()
 252     logger.info("Verifying OpenStack services...")
 253     cmd = ("%s/functest/ci/check_os.sh" %
 254            CONST.__getattribute__('dir_repo_functest'))
 255
 256     logger.debug("Executing command: %s" % cmd)
 257     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 258
 259     while p.poll() is None:
 260         line = p.stdout.readline().rstrip()
 261         if "ERROR" in line:
 262             logger.error(line)
 263             raise Exception("Problem while running 'check_os.sh'.")
 264         logger.info(line)
 265
 266
 267 def install_rally():
 268     print_separator()
 269
 270     if pod_arch and pod_arch in arch_filter:
 271         logger.info("Apply aarch64 specific to rally config...")
 272         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 273             rally_patch_conf = f.read()
 274
 275         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 276             print line,
 277             if "cirros|testvm" in line:
 278                 print rally_patch_conf
 279
 280     logger.info("Creating Rally environment...")
 281
 282     cmd = "rally deployment destroy opnfv-rally"
 283     ft_utils.execute_command(cmd, error_msg=(
 284         "Deployment %s does not exist."
 285         % CONST.__getattribute__('rally_deployment_name')),
 286         verbose=False)
 287
 288     rally_conf = os_utils.get_credentials_for_rally()
 289     with open('rally_conf.json', 'w') as fp:
 290         json.dump(rally_conf, fp)
 291     cmd = ("rally deployment create "
 292            "--file=rally_conf.json --name={0}"
 293            .format(CONST.__getattribute__('rally_deployment_name')))
 294     error_msg = "Problem while creating Rally deployment"
 295     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 296
 297     cmd = "rally deployment check"
 298     error_msg = "OpenStack not responding or faulty Rally deployment."
 299     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 300
 301     cmd = "rally deployment list"
 302     ft_utils.execute_command(cmd,
 303                              error_msg=("Problem while listing "
 304                                         "Rally deployment."))
 305
 306     cmd = "rally plugin list | head -5"
 307     ft_utils.execute_command(cmd,
 308                              error_msg=("Problem while showing "
 309                                         "Rally plugins."))
 310
 311
 312 def install_tempest():
 313     logger.info("Installing tempest from existing repo...")
 314     cmd = ("rally verify list-verifiers | "
 315            "grep '{0}' | wc -l".format(
 316                CONST.__getattribute__('tempest_deployment_name')))
 317     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 318     while p.poll() is None:
 319         line = p.stdout.readline().rstrip()
 320         if str(line) == '0':
 321             logger.debug("Tempest %s does not exist" %
 322                          CONST.__getattribute__('tempest_deployment_name'))
 323             cmd = ("rally verify create-verifier --source {0} "
 324                    "--name {1} --type tempest --system-wide"
 325                    .format(CONST.__getattribute__('dir_repo_tempest'),
 326                            CONST.__getattribute__('tempest_deployment_name')))
 327             error_msg = "Problem while installing Tempest."
 328             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 329
 330
 331 def create_flavor():
 332     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 333                                                  '512',
 334                                                  '1',
 335                                                  '1',
 336                                                  public=True)
 337     if flavor_id is None:
 338         raise Exception('Failed to create flavor')
 339
 340
 341 def check_environment():
 342     msg_not_active = "The Functest environment is not installed."
 343     if not os.path.isfile(CONST.__getattribute__('env_active')):
 344         raise Exception(msg_not_active)
 345
 346     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 347         s = env_file.read()
 348         if not re.search("1", s):
 349             raise Exception(msg_not_active)
 350
 351     logger.info("Functest environment is installed.")
 352
 353
 354 def print_deployment_info():
 355     if handler:
 356         logger.info('\n\nDeployment information:\n%s' %
 357                     handler.get_deployment_info())
 358
 359
 360 def main(**kwargs):
 361     try:
 362         if not (kwargs['action'] in actions):
 363             logger.error('Argument not valid.')
 364             return -1
 365         elif kwargs['action'] == "start":
 366             logger.info("######### Preparing Functest environment #########\n")
 367             check_env_variables()
 368             get_deployment_handler()
 369             create_directories()
 370             source_rc_file()
 371             update_config_file()
 372             verify_deployment()
 373             install_rally()
 374             install_tempest()
 375             create_flavor()
 376             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 377                 env_file.write("1")
 378             check_environment()
 379             print_deployment_info()
 380         elif kwargs['action'] == "check":
 381             check_environment()
 382     except Exception as e:
 383         logger.error(e)
 384         return -1
 385     return 0
 386
 387
 388 if __name__ == '__main__':
 389     logging.config.fileConfig(
 390         CONST.__getattribute__('dir_functest_logging_cfg'))
 391     parser = PrepareEnvParser()
 392     args = parser.parse_args(sys.argv[1:])
 393     sys.exit(main(**args))