functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML')
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.__getattribute__('INSTALLER_TYPE') is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  72     else:
  73         if (CONST.__getattribute__('INSTALLER_TYPE') not in
  74                 opnfv_constants.INSTALLERS):
  75             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  76                            "Available OPNFV Installers are : %s. "
  77                            "Setting INSTALLER_TYPE=undefined."
  78                            % (CONST.__getattribute__('INSTALLER_TYPE'),
  79                               opnfv_constants.INSTALLERS))
  80             CONST.__setattr__('INSTALLER_TYPE', 'undefined')
  81         else:
  82             logger.info("    INSTALLER_TYPE=%s"
  83                         % CONST.__getattribute__('INSTALLER_TYPE'))
  84
  85     if CONST.__getattribute__('INSTALLER_IP') is None:
  86         logger.warning(
  87             "The env variable 'INSTALLER_IP' is not defined. It is recommended"
  88             " to extract some information from the deployment")
  89     else:
  90         logger.info("    INSTALLER_IP=%s" %
  91                     CONST.__getattribute__('INSTALLER_IP'))
  92
  93     if CONST.__getattribute__('DEPLOY_SCENARIO') is None:
  94         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  95                        "Setting CI_SCENARIO=undefined.")
  96         CONST.__setattr__('DEPLOY_SCENARIO', 'undefined')
  97     else:
  98         logger.info("    DEPLOY_SCENARIO=%s"
  99                     % CONST.__getattribute__('DEPLOY_SCENARIO'))
 100     if CONST.__getattribute__('CI_DEBUG'):
 101         logger.info("    CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG'))
 102
 103     if CONST.__getattribute__('NODE_NAME'):
 104         logger.info("    NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME'))
 105
 106     if CONST.__getattribute__('BUILD_TAG'):
 107         logger.info("    BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG'))
 108
 109     if CONST.__getattribute__('IS_CI_RUN'):
 110         logger.info("    IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN'))
 111
 112
 113 def get_deployment_handler():
 114     global handler
 115     global pod_arch
 116
 117     installer_params_yaml = os.path.join(
 118         CONST.__getattribute__('dir_repo_functest'),
 119         'functest/ci/installer_params.yaml')
 120     if (CONST.__getattribute__('INSTALLER_IP') and
 121         CONST.__getattribute__('INSTALLER_TYPE') and
 122             CONST.__getattribute__('INSTALLER_TYPE') in
 123             opnfv_constants.INSTALLERS):
 124         try:
 125             installer_params = ft_utils.get_parameter_from_yaml(
 126                 CONST.__getattribute__('INSTALLER_TYPE'),
 127                 installer_params_yaml)
 128         except ValueError as e:
 129             logger.debug('Printing deployment info is not supported for %s' %
 130                          CONST.__getattribute__('INSTALLER_TYPE'))
 131             logger.debug(e)
 132         else:
 133             user = installer_params.get('user', None)
 134             password = installer_params.get('password', None)
 135             pkey = installer_params.get('pkey', None)
 136             try:
 137                 handler = factory.Factory.get_handler(
 138                     installer=CONST.__getattribute__('INSTALLER_TYPE'),
 139                     installer_ip=CONST.__getattribute__('INSTALLER_IP'),
 140                     installer_user=user,
 141                     installer_pwd=password,
 142                     pkey_file=pkey)
 143                 if handler:
 144                     pod_arch = handler.get_arch()
 145             except Exception as e:
 146                 logger.debug("Cannot get deployment information. %s" % e)
 147
 148
 149 def create_directories():
 150     print_separator()
 151     logger.info("Creating needed directories...")
 152     if not os.path.exists(CONST.__getattribute__('dir_functest_conf')):
 153         os.makedirs(CONST.__getattribute__('dir_functest_conf'))
 154         logger.info("    %s created." %
 155                     CONST.__getattribute__('dir_functest_conf'))
 156     else:
 157         logger.debug("   %s already exists." %
 158                      CONST.__getattribute__('dir_functest_conf'))
 159
 160     if not os.path.exists(CONST.__getattribute__('dir_functest_data')):
 161         os.makedirs(CONST.__getattribute__('dir_functest_data'))
 162         logger.info("    %s created." %
 163                     CONST.__getattribute__('dir_functest_data'))
 164     else:
 165         logger.debug("   %s already exists." %
 166                      CONST.__getattribute__('dir_functest_data'))
 167     if not os.path.exists(CONST.__getattribute__('dir_functest_images')):
 168         os.makedirs(CONST.__getattribute__('dir_functest_images'))
 169         logger.info("    %s created." %
 170                     CONST.__getattribute__('dir_functest_images'))
 171     else:
 172         logger.debug("   %s already exists." %
 173                      CONST.__getattribute__('dir_functest_images'))
 174
 175
 176 def source_rc_file():
 177     print_separator()
 178
 179     if CONST.__getattribute__('openstack_creds') is None:
 180         logger.warning("The environment variable 'creds' must be set and"
 181                        "pointing to the local RC file. Using default: "
 182                        "/home/opnfv/functest/conf/openstack.creds ...")
 183         os.path.join(
 184             CONST.__getattribute__('dir_functest_conf'), 'openstack.creds')
 185
 186     if not os.path.isfile(CONST.__getattribute__('openstack_creds')):
 187         raise Exception(
 188             "OpenStack credentials file not provided. "
 189             "The OpenStack credentials must be in {}"
 190             .format(CONST.__getattribute__('openstack_creds')))
 191     else:
 192         logger.info("RC file provided in %s."
 193                     % CONST.__getattribute__('openstack_creds'))
 194         if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0:
 195             raise Exception(
 196                 "The OpenStack RC file {} is empty."
 197                 .format(CONST.__getattribute__('openstack_creds')))
 198
 199     logger.info("Sourcing the OpenStack RC file...")
 200     os_utils.source_credentials(CONST.__getattribute__('openstack_creds'))
 201     for key, value in os.environ.iteritems():
 202         if re.search("OS_", key):
 203             if key == 'OS_AUTH_URL':
 204                 CONST.__setattr__('OS_AUTH_URL', value)
 205             elif key == 'OS_USERNAME':
 206                 CONST.__setattr__('OS_USERNAME', value)
 207             elif key == 'OS_TENANT_NAME':
 208                 CONST.__setattr__('OS_TENANT_NAME', value)
 209             elif key == 'OS_PASSWORD':
 210                 CONST.__setattr__('OS_PASSWORD', value)
 211
 212
 213 def patch_config_file():
 214     patch_file(CONFIG_PATCH_PATH)
 215
 216     if pod_arch and pod_arch in arch_filter:
 217         patch_file(CONFIG_AARCH64_PATCH_PATH)
 218
 219
 220 def patch_file(patch_file_path):
 221     logger.debug('Updating file: %s', patch_file_path)
 222     with open(patch_file_path) as f:
 223         patch_file = yaml.safe_load(f)
 224
 225     updated = False
 226     for key in patch_file:
 227         if key in CONST.__getattribute__('DEPLOY_SCENARIO'):
 228             new_functest_yaml = dict(ft_utils.merge_dicts(
 229                 ft_utils.get_functest_yaml(), patch_file[key]))
 230             updated = True
 231
 232     if updated:
 233         os.remove(CONFIG_FUNCTEST_PATH)
 234         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 235             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 236         f.close()
 237
 238
 239 def verify_deployment():
 240     print_separator()
 241     logger.info("Verifying OpenStack services...")
 242     cmd = ("%s/functest/ci/check_os.sh" %
 243            CONST.__getattribute__('dir_repo_functest'))
 244
 245     logger.debug("Executing command: %s" % cmd)
 246     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 247
 248     while p.poll() is None:
 249         line = p.stdout.readline().rstrip()
 250         if "ERROR" in line:
 251             logger.error(line)
 252             raise Exception("Problem while running 'check_os.sh'.")
 253         logger.info(line)
 254
 255
 256 def install_rally():
 257     print_separator()
 258
 259     if pod_arch and pod_arch in arch_filter:
 260         logger.info("Apply aarch64 specific to rally config...")
 261         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 262             rally_patch_conf = f.read()
 263
 264         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 265             print line,
 266             if "cirros|testvm" in line:
 267                 print rally_patch_conf
 268
 269     logger.info("Creating Rally environment...")
 270
 271     cmd = "rally deployment destroy opnfv-rally"
 272     ft_utils.execute_command(cmd, error_msg=(
 273         "Deployment %s does not exist."
 274         % CONST.__getattribute__('rally_deployment_name')),
 275         verbose=False)
 276
 277     rally_conf = os_utils.get_credentials_for_rally()
 278     with open('rally_conf.json', 'w') as fp:
 279         json.dump(rally_conf, fp)
 280     cmd = ("rally deployment create "
 281            "--file=rally_conf.json --name={0}"
 282            .format(CONST.__getattribute__('rally_deployment_name')))
 283     error_msg = "Problem while creating Rally deployment"
 284     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 285
 286     cmd = "rally deployment check"
 287     error_msg = "OpenStack not responding or faulty Rally deployment."
 288     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 289
 290     cmd = "rally deployment list"
 291     ft_utils.execute_command(cmd,
 292                              error_msg=("Problem while listing "
 293                                         "Rally deployment."))
 294
 295     cmd = "rally plugin list | head -5"
 296     ft_utils.execute_command(cmd,
 297                              error_msg=("Problem while showing "
 298                                         "Rally plugins."))
 299
 300
 301 def install_tempest():
 302     logger.info("Installing tempest from existing repo...")
 303     cmd = ("rally verify list-verifiers | "
 304            "grep '{0}' | wc -l".format(
 305                CONST.__getattribute__('tempest_deployment_name')))
 306     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 307     while p.poll() is None:
 308         line = p.stdout.readline().rstrip()
 309         if str(line) == '0':
 310             logger.debug("Tempest %s does not exist" %
 311                          CONST.__getattribute__('tempest_deployment_name'))
 312             cmd = ("rally verify create-verifier --source {0} "
 313                    "--name {1} --type tempest --system-wide"
 314                    .format(CONST.__getattribute__('dir_repo_tempest'),
 315                            CONST.__getattribute__('tempest_deployment_name')))
 316             error_msg = "Problem while installing Tempest."
 317             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 318
 319
 320 def create_flavor():
 321     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 322                                                  '512',
 323                                                  '1',
 324                                                  '1',
 325                                                  public=True)
 326     if flavor_id is None:
 327         raise Exception('Failed to create flavor')
 328
 329
 330 def check_environment():
 331     msg_not_active = "The Functest environment is not installed."
 332     if not os.path.isfile(CONST.__getattribute__('env_active')):
 333         raise Exception(msg_not_active)
 334
 335     with open(CONST.__getattribute__('env_active'), "r") as env_file:
 336         s = env_file.read()
 337         if not re.search("1", s):
 338             raise Exception(msg_not_active)
 339
 340     logger.info("Functest environment is installed.")
 341
 342
 343 def print_deployment_info():
 344     if handler:
 345         logger.info('\n\nDeployment information:\n%s' %
 346                     handler.get_deployment_info())
 347
 348
 349 def main(**kwargs):
 350     try:
 351         if not (kwargs['action'] in actions):
 352             logger.error('Argument not valid.')
 353             return -1
 354         elif kwargs['action'] == "start":
 355             logger.info("######### Preparing Functest environment #########\n")
 356             check_env_variables()
 357             get_deployment_handler()
 358             create_directories()
 359             source_rc_file()
 360             patch_config_file()
 361             verify_deployment()
 362             install_rally()
 363             install_tempest()
 364             create_flavor()
 365             with open(CONST.__getattribute__('env_active'), "w") as env_file:
 366                 env_file.write("1")
 367             check_environment()
 368             print_deployment_info()
 369         elif kwargs['action'] == "check":
 370             check_environment()
 371     except Exception as e:
 372         logger.error(e)
 373         return -1
 374     return 0
 375
 376
 377 if __name__ == '__main__':
 378     logging.config.fileConfig(
 379         CONST.__getattribute__('dir_functest_logging_cfg'))
 380     parser = PrepareEnvParser()
 381     args = parser.parse_args(sys.argv[1:])
 382     sys.exit(main(**args))