functest/ci/prepare_env.py

   1 #!/usr/bin/env python
   2 #
   3 # All rights reserved. This program and the accompanying materials
   4 # are made available under the terms of the Apache License, Version 2.0
   5 # which accompanies this distribution, and is available at
   6 # http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8
   9 import argparse
  10 import json
  11 import logging
  12 import logging.config
  13 import os
  14 import re
  15 import subprocess
  16 import sys
  17 import fileinput
  18
  19 import yaml
  20
  21 import functest.utils.functest_utils as ft_utils
  22 import functest.utils.openstack_utils as os_utils
  23 from functest.utils.constants import CONST
  24
  25 from opnfv.utils import constants as opnfv_constants
  26 from opnfv.deployment import factory
  27
  28 actions = ['start', 'check']
  29
  30 """ logging configuration """
  31 logger = logging.getLogger('functest.ci.prepare_env')
  32 handler = None
  33 # set the architecture to default
  34 pod_arch = None
  35 arch_filter = ['aarch64']
  36
  37 CONFIG_FUNCTEST_PATH = CONST.CONFIG_FUNCTEST_YAML
  38 CONFIG_PATCH_PATH = os.path.join(os.path.dirname(
  39     CONFIG_FUNCTEST_PATH), "config_patch.yaml")
  40 CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  41     CONFIG_FUNCTEST_PATH), "config_aarch64_patch.yaml")
  42 RALLY_CONF_PATH = os.path.join("/etc/rally/rally.conf")
  43 RALLY_AARCH64_PATCH_PATH = os.path.join(os.path.dirname(
  44     CONFIG_FUNCTEST_PATH), "rally_aarch64_patch.conf")
  45
  46
  47 class PrepareEnvParser(object):
  48
  49     def __init__(self):
  50         self.parser = argparse.ArgumentParser()
  51         self.parser.add_argument("action", help="Possible actions are: "
  52                                  "'{d[0]}|{d[1]}' ".format(d=actions),
  53                                  choices=actions)
  54         self.parser.add_argument("-d", "--debug", help="Debug mode",
  55                                  action="store_true")
  56
  57     def parse_args(self, argv=[]):
  58         return vars(self.parser.parse_args(argv))
  59
  60
  61 def print_separator():
  62     logger.info("==============================================")
  63
  64
  65 def check_env_variables():
  66     print_separator()
  67     logger.info("Checking environment variables...")
  68
  69     if CONST.INSTALLER_TYPE is None:
  70         logger.warning("The env variable 'INSTALLER_TYPE' is not defined.")
  71         CONST.INSTALLER_TYPE = "undefined"
  72     else:
  73         if CONST.INSTALLER_TYPE not in opnfv_constants.INSTALLERS:
  74             logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. "
  75                            "Available OPNFV Installers are : %s. "
  76                            "Setting INSTALLER_TYPE=undefined."
  77                            % (CONST.INSTALLER_TYPE,
  78                               opnfv_constants.INSTALLERS))
  79             CONST.INSTALLER_TYPE = "undefined"
  80         else:
  81             logger.info("    INSTALLER_TYPE=%s"
  82                         % CONST.INSTALLER_TYPE)
  83
  84     if CONST.INSTALLER_IP is None:
  85         logger.warning("The env variable 'INSTALLER_IP' is not defined. "
  86                        "It is needed to fetch the OpenStack credentials. "
  87                        "If the credentials are not provided to the "
  88                        "container as a volume, please add this env variable "
  89                        "to the 'docker run' command.")
  90     else:
  91         logger.info("    INSTALLER_IP=%s" % CONST.INSTALLER_IP)
  92
  93     if CONST.DEPLOY_SCENARIO is None:
  94         logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. "
  95                        "Setting CI_SCENARIO=undefined.")
  96         CONST.DEPLOY_SCENARIO = "undefined"
  97     else:
  98         logger.info("    DEPLOY_SCENARIO=%s" % CONST.DEPLOY_SCENARIO)
  99     if CONST.CI_DEBUG:
 100         logger.info("    CI_DEBUG=%s" % CONST.CI_DEBUG)
 101
 102     if CONST.NODE_NAME:
 103         logger.info("    NODE_NAME=%s" % CONST.NODE_NAME)
 104
 105     if CONST.BUILD_TAG:
 106         logger.info("    BUILD_TAG=%s" % CONST.BUILD_TAG)
 107
 108     if CONST.IS_CI_RUN:
 109         logger.info("    IS_CI_RUN=%s" % CONST.IS_CI_RUN)
 110
 111
 112 def get_deployment_handler():
 113     global handler
 114     global pod_arch
 115
 116     installer_params_yaml = os.path.join(CONST.dir_repo_functest,
 117                                          'functest/ci/installer_params.yaml')
 118     if (CONST.INSTALLER_IP and CONST.INSTALLER_TYPE and
 119             CONST.INSTALLER_TYPE in opnfv_constants.INSTALLERS):
 120         try:
 121             installer_params = ft_utils.get_parameter_from_yaml(
 122                 CONST.INSTALLER_TYPE, installer_params_yaml)
 123         except ValueError as e:
 124             logger.debug('Printing deployment info is not supported for %s' %
 125                          CONST.INSTALLER_TYPE)
 126             logger.debug(e)
 127         else:
 128             user = installer_params.get('user', None)
 129             password = installer_params.get('password', None)
 130             pkey = installer_params.get('pkey', None)
 131             try:
 132                 handler = factory.Factory.get_handler(
 133                     installer=CONST.INSTALLER_TYPE,
 134                     installer_ip=CONST.INSTALLER_IP,
 135                     installer_user=user,
 136                     installer_pwd=password,
 137                     pkey_file=pkey)
 138                 if handler:
 139                     pod_arch = handler.get_arch()
 140             except Exception as e:
 141                 logger.debug("Cannot get deployment information. %s" % e)
 142
 143
 144 def create_directories():
 145     print_separator()
 146     logger.info("Creating needed directories...")
 147     if not os.path.exists(CONST.dir_functest_conf):
 148         os.makedirs(CONST.dir_functest_conf)
 149         logger.info("    %s created." % CONST.dir_functest_conf)
 150     else:
 151         logger.debug("   %s already exists."
 152                      % CONST.dir_functest_conf)
 153
 154     if not os.path.exists(CONST.dir_functest_data):
 155         os.makedirs(CONST.dir_functest_data)
 156         logger.info("    %s created." % CONST.dir_functest_data)
 157     else:
 158         logger.debug("   %s already exists."
 159                      % CONST.dir_functest_data)
 160
 161
 162 def source_rc_file():
 163     print_separator()
 164     logger.info("Fetching RC file...")
 165
 166     if CONST.openstack_creds is None:
 167         logger.warning("The environment variable 'creds' must be set and"
 168                        "pointing to the local RC file. Using default: "
 169                        "/home/opnfv/functest/conf/openstack.creds ...")
 170         os.path.join(CONST.dir_functest_conf, 'openstack.creds')
 171
 172     if not os.path.isfile(CONST.openstack_creds):
 173         logger.info("RC file not provided. "
 174                     "Fetching it from the installer...")
 175         if CONST.INSTALLER_IP is None:
 176             logger.error("The env variable CI_INSTALLER_IP must be provided in"
 177                          " order to fetch the credentials from the installer.")
 178             raise Exception("Missing CI_INSTALLER_IP.")
 179         if CONST.INSTALLER_TYPE not in opnfv_constants.INSTALLERS:
 180             logger.error("Cannot fetch credentials. INSTALLER_TYPE=%s is "
 181                          "not a valid OPNFV installer. Available "
 182                          "installers are : %s." %
 183                          (CONST.INSTALLER_TYPE,
 184                           opnfv_constants.INSTALLERS))
 185             raise Exception("Wrong INSTALLER_TYPE.")
 186
 187         cmd = ("/home/opnfv/repos/releng/utils/fetch_os_creds.sh "
 188                "-d %s -i %s -a %s"
 189                % (CONST.openstack_creds,
 190                   CONST.INSTALLER_TYPE,
 191                   CONST.INSTALLER_IP))
 192         logger.debug("Executing command: %s" % cmd)
 193         p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
 194         output = p.communicate()[0]
 195         logger.debug("\n%s" % output)
 196         if p.returncode != 0:
 197             raise Exception("Failed to fetch credentials from installer.")
 198     else:
 199         logger.info("RC file provided in %s."
 200                     % CONST.openstack_creds)
 201         if os.path.getsize(CONST.openstack_creds) == 0:
 202             raise Exception("The file %s is empty." % CONST.openstack_creds)
 203
 204     logger.info("Sourcing the OpenStack RC file...")
 205     os_utils.source_credentials(CONST.openstack_creds)
 206     for key, value in os.environ.iteritems():
 207         if re.search("OS_", key):
 208             if key == 'OS_AUTH_URL':
 209                 CONST.OS_AUTH_URL = value
 210             elif key == 'OS_USERNAME':
 211                 CONST.OS_USERNAME = value
 212             elif key == 'OS_TENANT_NAME':
 213                 CONST.OS_TENANT_NAME = value
 214             elif key == 'OS_PASSWORD':
 215                 CONST.OS_PASSWORD = value
 216
 217
 218 def patch_config_file():
 219     patch_file(CONFIG_PATCH_PATH)
 220
 221     if pod_arch and pod_arch in arch_filter:
 222         patch_file(CONFIG_AARCH64_PATCH_PATH)
 223
 224
 225 def patch_file(patch_file_path):
 226     logger.debug('Updating file: %s', patch_file_path)
 227     with open(patch_file_path) as f:
 228         patch_file = yaml.safe_load(f)
 229
 230     updated = False
 231     for key in patch_file:
 232         if key in CONST.DEPLOY_SCENARIO:
 233             new_functest_yaml = dict(ft_utils.merge_dicts(
 234                 ft_utils.get_functest_yaml(), patch_file[key]))
 235             updated = True
 236
 237     if updated:
 238         os.remove(CONFIG_FUNCTEST_PATH)
 239         with open(CONFIG_FUNCTEST_PATH, "w") as f:
 240             f.write(yaml.dump(new_functest_yaml, default_style='"'))
 241         f.close()
 242
 243
 244 def verify_deployment():
 245     print_separator()
 246     logger.info("Verifying OpenStack services...")
 247     cmd = ("%s/functest/ci/check_os.sh" % CONST.dir_repo_functest)
 248
 249     logger.debug("Executing command: %s" % cmd)
 250     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 251
 252     while p.poll() is None:
 253         line = p.stdout.readline().rstrip()
 254         if "ERROR" in line:
 255             logger.error(line)
 256             raise Exception("Problem while running 'check_os.sh'.")
 257         logger.info(line)
 258
 259
 260 def install_rally():
 261     print_separator()
 262
 263     if pod_arch and pod_arch in arch_filter:
 264         logger.info("Apply aarch64 specific to rally config...")
 265         with open(RALLY_AARCH64_PATCH_PATH, "r") as f:
 266             rally_patch_conf = f.read()
 267
 268         for line in fileinput.input(RALLY_CONF_PATH, inplace=1):
 269             print line,
 270             if "cirros|testvm" in line:
 271                 print rally_patch_conf
 272
 273     logger.info("Creating Rally environment...")
 274
 275     cmd = "rally deployment destroy opnfv-rally"
 276     ft_utils.execute_command(cmd, error_msg=(
 277         "Deployment %s does not exist."
 278         % CONST.rally_deployment_name),
 279         verbose=False)
 280
 281     rally_conf = os_utils.get_credentials_for_rally()
 282     with open('rally_conf.json', 'w') as fp:
 283         json.dump(rally_conf, fp)
 284     cmd = ("rally deployment create "
 285            "--file=rally_conf.json --name={0}"
 286            .format(CONST.rally_deployment_name))
 287     error_msg = "Problem while creating Rally deployment"
 288     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 289
 290     cmd = "rally deployment check"
 291     error_msg = "OpenStack not responding or faulty Rally deployment."
 292     ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 293
 294     cmd = "rally deployment list"
 295     ft_utils.execute_command(cmd,
 296                              error_msg=("Problem while listing "
 297                                         "Rally deployment."))
 298
 299     cmd = "rally plugin list | head -5"
 300     ft_utils.execute_command(cmd,
 301                              error_msg=("Problem while showing "
 302                                         "Rally plugins."))
 303
 304
 305 def install_tempest():
 306     logger.info("Installing tempest from existing repo...")
 307     cmd = ("rally verify list-verifiers | "
 308            "grep '{0}' | wc -l".format(CONST.tempest_deployment_name))
 309     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)
 310     while p.poll() is None:
 311         line = p.stdout.readline().rstrip()
 312         if str(line) == '0':
 313             logger.debug("Tempest %s does not exist" %
 314                          CONST.tempest_deployment_name)
 315             cmd = ("rally verify create-verifier --source {0} "
 316                    "--name {1} --type tempest --system-wide"
 317                    .format(CONST.dir_repo_tempest,
 318                            CONST.tempest_deployment_name))
 319             error_msg = "Problem while installing Tempest."
 320             ft_utils.execute_command_raise(cmd, error_msg=error_msg)
 321
 322
 323 def create_flavor():
 324     _, flavor_id = os_utils.get_or_create_flavor('m1.tiny',
 325                                                  '512',
 326                                                  '1',
 327                                                  '1',
 328                                                  public=True)
 329     if flavor_id is None:
 330         raise Exception('Failed to create flavor')
 331
 332
 333 def check_environment():
 334     msg_not_active = "The Functest environment is not installed."
 335     if not os.path.isfile(CONST.env_active):
 336         raise Exception(msg_not_active)
 337
 338     with open(CONST.env_active, "r") as env_file:
 339         s = env_file.read()
 340         if not re.search("1", s):
 341             raise Exception(msg_not_active)
 342
 343     logger.info("Functest environment is installed.")
 344
 345
 346 def print_deployment_info():
 347     if handler:
 348         logger.info('\n\nDeployment information:\n%s' %
 349                     handler.get_deployment_info())
 350
 351
 352 def main(**kwargs):
 353     try:
 354         if not (kwargs['action'] in actions):
 355             logger.error('Argument not valid.')
 356             return -1
 357         elif kwargs['action'] == "start":
 358             logger.info("######### Preparing Functest environment #########\n")
 359             check_env_variables()
 360             get_deployment_handler()
 361             create_directories()
 362             source_rc_file()
 363             patch_config_file()
 364             verify_deployment()
 365             install_rally()
 366             install_tempest()
 367             create_flavor()
 368             with open(CONST.env_active, "w") as env_file:
 369                 env_file.write("1")
 370             check_environment()
 371             print_deployment_info()
 372         elif kwargs['action'] == "check":
 373             check_environment()
 374     except Exception as e:
 375         logger.error(e)
 376         return -1
 377     return 0
 378
 379
 380 if __name__ == '__main__':
 381     logging.config.fileConfig(
 382         CONST.__getattribute__('dir_functest_logging_cfg'))
 383     parser = PrepareEnvParser()
 384     args = parser.parse_args(sys.argv[1:])
 385     sys.exit(main(**args))